Merge pull request #248 from crazy-max/buildkit-secrets

build/bake: secrets support

Author: crazy-max

.github/workflows/.test-bake.yml

@@ -477,6 +477,27 @@ jobs:
             const builderOutputs = JSON.parse(core.getInput('builder-outputs'));
             core.info(JSON.stringify(builderOutputs, null, 2));
 
+  bake-secret:
+    uses: ./.github/workflows/bake.yml
+    permissions:
+      contents: read
+      id-token: write
+    with:
+      artifact-upload: false
+      context: test
+      output: local
+      target: foosec
+    secrets:
+      build-secrets: |+
+        fixture.plain: |
+          alpha-line
+          beta-line
+        foosec:
+          fixture.json: ${{ toJSON(format('gamma-line{0}delta-line{0}', fromJSON('"\n"'))) }}
+          fixture_fuu: standard-secret
+          fixture_keep: |+
+            keep-line
+
   bake-set-runner:
     uses: ./.github/workflows/bake.yml
     permissions:

.github/workflows/.test-build.yml

@@ -523,6 +523,25 @@ jobs:
             const builderOutputs = JSON.parse(core.getInput('builder-outputs'));
             core.info(JSON.stringify(builderOutputs, null, 2));
 
+  build-secret:
+    uses: ./.github/workflows/build.yml
+    permissions:
+      contents: read
+      id-token: write
+    with:
+      artifact-upload: false
+      file: test/secret.Dockerfile
+      output: local
+    secrets:
+      build-secrets: |+
+        fixture.plain: |
+          alpha-line
+          beta-line
+        fixture.json: ${{ toJSON(format('gamma-line{0}delta-line{0}', fromJSON('"\n"'))) }}
+        fixture_fuu: standard-secret
+        fixture_keep: |+
+          keep-line
+
   build-set-runner:
     uses: ./.github/workflows/build.yml
     permissions:

.github/workflows/bake.yml

@@ -149,6 +149,9 @@ on:
       registry-auths:
         description: "Raw authentication to registries, defined as YAML objects (for image output)"
         required: false
+      build-secrets:
+        description: "YAML object mapping BuildKit secret IDs to values, with optional nested target mappings"
+        required: false
       github-token:
         description: "GitHub Token used to authenticate against the repository for Git context"
         required: false
@@ -215,6 +218,7 @@ jobs:
       metaImages: ${{ steps.set.outputs.metaImages }}
       sign: ${{ steps.set.outputs.sign }}
       privateRepo: ${{ steps.set.outputs.privateRepo }}
+      targets: ${{ steps.set.outputs.targets }}
       ghaCacheSign: ${{ steps.set.outputs.ghaCacheSign }}
       proxyNetwork: ${{ steps.set.outputs.proxyNetwork }}
     steps:
@@ -400,7 +404,7 @@ jobs:
               }
             );
             await core.group(`Set envs`, async () => {
-              core.info(JSON.stringify(envs, null, 2));
+              core.info(JSON.stringify(Object.keys(envs).sort(), null, 2));
             });
 
             const metaImages = inpMetaImages.map(image => image.toLowerCase());
@@ -414,9 +418,15 @@ jobs:
             try {
               await core.group(`Validating definition`, async () => {
                 const bake = new Bake();
+                // Resolve the graph without local secret files. The build job validates
+                // secrets after applying the workflow-provided source overrides.
+                const validationOverrides = inpSet.filter(override => {
+                  const key = override.split('=', 1)[0].split('.')[1];
+                  return !['secret', 'secrets', 'secrets+'].includes(key);
+                });
                 def = await bake.getDefinition({
                   files: inpFiles,
-                  overrides: inpSet,
+                  overrides: [...validationOverrides, '*.secrets='],
                   sbom: inpSbom ? `generator=${inpSbomImage}` : 'false',
                   source: bakeSource,
                   targets: [inpTarget]
@@ -426,8 +436,9 @@ jobs:
                 if (!def) {
                   throw new Error('Bake definition not set');
                 }
-                BakeTargets.resolve(def, inpTarget);
+                const targets = BakeTargets.resolve(def, inpTarget);
                 target = inpTarget;
+                core.setOutput('targets', JSON.stringify(targets));
               });
             } catch (error) {
               core.setFailed(error);
@@ -710,6 +721,82 @@ jobs:
               cosignPath,
               `${containerName}:/usr/bin/cosign`
             ]);
+      -
+        name: Configure AWS credentials
+        if: ${{ needs.registry-identities.outputs.aws-ecr-enabled == 'true' }}
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ needs.registry-identities.outputs.aws-ecr-role-to-assume }}
+          aws-region: ${{ needs.registry-identities.outputs.aws-ecr-region }}
+      -
+        name: Login to Amazon ECR
+        if: ${{ needs.registry-identities.outputs.aws-ecr-enabled == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        env:
+          AWS_ACCOUNT_IDS: ${{ needs.registry-identities.outputs.aws-ecr-account-ids }}
+        with:
+          registry-auth: |
+            - registry: ${{ needs.registry-identities.outputs.aws-ecr-registry }}
+      -
+        name: Authenticate to Google Cloud
+        id: gcp-wif-auth
+        if: ${{ needs.registry-identities.outputs.gcp-wif-enabled == 'true' }}
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ needs.registry-identities.outputs.gcp-wif-workload-identity-provider }}
+          service_account: ${{ needs.registry-identities.outputs.gcp-wif-service-account }}
+          project_id: ${{ needs.registry-identities.outputs.gcp-wif-project-id }}
+          create_credentials_file: false
+          export_environment_variables: false
+      -
+        name: Login to Google Artifact Registry
+        if: ${{ needs.registry-identities.outputs.gcp-wif-enabled == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry-auth: |
+            - registry: ${{ needs.registry-identities.outputs.gcp-wif-registry }}
+              username: oauth2accesstoken
+              password: ${{ steps.gcp-wif-auth.outputs.access_token }}
+      -
+        name: Login to Docker Hub with OIDC
+        if: ${{ needs.registry-identities.outputs.dockerhub-oidc-enabled == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        env:
+          DOCKERHUB_OIDC_CONNECTIONID: ${{ needs.registry-identities.outputs.dockerhub-oidc-connection-id }}
+        with:
+          registry-auth: |
+            - registry: ${{ needs.registry-identities.outputs.dockerhub-oidc-registry }}
+              username: ${{ needs.registry-identities.outputs.dockerhub-oidc-username }}
+      -
+        name: Authenticate to Azure
+        if: ${{ needs.registry-identities.outputs.azure-acr-enabled == 'true' }}
+        uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
+        with:
+          client-id: ${{ needs.registry-identities.outputs.azure-acr-client-id }}
+          tenant-id: ${{ needs.registry-identities.outputs.azure-acr-tenant-id }}
+          subscription-id: ${{ needs.registry-identities.outputs.azure-acr-subscription-id }}
+      -
+        name: Login to Azure Container Registry
+        if: ${{ needs.registry-identities.outputs.azure-acr-enabled == 'true' }}
+        env:
+          ACR_REGISTRY: ${{ needs.registry-identities.outputs.azure-acr-registry }}
+          AZURE_CORE_OUTPUT: none
+        run: az acr login --name "${ACR_REGISTRY%.azurecr.io}"
+      -
+        name: Set up chainctl
+        if: ${{ needs.registry-identities.outputs.chainguard-enabled == 'true' }}
+        uses: chainguard-dev/setup-chainctl@2cddd35a2f120d9973e58094dc6878c93cf58c28 # v0.5.1
+        with:
+          identity: ${{ needs.registry-identities.outputs.chainguard-identity }}
+          apk-host: ${{ needs.registry-identities.outputs.chainguard-apk-host }}
+          libraries-host: ${{ needs.registry-identities.outputs.chainguard-libraries-host }}
+      -
+        name: Login to registry
+        if: ${{ inputs.push && inputs.output == 'image' && env.REGISTRY_AUTHS_PRESENT == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry-auth: ${{ secrets.registry-auths }}
       -
         name: Prepare
         id: prepare
@@ -721,6 +808,8 @@ jobs:
           INPUT_CACHE: ${{ inputs.cache }}
           INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }}
           INPUT_CACHE-MODE: ${{ inputs.cache-mode }}
+          INPUT_BUILD-SECRETS: ${{ secrets.build-secrets }}
+          INPUT_TARGETS: ${{ needs.prepare.outputs.targets }}
           INPUT_CONTEXT: ${{ inputs.context }}
           INPUT_FILES: ${{ inputs.files }}
           INPUT_OUTPUT: ${{ inputs.output }}
@@ -737,15 +826,15 @@ jobs:
           INPUT_BAKE-FILE-TAGS: ${{ steps.meta.outputs.bake-file-tags }}
           INPUT_BAKE-FILE-ANNOTATIONS: ${{ steps.meta.outputs.bake-file-annotations }}
           INPUT_BAKE-FILE-LABELS: ${{ steps.meta.outputs.bake-file-labels }}
-          INPUT_GITHUB-TOKEN: ${{ secrets.github-token || github.token }}
           INPUT_BUILDKIT-PROXY-NETWORK: ${{ inputs.buildkit-proxy-network }}
         with:
           script: |
             const os = require('os');
             const { Build } = require('@docker/github-builder-runtime/lib/buildx/build');
+            const { BuildSecrets } = require('@docker/github-builder-runtime/lib/github-builder/build-secrets');
             const { GitHub } = require('@docker/github-builder-runtime/lib/github/github');
             const { Util } = require('@docker/github-builder-runtime/lib/util');
-            
+
             const inpPlatform = core.getInput('platform');
             const platformPairSuffix = inpPlatform ? `-${inpPlatform.replace(/\//g, '-')}` : '';
             core.setOutput('platform-pair-suffix', platformPairSuffix);
@@ -756,6 +845,8 @@ jobs:
             const inpCache = core.getBooleanInput('cache');
             const inpCacheScope = core.getInput('cache-scope');
             const inpCacheMode = core.getInput('cache-mode');
+            const inpBuildSecrets = core.getInput('build-secrets', {trimWhitespace: false});
+            const inpTargets = core.getInput('targets');
             const inpContext = core.getInput('context');
             const inpFiles = Util.getInputList('files');
             const inpOutput = core.getInput('output');
@@ -764,7 +855,6 @@ jobs:
             const inpSet = Util.getInputList('set', {ignoreComma: true, quote: false});
             const inpTarget = core.getInput('target');
             const inpVars = Util.getInputList('vars');
-            const inpGitHubToken = core.getInput('github-token');
             const inpBuildkitProxyNetwork = core.getBooleanInput('buildkit-proxy-network');
 
             const inpMetaImages = core.getMultilineInput('meta-images');
@@ -781,7 +871,7 @@ jobs:
               tags: inpMetaTags
             };
             const renderTemplate = value => Util.compileHandlebars(value, {noEscape: true}, {meta});
-            
+
             const gitContextAttrs = GitHub.context.ref.startsWith('refs/tags/') ? {checksum: GitHub.context.sha} : {'fetch-by-commit': 'true'};
             const bakeSource = await new Build().gitContext({subdir: inpContext, attrs: gitContextAttrs});
             await core.group(`Set source output`, async () => {
@@ -799,7 +889,16 @@ jobs:
               core.info(sbom);
               core.setOutput('sbom', sbom);
             });
-            
+
+            let buildSecrets;
+            try {
+              buildSecrets = BuildSecrets.prepareBake(inpBuildSecrets, inpTarget, JSON.parse(inpTargets || '[]'));
+            } catch (err) {
+              core.setFailed(err.message);
+              return;
+            }
+            core.setOutput('secret-dir', buildSecrets.directory);
+
             const envs = Object.assign({},
               inpVars ? inpVars.reduce((acc, curr) => {
                 const idx = curr.indexOf('=');
@@ -809,12 +908,15 @@ jobs:
                 return acc;
               }, {}) : {},
               {
-                BUILDKIT_MULTI_PLATFORM: '1',
-                BUILDX_BAKE_GIT_AUTH_TOKEN: inpGitHubToken
+                BUILDKIT_MULTI_PLATFORM: '1'
               }
             );
+
+            // Git authentication is supplied directly to the Bake action.
+            delete envs.BUILDX_BAKE_GIT_AUTH_TOKEN;
+
             await core.group(`Set envs`, async () => {
-              core.info(JSON.stringify(envs, null, 2));
+              core.info(JSON.stringify(Object.keys(envs).sort(), null, 2));
               core.setOutput('envs', JSON.stringify(envs));
             });
             
@@ -878,85 +980,10 @@ jobs:
                 bakeOverrides.push(`*.cache-from=type=gha,scope=${inpCacheScope || inpTarget}${platformPairSuffix}${proxyNetworkSuffix}`);
                 bakeOverrides.push(`*.cache-to=type=gha,ignore-error=true,scope=${inpCacheScope || inpTarget}${platformPairSuffix}${proxyNetworkSuffix},mode=${inpCacheMode}`);
               }
+              bakeOverrides.push(...buildSecrets.inputs);
               core.info(JSON.stringify(bakeOverrides, null, 2));
               core.setOutput('overrides', bakeOverrides.join(os.EOL));
             });
-      -
-        name: Configure AWS credentials
-        if: ${{ needs.registry-identities.outputs.aws-ecr-enabled == 'true' }}
-        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
-        with:
-          role-to-assume: ${{ needs.registry-identities.outputs.aws-ecr-role-to-assume }}
-          aws-region: ${{ needs.registry-identities.outputs.aws-ecr-region }}
-      -
-        name: Login to Amazon ECR
-        if: ${{ needs.registry-identities.outputs.aws-ecr-enabled == 'true' }}
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        env:
-          AWS_ACCOUNT_IDS: ${{ needs.registry-identities.outputs.aws-ecr-account-ids }}
-        with:
-          registry-auth: |
-            - registry: ${{ needs.registry-identities.outputs.aws-ecr-registry }}
-      -
-        name: Authenticate to Google Cloud
-        id: gcp-wif-auth
-        if: ${{ needs.registry-identities.outputs.gcp-wif-enabled == 'true' }}
-        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
-        with:
-          token_format: access_token
-          workload_identity_provider: ${{ needs.registry-identities.outputs.gcp-wif-workload-identity-provider }}
-          service_account: ${{ needs.registry-identities.outputs.gcp-wif-service-account }}
-          project_id: ${{ needs.registry-identities.outputs.gcp-wif-project-id }}
-          create_credentials_file: false
-          export_environment_variables: false
-      -
-        name: Login to Google Artifact Registry
-        if: ${{ needs.registry-identities.outputs.gcp-wif-enabled == 'true' }}
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry-auth: |
-            - registry: ${{ needs.registry-identities.outputs.gcp-wif-registry }}
-              username: oauth2accesstoken
-              password: ${{ steps.gcp-wif-auth.outputs.access_token }}
-      -
-        name: Login to Docker Hub with OIDC
-        if: ${{ needs.registry-identities.outputs.dockerhub-oidc-enabled == 'true' }}
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        env:
-          DOCKERHUB_OIDC_CONNECTIONID: ${{ needs.registry-identities.outputs.dockerhub-oidc-connection-id }}
-        with:
-          registry-auth: |
-            - registry: ${{ needs.registry-identities.outputs.dockerhub-oidc-registry }}
-              username: ${{ needs.registry-identities.outputs.dockerhub-oidc-username }}
-      -
-        name: Authenticate to Azure
-        if: ${{ needs.registry-identities.outputs.azure-acr-enabled == 'true' }}
-        uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
-        with:
-          client-id: ${{ needs.registry-identities.outputs.azure-acr-client-id }}
-          tenant-id: ${{ needs.registry-identities.outputs.azure-acr-tenant-id }}
-          subscription-id: ${{ needs.registry-identities.outputs.azure-acr-subscription-id }}
-      -
-        name: Login to Azure Container Registry
-        if: ${{ needs.registry-identities.outputs.azure-acr-enabled == 'true' }}
-        env:
-          ACR_REGISTRY: ${{ needs.registry-identities.outputs.azure-acr-registry }}
-          AZURE_CORE_OUTPUT: none
-        run: az acr login --name "${ACR_REGISTRY%.azurecr.io}"
-      -
-        name: Set up chainctl
-        if: ${{ needs.registry-identities.outputs.chainguard-enabled == 'true' }}
-        uses: chainguard-dev/setup-chainctl@2cddd35a2f120d9973e58094dc6878c93cf58c28 # v0.5.1
-        with:
-          identity: ${{ needs.registry-identities.outputs.chainguard-identity }}
-          apk-host: ${{ needs.registry-identities.outputs.chainguard-apk-host }}
-          libraries-host: ${{ needs.registry-identities.outputs.chainguard-libraries-host }}
-      -
-        name: Login to registry
-        if: ${{ inputs.push && inputs.output == 'image' && env.REGISTRY_AUTHS_PRESENT == 'true' }}
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry-auth: ${{ secrets.registry-auths }}
       -
         name: Build
         id: bake
@@ -967,7 +994,18 @@ jobs:
           targets: ${{ steps.prepare.outputs.target }}
           sbom: ${{ steps.prepare.outputs.sbom }}
           set: ${{ steps.prepare.outputs.overrides }}
+          github-token: ${{ secrets.github-token || github.token }}
         env: ${{ fromJson(steps.prepare.outputs.envs || '{}') }}
+      -
+        name: Remove build secrets
+        if: ${{ always() && steps.prepare.outputs.secret-dir != '' }}
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          INPUT_SECRET-DIR: ${{ steps.prepare.outputs.secret-dir }}
+        with:
+          script: |
+            const { BuildSecrets } = require('@docker/github-builder-runtime/lib/github-builder/build-secrets');
+            BuildSecrets.cleanup(core.getInput('secret-dir', {required: true}));
       -
         name: Get image digest
         id: get-image-digest