.github/workflows/release.yaml #4175
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - README.md | |
| - withdrawn-images.txt | |
| - withdrawn-repos.txt | |
| - reinstated-images.txt | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| only: | |
| description: "Specific image name to build" | |
| type: string | |
| required: false | |
| default: "" | |
| concurrency: | |
| group: ${{ inputs.only || 'release' }} | |
| cancel-in-progress: false | |
| env: | |
| TF_VAR_target_repository: cgr.dev/chainguard | |
| permissions: {} | |
| jobs: | |
| shard: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - id: shard | |
| name: Shard | |
| shell: bash # bash math foo required | |
| run: | | |
| images=($(find ./images -maxdepth 1 -type d -not -path "./images/TEMPLATE" | awk -F'/' '{print $3}' | sort -u | shuf)) | |
| total=${#images[@]} | |
| # put each image module into its own shard | |
| declare -a bins | |
| for ((i = 0; i < total; i++)); do | |
| bins[$i]="${images[$i]}" | |
| done | |
| matrix=$(printf "%s\n" "${bins[@]}" | jq -cRnjr '[inputs] | [ range(0; length) as $i | { "index": $i | tostring, "images": .[$i] } ]') | |
| echo "matrix=${matrix}" >> $GITHUB_OUTPUT | |
| # Overwrite the output above if workflow_dispatch'd with `only` | |
| if [ -n "${{ inputs.only }}" ]; then | |
| shard='[{"index": 0, "images": "${{ inputs.only }}"}]' | |
| echo "matrix=${shard}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Shard Results | |
| run: echo '${{ steps.shard.outputs.matrix }}' | |
| outputs: | |
| # This is of the format [{"index": 0, "images": "a b c"}, {"index": 1, "images": "d e f"}, ...] | |
| matrix: "${{steps.shard.outputs.matrix}}" | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: shard | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJson(needs.shard.outputs.matrix) }} | |
| permissions: | |
| id-token: write | |
| packages: write | |
| contents: read | |
| actions: read | |
| steps: | |
| - uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 | |
| with: | |
| egress-policy: audit | |
| - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 | |
| with: | |
| terraform_version: "1.8.*" | |
| terraform_wrapper: false | |
| - uses: chainguard-dev/setup-chainctl@f4ed65b781b048c44d4f033ae854c025c5531c19 # v0.3.2 | |
| with: | |
| # This allows chainguard-images/images-private to publish images to cgr.dev/chainguard-private | |
| # We maintain this identity here: | |
| # https://github.com/chainguard-dev/mono/blob/main/env/chainguard-images/iac/images-pusher.tf | |
| identity: 720909c9f5279097d847ad02a2f24ba8f59de36a/b6461e99e132298f | |
| # Make cosign/crane CLI available to the tests | |
| - uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2 | |
| - uses: imjasonh/setup-crane@31b88efe9de28ae0ffa220711af4b60be9435f6e # v0.4 | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| # note this step takes a long time, thus only do it when we | |
| # really need lots of disk space, ie. for pytorch | |
| - name: Disk cleanup | |
| if: ${{ contains(matrix.shard.images, 'pytorch') }} | |
| run: | | |
| ## All disk reclaim actions look sus and are not actively | |
| ## maintained, doing it by hand as per https://dev.to/mathio/squeezing-disk-space-from-github-actions-runners-an-engineers-guide-3pjg#6-how-to-automate-cleanup-in-your-ci | |
| # Remove Java (JDKs) | |
| sudo rm -rf /usr/lib/jvm | |
| # Remove .NET SDKs | |
| sudo rm -rf /usr/share/dotnet | |
| # Remove Swift toolchain | |
| sudo rm -rf /usr/share/swift | |
| # Remove Haskell (GHC) | |
| sudo rm -rf /usr/local/.ghcup | |
| # Remove Julia | |
| sudo rm -rf /usr/local/julia* | |
| # Remove Android SDKs | |
| sudo rm -rf /usr/local/lib/android | |
| # Remove Chromium (optional if not using for browser tests) | |
| sudo rm -rf /usr/local/share/chromium | |
| # Remove Microsoft/Edge and Google Chrome builds | |
| sudo rm -rf /opt/microsoft /opt/google | |
| # Remove Azure CLI | |
| sudo rm -rf /opt/az | |
| # Remove PowerShell | |
| sudo rm -rf /usr/local/share/powershell | |
| # Remove CodeQL and other toolcaches | |
| sudo rm -rf /opt/hostedtoolcache | |
| docker system prune -af || true | |
| docker builder prune -af || true | |
| df -h | |
| - name: Terraform apply | |
| timeout-minutes: 60 | |
| run: | | |
| set -exo pipefail | |
| env | grep '^TF_VAR_' | |
| make init-upgrade | |
| targets="" | |
| for image in ${{ matrix.shard.images }}; do | |
| targets+=' -target='module."${image}"'' | |
| done | |
| terraform plan ${targets} -out mega-module.tfplan | |
| terraform apply ${targets} -auto-approve --parallelism=$(nproc) -json mega-module.tfplan | tee /tmp/mega-module.tf.json | jq -r '.["@message"]' | |
| - name: Collect TF diagnostics | |
| if: ${{ always() }} | |
| id: tf-diag | |
| uses: chainguard-dev/actions/terraform-diag@708219d4822f33611ac1a2653815cc10e1ab54a6 # v1.4.7 | |
| with: | |
| json-file: /tmp/mega-module.tf.json | |
| - name: Collect K8s diagnostics and upload | |
| if: ${{ failure() }} | |
| run: | | |
| echo 'This step is deprecated. Please use the imagetest provider and reference the imagetest logs' | |
| - name: Upload terraform logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: "mega-module-${{ matrix.shard.index }}.tf.json" | |
| path: /tmp/mega-module.tf.json | |
| - name: Upload imagetest logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: "mega-module-${{ matrix.shard.index }}-imagetest-logs" | |
| path: imagetest-logs | |
| - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3 | |
| if: ${{ failure() && github.event_name == 'schedule' }} | |
| env: | |
| SLACK_ICON: http://github.com/chainguard-dev.png?size=48 | |
| SLACK_USERNAME: guardian | |
| SLACK_WEBHOOK: ${{ secrets.DISTROLESS_SLACK_WEBHOOK }} | |
| SLACK_MSG_AUTHOR: chainguardian | |
| SLACK_CHANNEL: chainguard-images-alerts | |
| SLACK_COLOR: "#8E1600" | |
| MSG_MINIMAL: "true" | |
| SLACK_TITLE: "[images] release failed (shard ${{ matrix.shard.index }} of ${{ env.TOTAL_SHARDS }})" | |
| SLACK_MESSAGE: | | |
| https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| ${{ steps.tf-diag.outputs.errors }} | |
| summary: | |
| name: "Build Summary" | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| needs: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 | |
| with: | |
| egress-policy: audit | |
| - name: "Download shard logs" | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| path: /tmp/shard-logs | |
| pattern: mega-module-* | |
| # Cat all the files into one, while maintaining their shard | |
| - run: | | |
| find '/tmp/shard-logs' -name 'mega-module.tf.json' | while read file; do | |
| shard_index=$(echo "$file" | sed -E 's/.*mega-module-([0-9]+)\.tf\.json.*/\1/') | |
| echo $shard_index | |
| jq -cr --arg shard_index "$shard_index" '. + {"shard_index":$shard_index}' $file >> logs.tf.json | |
| done | |
| # process the logs | |
| - run: | | |
| # Create a file just for errors | |
| jq -r 'select(.["@level"]=="error")' logs.tf.json > errors.tf.json | |
| # Build the markdown table | |
| - run: | | |
| echo "| Status | Shard | Image | Summary | Address |" >> $GITHUB_STEP_SUMMARY | |
| echo "| :-: | ----- | ----- | ------- | ------- |" >> $GITHUB_STEP_SUMMARY | |
| # append the rows to the table | |
| export rows="$(jq -r '"| ❌ | " + .shard_index + " | " + (.diagnostic.address | split(".")[1]) + " | ```" + .diagnostic.summary + "``` | ```" + .diagnostic.address + "``` |"' errors.tf.json)" | |
| echo "${rows}" | |
| cat >> $GITHUB_STEP_SUMMARY <<EOR | |
| ${rows} | |
| EOR | |
| - name: Error Details | |
| run: | | |
| # Print the errors as expandable groups | |
| jq -r '"::group:: shard: " + .shard_index + " | " + (.diagnostic.address | split(".")[1]) + "\nresource: " + .diagnostic.address + "\n\nsummary: " + .diagnostic.summary + "\n\ndetails:\n\n" + .diagnostic.detail + "\n::endgroup::"' errors.tf.json || true |