From 854fa2af6251e5d8b606de13484133ce32b51d4e Mon Sep 17 00:00:00 2001 From: Scott Gress Date: Fri, 3 Apr 2026 10:40:56 -0500 Subject: [PATCH] Cleanup docker publish (#42693) **Related issue:** Resolves #42691 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [ ] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. n/a ## Testing - [ ] Added/updated automated tests - [X] QA'd all new/changed functionality manually - I ran the updated snapshot action on this branch and verified that it pushed the branch-tagged image, but not the SHA-tagged one. - I ran the cleanup script in dry-run mode and verified that it didn't expect to delete any non-sha-tagged images - I wasn't able to test the delete-image-on-branch-delete action for obvious reasons. - I haven't tested the cleanup script in non-dry-run mode... I could do on my personal dockerhub... ## Summary by CodeRabbit ## Release Notes * **New Features** * Automated cleanup of Docker images when development branches are deleted to maintain registry hygiene. * New utility for managing and cleaning up legacy Docker image tags. * **Chores** * Enhanced Docker image tagging in snapshot builds with improved branch name handling. --- .github/workflows/docker-cleanup-branch.yaml | 86 +++++++++++ .../workflows/goreleaser-snapshot-fleet.yaml | 38 ++--- .goreleaser-snapshot.yml | 2 +- .../cleanup-dockerhub-sha-tags.sh | 146 ++++++++++++++++++ .../cleanup-quay-sha-tags.sh | 129 ++++++++++++++++ 5 files changed, 379 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/docker-cleanup-branch.yaml create mode 100755 tools/cleanup-docker-sha-tags/cleanup-dockerhub-sha-tags.sh create mode 100755 tools/cleanup-docker-sha-tags/cleanup-quay-sha-tags.sh diff --git a/.github/workflows/docker-cleanup-branch.yaml b/.github/workflows/docker-cleanup-branch.yaml new file mode 100644 index 0000000000..a535cb3130 --- /dev/null +++ b/.github/workflows/docker-cleanup-branch.yaml @@ -0,0 +1,86 @@ +name: Docker cleanup (branch deletion) + +on: + delete: + +permissions: + contents: read + +jobs: + cleanup: + # Only run for branch deletions (not tag deletions) in the fleetdm/fleet repo. + if: ${{ github.event.ref_type == 'branch' && github.repository == 'fleetdm/fleet' }} + runs-on: ubuntu-latest + environment: Docker Hub + steps: + - name: Sanitize branch name + id: sanitize + env: + BRANCH: ${{ github.event.ref }} + run: | + SANITIZED="${BRANCH//\//-}" + echo "TAG=$SANITIZED" >> $GITHUB_OUTPUT + + - name: Skip protected branches + id: check_protected + env: + TAG: ${{ steps.sanitize.outputs.TAG }} + run: | + if [[ "$TAG" == "main" || "$TAG" == rc-minor-* || "$TAG" == rc-patch-* ]]; then + echo "skip=true" >> $GITHUB_OUTPUT + echo "Skipping cleanup for protected branch tag: $TAG" + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + + - name: Delete tag from Docker Hub + if: steps.check_protected.outputs.skip == 'false' + env: + TAG: ${{ steps.sanitize.outputs.TAG }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_ACCESS_TOKEN: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + run: | + # Authenticate and get JWT + TOKEN=$(curl -s -X POST "https://hub.docker.com/v2/users/login/" \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_ACCESS_TOKEN\"}" \ + | jq -r .token) + + # Bail if the token is empty (authentication failed) + if [[ -z "$TOKEN" ]]; then + echo "Failed to authenticate with Docker Hub. Check credentials." + exit 1 + fi + + # Delete the tag (ignore 404 — tag may not exist) + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + "https://hub.docker.com/v2/repositories/fleetdm/fleet/tags/${TAG}/" \ + -H "Authorization: Bearer $TOKEN") + + if [[ "$HTTP_STATUS" == "204" ]]; then + echo "Deleted Docker Hub tag: $TAG" + elif [[ "$HTTP_STATUS" == "404" ]]; then + echo "Docker Hub tag not found (already deleted or never published): $TAG" + else + echo "Unexpected response from Docker Hub: HTTP $HTTP_STATUS" + exit 1 + fi + + - name: Delete tag from Quay.io + if: steps.check_protected.outputs.skip == 'false' + env: + TAG: ${{ steps.sanitize.outputs.TAG }} + QUAY_REGISTRY_PASSWORD: ${{ secrets.QUAY_REGISTRY_PASSWORD }} + run: | + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + "https://quay.io/api/v1/repository/fleetdm/fleet/tag/${TAG}" \ + -H "Authorization: Bearer $QUAY_REGISTRY_PASSWORD") + + if [[ "$HTTP_STATUS" == "204" || "$HTTP_STATUS" == "200" ]]; then + echo "Deleted Quay.io tag: $TAG" + elif [[ "$HTTP_STATUS" == "404" ]]; then + echo "Quay.io tag not found (already deleted or never published): $TAG" + else + echo "Unexpected response from Quay.io: HTTP $HTTP_STATUS" + exit 1 + fi diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml index d2bdbce662..3d951013ca 100644 --- a/.github/workflows/goreleaser-snapshot-fleet.yaml +++ b/.github/workflows/goreleaser-snapshot-fleet.yaml @@ -69,6 +69,14 @@ jobs: - name: Install Dependencies run: make deps + - name: Sanitize branch name for Docker tag + id: sanitize_branch + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + SANITIZED="${BRANCH//\//-}" + echo "DOCKER_IMAGE_TAG=$SANITIZED" >> $GITHUB_OUTPUT + - name: Compute version from branch id: compute_version env: @@ -90,14 +98,7 @@ jobs: env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} FLEET_VERSION: ${{ steps.compute_version.outputs.FLEET_VERSION }} - - - name: Tag image with branch name - run: docker tag fleetdm/fleet:$(git rev-parse --short HEAD) fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) - - - name: Generate tag - id: generate_tag - run: | - echo "FLEET_IMAGE_TAG=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT + DOCKER_IMAGE_TAG: ${{ steps.sanitize_branch.outputs.DOCKER_IMAGE_TAG }} - name: List VEX files id: generate_vex_files @@ -125,7 +126,7 @@ jobs: --pkg-types=os,library \ --severity=HIGH,CRITICAL \ --vex="${{ steps.generate_vex_files.outputs.VEX_FILES }}" \ - fleetdm/fleet:${{ steps.generate_tag.outputs.FLEET_IMAGE_TAG }} + fleetdm/fleet:${{ steps.sanitize_branch.outputs.DOCKER_IMAGE_TAG }} - name: Check high/critical vulnerabilities before publishing (docker scout) # Only run this when tagging RCs. @@ -133,7 +134,7 @@ jobs: uses: docker/scout-action@381b657c498a4d287752e7f2cfb2b41823f566d9 # v1.17.1 with: command: cves - image: fleetdm/fleet:${{ steps.generate_tag.outputs.FLEET_IMAGE_TAG }} + image: fleetdm/fleet:${{ steps.sanitize_branch.outputs.DOCKER_IMAGE_TAG }} only-severities: critical,high only-fixed: true only-vex-affected: true @@ -145,14 +146,9 @@ jobs: - name: Publish Docker images run: docker push fleetdm/fleet --all-tags - - name: Get tags - run: | - echo "TAG=$(git rev-parse --abbrev-ref HEAD) $(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - id: docker - - name: List tags for push run: | - echo "The following TAGs are to be pushed: ${{ steps.docker.outputs.TAG }}" + echo "The following tag will be pushed: ${{ steps.sanitize_branch.outputs.DOCKER_IMAGE_TAG }}" - name: Login to quay.io uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 @@ -162,12 +158,12 @@ jobs: password: ${{ secrets.QUAY_REGISTRY_PASSWORD }} - name: Tag and push to quay.io + env: + TAG: ${{ steps.sanitize_branch.outputs.DOCKER_IMAGE_TAG }} run: | - for TAG in ${{ steps.docker.outputs.TAG }}; do - docker tag fleetdm/fleet:${TAG} quay.io/fleetdm/fleet:${TAG} - for i in {1..5}; do - docker push quay.io/fleetdm/fleet:${TAG} && break || sleep 10 - done + docker tag fleetdm/fleet:${TAG} quay.io/fleetdm/fleet:${TAG} + for i in {1..5}; do + docker push quay.io/fleetdm/fleet:${TAG} && break || sleep 10 done - name: Slack notification diff --git a/.goreleaser-snapshot.yml b/.goreleaser-snapshot.yml index 55c6a9d3f1..6e3da52cb3 100644 --- a/.goreleaser-snapshot.yml +++ b/.goreleaser-snapshot.yml @@ -66,4 +66,4 @@ dockers: - fleetctl dockerfile: tools/fleet-docker/Dockerfile image_templates: - - "fleetdm/fleet:{{ .ShortCommit }}" + - 'fleetdm/fleet:{{ envOrDefault "DOCKER_IMAGE_TAG" .Branch }}' diff --git a/tools/cleanup-docker-sha-tags/cleanup-dockerhub-sha-tags.sh b/tools/cleanup-docker-sha-tags/cleanup-dockerhub-sha-tags.sh new file mode 100755 index 0000000000..211f981dd8 --- /dev/null +++ b/tools/cleanup-docker-sha-tags/cleanup-dockerhub-sha-tags.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# +# cleanup-dockerhub-sha-tags.sh — One-time script to remove commit-SHA-tagged +# Docker images from Docker Hub for the fleetdm/fleet repository. +# +# Usage: +# ./cleanup-dockerhub-sha-tags.sh [--dry-run] [--delay SECONDS] +# +# Environment variables: +# DOCKERHUB_USERNAME Docker Hub username (required) +# DOCKERHUB_ACCESS_TOKEN Docker Hub access token (required) +# +# This script is intended to be run manually by an engineer. It is NOT wired +# into any CI workflow. Requires jq and curl to be installed. + +set -euo pipefail + +DRY_RUN=false +DELAY=1 +REPO="fleetdm/fleet" +SHA_PATTERN="^[0-9a-f]{7,12}$" +PAGE_SIZE=100 + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --delay) DELAY="$2"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +if [[ -z "${DOCKERHUB_USERNAME:-}" || -z "${DOCKERHUB_ACCESS_TOKEN:-}" ]]; then + echo "Error: DOCKERHUB_USERNAME and DOCKERHUB_ACCESS_TOKEN must be set." >&2 + exit 1 +fi + +# Authenticate +echo "Authenticating to Docker Hub..." +DOCKERHUB_TOKEN=$(curl -s -X POST "https://hub.docker.com/v2/users/login/" \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_ACCESS_TOKEN\"}" \ + | jq -r .token) + +if [[ -z "$DOCKERHUB_TOKEN" || "$DOCKERHUB_TOKEN" == "null" ]]; then + echo "Error: Failed to authenticate to Docker Hub." >&2 + exit 1 +fi +echo "Authenticated." + +# Collect all tags via pagination +echo "Fetching tags from Docker Hub (this may take a while)..." +ALL_TAGS=() +NEXT_URL="https://hub.docker.com/v2/repositories/${REPO}/tags/?page_size=${PAGE_SIZE}" + +while [[ -n "$NEXT_URL" && "$NEXT_URL" != "null" ]]; do + RESPONSE=$(curl -s "$NEXT_URL" -H "Authorization: Bearer $DOCKERHUB_TOKEN") + PAGE_TAGS=$(echo "$RESPONSE" | jq -r '.results[].name') + while IFS= read -r tag; do + [[ -n "$tag" ]] && ALL_TAGS+=("$tag") + done <<< "$PAGE_TAGS" + NEXT_URL=$(echo "$RESPONSE" | jq -r '.next') + echo " Fetched ${#ALL_TAGS[@]} tags so far..." +done + +echo "Total tags found: ${#ALL_TAGS[@]}" + +# Filter to SHA-only tags +SHA_TAGS=() +for tag in "${ALL_TAGS[@]}"; do + if [[ "$tag" =~ $SHA_PATTERN ]]; then + SHA_TAGS+=("$tag") + fi +done + +echo "Tags matching commit SHA pattern: ${#SHA_TAGS[@]}" + +if [[ ${#SHA_TAGS[@]} -eq 0 ]]; then + echo "No SHA tags to delete. Done." + exit 0 +fi + +if [[ "$DRY_RUN" == "true" ]]; then + echo "" + echo "=== Tags that will be KEPT (do not match SHA pattern) ===" + KEPT=0 + for tag in "${ALL_TAGS[@]}"; do + if ! [[ "$tag" =~ $SHA_PATTERN ]]; then + echo " $tag" + KEPT=$((KEPT + 1)) + fi + done + echo "=== ${KEPT} tags kept ===" + echo "" + echo "=== DRY RUN — ${#SHA_TAGS[@]} tags would be deleted ===" + exit 0 +fi + +# Delete tags +DELETED=0 +FAILED=0 +TOTAL=${#SHA_TAGS[@]} + +echo "" +echo "Deleting ${TOTAL} SHA tags from Docker Hub..." + +for tag in "${SHA_TAGS[@]}"; do + ATTEMPT=0 + MAX_RETRIES=3 + while true; do + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + "https://hub.docker.com/v2/repositories/${REPO}/tags/${tag}/" \ + -H "Authorization: Bearer $DOCKERHUB_TOKEN") + + if [[ "$HTTP_STATUS" == "204" ]]; then + DELETED=$((DELETED + 1)) + break + elif [[ "$HTTP_STATUS" == "404" ]]; then + break + elif [[ "$HTTP_STATUS" == "429" ]]; then + ATTEMPT=$((ATTEMPT + 1)) + if [[ $ATTEMPT -ge $MAX_RETRIES ]]; then + echo " FAILED (rate limited, max retries): $tag" + FAILED=$((FAILED + 1)) + break + fi + BACKOFF=$((DELAY * ATTEMPT * 5)) + echo " Rate limited, waiting ${BACKOFF}s before retry..." + sleep "$BACKOFF" + else + echo " FAILED (HTTP $HTTP_STATUS): $tag" + FAILED=$((FAILED + 1)) + break + fi + done + + # Progress + PROCESSED=$((DELETED + FAILED)) + if (( PROCESSED % 50 == 0 )); then + echo " Progress: ${PROCESSED}/${TOTAL} processed (${DELETED} deleted, ${FAILED} failed)" + fi + + sleep "$DELAY" +done + +echo "" +echo "Done. ${DELETED} deleted, ${FAILED} failed (out of ${TOTAL})" diff --git a/tools/cleanup-docker-sha-tags/cleanup-quay-sha-tags.sh b/tools/cleanup-docker-sha-tags/cleanup-quay-sha-tags.sh new file mode 100755 index 0000000000..8af2da839c --- /dev/null +++ b/tools/cleanup-docker-sha-tags/cleanup-quay-sha-tags.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# cleanup-quay-sha-tags.sh — One-time script to remove commit-SHA-tagged +# Docker images from Quay.io for the fleetdm/fleet repository. +# +# Usage: +# ./cleanup-quay-sha-tags.sh [--dry-run] [--delay SECONDS] +# +# Environment variables: +# QUAY_REGISTRY_PASSWORD Quay.io bearer token (required) +# +# This script is intended to be run manually by an engineer. It is NOT wired +# into any CI workflow. Requires jq and curl to be installed. + +set -euo pipefail + +DRY_RUN=false +DELAY=1 +REPO="fleetdm/fleet" +SHA_PATTERN="^[0-9a-f]{7,12}$" +PAGE_SIZE=100 + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --delay) DELAY="$2"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +if [[ -z "${QUAY_REGISTRY_PASSWORD:-}" ]]; then + echo "Error: QUAY_REGISTRY_PASSWORD must be set." >&2 + exit 1 +fi + +# Collect all tags via pagination +echo "Fetching tags from Quay.io (this may take a while)..." +ALL_TAGS=() +PAGE=1 +HAS_MORE=true + +while [[ "$HAS_MORE" == "true" ]]; do + RESPONSE=$(curl -s "https://quay.io/api/v1/repository/${REPO}/tag/?page=${PAGE}&limit=${PAGE_SIZE}" \ + -H "Authorization: Bearer $QUAY_REGISTRY_PASSWORD") + + PAGE_TAGS=$(echo "$RESPONSE" | jq -r '.tags[].name // empty') + COUNT=0 + while IFS= read -r tag; do + if [[ -n "$tag" ]]; then + ALL_TAGS+=("$tag") + COUNT=$((COUNT + 1)) + fi + done <<< "$PAGE_TAGS" + + HAS_MORE=$(echo "$RESPONSE" | jq -r '.has_additional') + PAGE=$((PAGE + 1)) + echo " Fetched ${#ALL_TAGS[@]} tags so far..." + + if [[ $COUNT -eq 0 ]]; then + break + fi +done + +echo "Total tags found: ${#ALL_TAGS[@]}" + +# Filter to SHA-only tags +SHA_TAGS=() +for tag in "${ALL_TAGS[@]}"; do + if [[ "$tag" =~ $SHA_PATTERN ]]; then + SHA_TAGS+=("$tag") + fi +done + +echo "Tags matching commit SHA pattern: ${#SHA_TAGS[@]}" + +if [[ ${#SHA_TAGS[@]} -eq 0 ]]; then + echo "No SHA tags to delete. Done." + exit 0 +fi + +if [[ "$DRY_RUN" == "true" ]]; then + echo "" + echo "=== Tags that will be KEPT (do not match SHA pattern) ===" + KEPT=0 + for tag in "${ALL_TAGS[@]}"; do + if ! [[ "$tag" =~ $SHA_PATTERN ]]; then + echo " $tag" + KEPT=$((KEPT + 1)) + fi + done + echo "=== ${KEPT} tags kept ===" + echo "" + echo "=== DRY RUN — ${#SHA_TAGS[@]} tags would be deleted ===" + exit 0 +fi + +# Delete tags +DELETED=0 +FAILED=0 +TOTAL=${#SHA_TAGS[@]} + +echo "" +echo "Deleting ${TOTAL} SHA tags from Quay.io..." + +for tag in "${SHA_TAGS[@]}"; do + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + "https://quay.io/api/v1/repository/${REPO}/tag/${tag}" \ + -H "Authorization: Bearer $QUAY_REGISTRY_PASSWORD") + + if [[ "$HTTP_STATUS" == "200" || "$HTTP_STATUS" == "204" ]]; then + DELETED=$((DELETED + 1)) + elif [[ "$HTTP_STATUS" == "404" ]]; then + : # already gone, not a failure + else + echo " FAILED (HTTP $HTTP_STATUS): $tag" + FAILED=$((FAILED + 1)) + fi + + # Progress + PROCESSED=$((DELETED + FAILED)) + if (( PROCESSED % 50 == 0 )); then + echo " Progress: ${PROCESSED}/${TOTAL} processed (${DELETED} deleted, ${FAILED} failed)" + fi + + sleep "$DELAY" +done + +echo "" +echo "Done. ${DELETED} deleted, ${FAILED} failed (out of ${TOTAL})"