From 82df3f17f7a25244569acb012c28a7e31e648d63 Mon Sep 17 00:00:00 2001 From: florian Date: Wed, 16 Nov 2022 20:52:41 +0100 Subject: [PATCH] ci/cd - init work --- .github/workflows/dev.yml | 615 +++++++--------------------------- .github/workflows/prod.yml | 546 +----------------------------- .github/workflows/staging.yml | 1 + 3 files changed, 123 insertions(+), 1039 deletions(-) create mode 100644 .github/workflows/staging.yml diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 4e236eab3..f82fd2a05 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -1,12 +1,13 @@ -name: Automatic test, build, push and deploy (DEV) +name: Automatic tests (DEV) on: push: branches: [dev] jobs: - # Build for amd64 - build-bw-amd64: + + # BW dev tests + bw: runs-on: ubuntu-latest steps: # Prepare @@ -25,388 +26,39 @@ jobs: registry: ${{ secrets.PRIVATE_REGISTRY }} username: registry password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build images - - name: Build BW for amd64 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest - cache-from: type=registry,ref=bunkerity/cache:bw-amd64-cache - cache-to: type=registry,ref=bunkerity/cache:bw-amd64-cache,mode=min - - name: Build BW autoconf for amd64 - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest - cache-from: type=registry,ref=bunkerity/cache:bw-autoconf-amd64-cache - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-amd64-cache,mode=min - - name: Build BW UI for amd64 - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest - cache-from: type=registry,ref=bunkerity/cache:bw-ui-amd64-cache - cache-to: type=registry,ref=bunkerity/cache:bw-ui-amd64-cache,mode=min - - # Build bunkerweb/386 - build-bw-386: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - # Build images - - name: Build BW for 386 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/386 - tags: bunkerweb-tests-386:latest - cache-from: type=registry,ref=bunkerity/cache:bw-386-cache - cache-to: type=registry,ref=bunkerity/cache:bw-386-cache,mode=min - - name: Build BW autoconf for 386 - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/386 - tags: bunkerweb-autoconf-tests-386:latest - cache-from: type=registry,ref=bunkerity/cache:bw-autoconf-386-cache - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-386-cache,mode=min - - name: Build BW UI for 386 - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/386 - tags: bunkerweb-autoconf-tests-386:latest - cache-from: type=registry,ref=bunkerity/cache:bw-ui-386-cache - cache-to: type=registry,ref=bunkerity/cache:bw-ui-386-cache,mode=min - - # Build bunkerweb/arm - build-bw-arm: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Setup SSH for ARM node - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa_arm - chmod 600 ~/.ssh/id_rsa_arm - echo "$SSH_CONFIG" > ~/.ssh/config - env: - SSH_KEY: ${{ secrets.ARM_SSH_KEY }} - SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }} - - name: Append ARM node to buildx - run: | - docker buildx create --append --name ${{ steps.buildx.outputs.name }} --node arm --platform linux/arm64,linux/arm/v7,linux/arm/v6 ssh://ubuntu@arm - - # Build images - - name: Build BW for ARM - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-tests-arm:latest - cache-from: type=registry,ref=bunkerity/cache:bw-arm-cache - cache-to: type=registry,ref=bunkerity/cache:bw-arm-cache,mode=min - - name: Build BW autoconf for ARM - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-autoconf-tests-arm:latest - cache-from: type=registry,ref=bunkerity/cache:bw-autoconf-arm-cache - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-arm-cache,mode=min - - name: Build BW UI for ARM - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-ui-tests-arm:latest - cache-from: type=registry,ref=bunkerity/cache:bw-ui-arm-cache - cache-to: type=registry,ref=bunkerity/cache:bw-ui-arm-cache,mode=min - - # Build linux ubuntu - build-bw-ubuntu: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - + # TODO : code scan # Build image - - name: Build BW ubuntu + - name: Build image uses: docker/build-push-action@v3 with: context: . - file: linux/Dockerfile-ubuntu + file: src/bw/Dockerfile platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest - cache-from: type=registry,ref=bunkerity/cache:bw-ubuntu-cache - cache-to: type=registry,ref=bunkerity/cache:bw-ubuntu-cache,mode=min - - # Build linux debian - build-bw-debian: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW debian - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-debian - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest - cache-from: type=registry,ref=bunkerity/cache:bw-debian-cache - cache-to: type=registry,ref=bunkerity/cache:bw-debian-cache,mode=min - - # Build linux centos - build-bw-centos: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW centos - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-centos - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest - cache-from: type=registry,ref=bunkerity/cache:bw-centos-cache - cache-to: type=registry,ref=bunkerity/cache:bw-centos-cache,mode=min - - # Build linux fedora - build-bw-fedora: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW fedora - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-fedora - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest - cache-from: type=registry,ref=bunkerity/cache:bw-fedora-cache - cache-to: type=registry,ref=bunkerity/cache:bw-fedora-cache,mode=min - - # Run tests - tests: - needs: - [ - build-bw-amd64, - build-bw-ubuntu, - build-bw-debian, - build-bw-centos, - build-bw-fedora, - ] - runs-on: [self-hosted, X64] - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - name: Set variables - run: | - VER=$(cat VERSION | tr -d '\n') - if [ "$GITHUB_REF" = "refs/heads/master" ] ; then - echo "BUILD_MODE=prod" >> $GITHUB_ENV - else - echo "BUILD_MODE=dev" >> $GITHUB_ENV - fi - - # Import images to local registry - - name: Import BW image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest 10.20.1.1:5000/bw-tests:latest && docker push 10.20.1.1:5000/bw-tests:latest - - name: Import BW autoconf image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest 10.20.1.1:5000/bw-autoconf-tests:latest && docker push 10.20.1.1:5000/bw-autoconf-tests:latest - - name: Import BW UI image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest 10.20.1.1:5000/bw-ui-tests:latest && docker push 10.20.1.1:5000/bw-ui-tests:latest - - name: Import Ubuntu image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest bw-ubuntu-tests:latest - - name: Import Debian image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest bw-debian-tests:latest - - name: Import Centos image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest bw-centos-tests:latest - - name: Import Fedora image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest bw-fedora-tests:latest - - # CVE check on OS - - name: Check security vulnerabilities for BW + load: true + #push: true + tags: local/bw:dev + cache-from: type=registry,ref=bunkerity/cache:dev-bw-amd64-cache + cache-to: type=registry,ref=bunkerity/cache:dev-bw-amd64-cache,mode=min + # Check OS vulnerabilities + - name: Check OS vulnerabilities uses: aquasecurity/trivy-action@master with: vuln-type: os - image-ref: 10.20.1.1:5000/bw-tests:latest + image-ref: local/bw:dev format: table exit-code: 1 ignore-unfixed: false severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore - - name: Check security vulnerabilities for autoconf - uses: aquasecurity/trivy-action@master - with: - vuln-type: os - image-ref: 10.20.1.1:5000/bw-autoconf-tests:latest - format: table - exit-code: 1 - ignore-unfixed: false - severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore - - name: Check security vulnerabilities for UI - uses: aquasecurity/trivy-action@master - with: - vuln-type: os - image-ref: 10.20.1.1:5000/bw-ui-tests:latest - format: table - exit-code: 1 - ignore-unfixed: false - severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore + #trivyignores: .trivyignore - # Prepare tests - - name: Install tests dependencies - run: pip3 install -r ./tests/requirements.txt - - name: Set variables - run: cat /opt/.runner_env >> $GITHUB_ENV - - # Run tests - - name: Run Docker tests - run: ./tests/main.py "docker" - - name: Run Autoconf tests - run: ./tests/main.py "autoconf" - - name: Run Swarm tests - run: ./tests/main.py "swarm" - - name: Run Kubernetes tests - run: ./tests/main.py "kubernetes" - - name: Generate Linux packages and build test images - run: ./tests/linux.sh ${{ env.BUILD_MODE }} - - name: Run Linux Ubuntu tests - run: ./tests/main.py "linux" "ubuntu" - - name: Run Linux Debian tests - run: ./tests/main.py "linux" "debian" - - name: Run Linux CentOS tests - run: ./tests/main.py "linux" "centos" - - name: Run Linux Fedora tests - run: ./tests/main.py "linux" "fedora" - - # Push to dev registries - push-docker: - needs: [tests, build-bw-386, build-bw-arm] + # BW scheduler tests + scheduler: runs-on: ubuntu-latest steps: # Prepare - name: Checkout source code uses: actions/checkout@v3 - name: Setup Buildx - id: buildx uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub uses: docker/login-action@v2 @@ -419,142 +71,117 @@ jobs: registry: ${{ secrets.PRIVATE_REGISTRY }} username: registry password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - name: Setup SSH for ARM node - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa_arm - chmod 600 ~/.ssh/id_rsa_arm - echo "$SSH_CONFIG" > ~/.ssh/config - env: - SSH_KEY: ${{ secrets.ARM_SSH_KEY }} - SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }} - - name: Append ARM node to buildx - run: | - docker buildx create --append --name ${{ steps.buildx.outputs.name }} --node arm --platform linux/arm64,linux/arm/v7,linux/arm/v6 ssh://ubuntu@arm - - # Build and push - - name: Build and push BW + # TODO : code scan + # Build image + - name: Build image uses: docker/build-push-action@v3 with: context: . - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb:staging,bunkerity/bunkerweb:dev - cache-from: | - type=registry,ref=bunkerity/cache:bw-amd64-cache - type=registry,ref=bunkerity/cache:bw-386-cache - type=registry,ref=bunkerity/cache:bw-arm-cache - - name: Build and push BW autoconf - uses: docker/build-push-action@v3 + file: src/scheduler/Dockerfile + platforms: linux/amd64 + load: true + #push: true + tags: local/scheduler:dev + cache-from: type=registry,ref=bunkerity/cache:dev-scheduler-amd64-cache + cache-to: type=registry,ref=bunkerity/cache:dev-scheduler-amd64-cache,mode=min + # Check OS vulnerabilities + - name: Check OS vulnerabilities + uses: aquasecurity/trivy-action@master with: - context: . - file: autoconf/Dockerfile - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf:staging,bunkerity/bunkerweb-autoconf:dev - cache-from: | - type=registry,ref=bunkerity/cache:bw-autoconf-amd64-cache - type=registry,ref=bunkerity/cache:bw-autoconf-386-cache - type=registry,ref=bunkerity/cache:bw-autoconf-arm-cache - - name: Build and push BW UI - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui:staging,bunkerity/bunkerweb-ui:dev - cache-from: | - type=registry,ref=bunkerity/cache:bw-ui-amd64-cache - type=registry,ref=bunkerity/cache:bw-ui-386-cache - type=registry,ref=bunkerity/cache:bw-ui-arm-cache + vuln-type: os + image-ref: local/scheduler:dev + format: table + exit-code: 1 + ignore-unfixed: false + severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL + #trivyignores: .trivyignore - # Push to PackageCloud - push-linux: - needs: tests - runs-on: [self-hosted, X64] - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - - name: Set variables - run: | - VER=$(cat VERSION | tr -d '\n') - echo "VERSION=$VER" >> $GITHUB_ENV - - - name: Remove Ubuntu DEB from packagecloud - run: package_cloud yank bunkerity/bunkerweb-dev/ubuntu/jammy bunkerweb_${{ env.VERSION }}_amd64.deb - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Ubuntu DEB to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/dev/ubuntu/bunkerweb_${{ env.VERSION }}-1_amd64.deb - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb-dev - PACKAGECLOUD-DISTRIB: ubuntu/jammy - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove Debian DEB from packagecloud - run: package_cloud yank bunkerity/bunkerweb-dev/debian/bullseye bunkerweb_${{ env.VERSION }}_amd64.deb - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Debian DEB to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/dev/debian/bunkerweb_${{ env.VERSION }}-1_amd64.deb - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb-dev - PACKAGECLOUD-DISTRIB: debian/bullseye - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove CentOS RPM from packagecloud - run: package_cloud yank bunkerity/bunkerweb-dev/el/8 bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push CentOS RPM to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/dev/centos/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb-dev - PACKAGECLOUD-DISTRIB: el/8 - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove Fedora RPM from packagecloud - run: package_cloud yank bunkerity/bunkerweb-dev/fedora/36 bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Fedora RPM to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/dev/fedora/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb-dev - PACKAGECLOUD-DISTRIB: fedora/36 - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - # Deploy to staging infrastructure - deploy: - needs: push-docker + # BW autoconf tests + autoconf: runs-on: ubuntu-latest steps: + # Prepare - name: Checkout source code uses: actions/checkout@v3 - - - name: k8s login (staging) - uses: azure/k8s-set-context@v2 + - name: Setup Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 with: - method: kubeconfig - kubeconfig: ${{ secrets.KUBE_CONFIG_STAGING }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to private repository + uses: docker/login-action@v2 + with: + registry: ${{ secrets.PRIVATE_REGISTRY }} + username: registry + password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} + # TODO : code scan + # Build image + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + file: src/autoconf/Dockerfile + platforms: linux/amd64 + load: true + #push: true + tags: local/autoconf:dev + cache-from: type=registry,ref=bunkerity/cache:dev-ui-amd64-cache + cache-to: type=registry,ref=bunkerity/cache:dev-ui-amd64-cache,mode=min + # Check OS vulnerabilities + - name: Check OS vulnerabilities + uses: aquasecurity/trivy-action@master + with: + vuln-type: os + image-ref: local/autoconf:dev + format: table + exit-code: 1 + ignore-unfixed: false + severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL + #trivyignores: .trivyignore - - name: k8s deploy (staging) - run: kubectl rollout restart deployment bunkerweb-controller && kubectl rollout restart daemonset bunkerweb + # BW UI tests + ui: + runs-on: ubuntu-latest + steps: + # Prepare + - name: Checkout source code + uses: actions/checkout@v3 + - name: Setup Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to private repository + uses: docker/login-action@v2 + with: + registry: ${{ secrets.PRIVATE_REGISTRY }} + username: registry + password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} + # TODO : code scan + # Build image + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + file: src/ui/Dockerfile + platforms: linux/amd64 + load: true + #push: true + tags: local/ui:dev + cache-from: type=registry,ref=bunkerity/cache:dev-ui-amd64-cache + cache-to: type=registry,ref=bunkerity/cache:dev-ui-amd64-cache,mode=min + # Check OS vulnerabilities + - name: Check OS vulnerabilities + uses: aquasecurity/trivy-action@master + with: + vuln-type: os + image-ref: local/ui:dev + format: table + exit-code: 1 + ignore-unfixed: false + severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL + #trivyignores: .trivyignore diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 854adb37b..f87f5c14c 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -1,545 +1 @@ -name: Automatic test, build, push and deploy (PROD) - -on: - push: - branches: [master] - -jobs: - # Build for amd64 - build-bw-amd64: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build images - - name: Build BW for amd64 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest - cache-to: type=registry,ref=bunkerity/cache:bw-amd64-cache,mode=min - - name: Build BW autoconf for amd64 - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-amd64-cache,mode=min - - name: Build BW UI for amd64 - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest - cache-to: type=registry,ref=bunkerity/cache:bw-ui-amd64-cache,mode=min - - # Build bunkerweb/386 - build-bw-386: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - # Build images - - name: Build BW for 386 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/386 - tags: bunkerweb-tests-386:latest - cache-to: type=registry,ref=bunkerity/cache:bw-386-cache,mode=min - - name: Build BW autoconf for 386 - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/386 - tags: bunkerweb-autoconf-tests-386:latest - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-386-cache,mode=min - - name: Build BW UI for 386 - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/386 - tags: bunkerweb-autoconf-tests-386:latest - cache-to: type=registry,ref=bunkerity/cache:bw-ui-386-cache,mode=min - - # Build bunkerweb/arm - build-bw-arm: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Setup SSH for ARM node - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa_arm - chmod 600 ~/.ssh/id_rsa_arm - echo "$SSH_CONFIG" > ~/.ssh/config - env: - SSH_KEY: ${{ secrets.ARM_SSH_KEY }} - SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }} - - name: Append ARM node to buildx - run: | - docker buildx create --append --name ${{ steps.buildx.outputs.name }} --node arm --platform linux/arm64,linux/arm/v7,linux/arm/v6 ssh://ubuntu@arm - - # Build images - - name: Build BW for ARM - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-tests-arm:latest - cache-to: type=registry,ref=bunkerity/cache:bw-arm-cache,mode=min - - name: Build BW autoconf for ARM - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-autoconf-tests-arm:latest - cache-to: type=registry,ref=bunkerity/cache:bw-autoconf-arm-cache,mode=min - - name: Build BW UI for ARM - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/arm64,linux/arm/v7 - tags: bunkerweb-ui-tests-arm:latest - cache-to: type=registry,ref=bunkerity/cache:bw-ui-arm-cache,mode=min - - # Build linux ubuntu - build-bw-ubuntu: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW ubuntu - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-ubuntu - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest - - # Build linux debian - build-bw-debian: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW debian - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-debian - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest - - # Build linux centos - build-bw-centos: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW centos - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-centos - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest - - # Build linux fedora - build-bw-fedora: - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - # Build image - - name: Build BW fedora - uses: docker/build-push-action@v3 - with: - context: . - file: linux/Dockerfile-fedora - platforms: linux/amd64 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest - - # Run tests - tests: - needs: - [ - build-bw-amd64, - build-bw-ubuntu, - build-bw-debian, - build-bw-centos, - build-bw-fedora, - ] - runs-on: [self-hosted, X64] - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - name: Set variables - run: | - VER=$(cat VERSION | tr -d '\n') - if [ "$GITHUB_REF" = "refs/heads/master" ] ; then - echo "BUILD_MODE=prod" >> $GITHUB_ENV - else - echo "BUILD_MODE=dev" >> $GITHUB_ENV - fi - - # Import images to local registry - - name: Import BW image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests-amd64:latest 10.20.1.1:5000/bw-tests:latest && docker push 10.20.1.1:5000/bw-tests:latest - - name: Import BW autoconf image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf-tests-amd64:latest 10.20.1.1:5000/bw-autoconf-tests:latest && docker push 10.20.1.1:5000/bw-autoconf-tests:latest - - name: Import BW UI image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui-tests-amd64:latest 10.20.1.1:5000/bw-ui-tests:latest && docker push 10.20.1.1:5000/bw-ui-tests:latest - - name: Import Ubuntu image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ubuntu:latest bw-ubuntu-tests:latest - - name: Import Debian image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-debian:latest bw-debian-tests:latest - - name: Import Centos image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-centos:latest bw-centos-tests:latest - - name: Import Fedora image - run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-fedora:latest bw-fedora-tests:latest - - # CVE check on OS - - name: Check security vulnerabilities for BW - uses: aquasecurity/trivy-action@master - with: - vuln-type: os - image-ref: 10.20.1.1:5000/bw-tests:latest - format: table - exit-code: 1 - ignore-unfixed: false - severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - - name: Check security vulnerabilities for autoconf - uses: aquasecurity/trivy-action@master - with: - vuln-type: os - image-ref: 10.20.1.1:5000/bw-autoconf-tests:latest - format: table - exit-code: 1 - ignore-unfixed: false - severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - - name: Check security vulnerabilities for UI - uses: aquasecurity/trivy-action@master - with: - vuln-type: os - image-ref: 10.20.1.1:5000/bw-ui-tests:latest - format: table - exit-code: 1 - ignore-unfixed: false - severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - - # Prepare tests - - name: Install tests dependencies - run: pip3 install -r ./tests/requirements.txt - - name: Set variables - run: cat /opt/.runner_env >> $GITHUB_ENV - - # Run tests - #- name: Run Docker tests - #run: ./tests/main.py "docker" - #- name: Run Autoconf tests - #run: ./tests/main.py "autoconf" - #- name: Run Swarm tests - #run: ./tests/main.py "swarm" - #- name: Run Kubernetes tests - #run: ./tests/main.py "kubernetes" - - name: Generate Linux packages and build test images - run: ./tests/linux.sh ${{ env.BUILD_MODE }} - - name: Run Linux Ubuntu tests - run: ./tests/main.py "linux" "ubuntu" - - name: Run Linux Debian tests - run: ./tests/main.py "linux" "debian" - - name: Run Linux CentOS tests - run: ./tests/main.py "linux" "centos" - - name: Run Linux Fedora tests - run: ./tests/main.py "linux" "fedora" - - # Push to prod registries - push-docker: - needs: [tests, build-bw-386, build-bw-arm] - runs-on: ubuntu-latest - steps: - # Prepare - - name: Checkout source code - uses: actions/checkout@v3 - - name: Set variables - run: | - VER=$(cat VERSION | tr -d '\n') - echo "VERSION=$VER" >> $GITHUB_ENV - - name: Setup Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to private repository - uses: docker/login-action@v2 - with: - registry: ${{ secrets.PRIVATE_REGISTRY }} - username: registry - password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - - name: Setup SSH for ARM node - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa_arm - chmod 600 ~/.ssh/id_rsa_arm - echo "$SSH_CONFIG" > ~/.ssh/config - env: - SSH_KEY: ${{ secrets.ARM_SSH_KEY }} - SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }} - - name: Append ARM node to buildx - run: | - docker buildx create --append --name ${{ steps.buildx.outputs.name }} --node arm --platform linux/arm64,linux/arm/v7,linux/arm/v6 ssh://ubuntu@arm - - # Build and push - - name: Build and push BW - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb:latest,bunkerity/bunkerweb:latest,bunkerity/bunkerweb:${{ env.VERSION }} - cache-from: | - type=registry,ref=bunkerity/cache:bw-amd64-cache - type=registry,ref=bunkerity/cache:bw-386-cache - type=registry,ref=bunkerity/cache:bw-arm-cache - - name: Build and push BW autoconf - uses: docker/build-push-action@v3 - with: - context: . - file: autoconf/Dockerfile - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-autoconf:latest,bunkerity/bunkerweb-autoconf:latest,bunkerity/bunkerweb-autoconf:${{ env.VERSION }} - cache-from: | - type=registry,ref=bunkerity/cache:bw-autoconf-amd64-cache - type=registry,ref=bunkerity/cache:bw-autoconf-386-cache - type=registry,ref=bunkerity/cache:bw-autoconf-arm-cache - - name: Build and push BW UI - uses: docker/build-push-action@v3 - with: - context: . - file: ui/Dockerfile - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7 - push: true - tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-ui:latest,bunkerity/bunkerweb-ui:latest,bunkerity/bunkerweb-ui:${{ env.VERSION }} - cache-from: | - type=registry,ref=bunkerity/cache:bw-ui-amd64-cache - type=registry,ref=bunkerity/cache:bw-ui-386-cache - type=registry,ref=bunkerity/cache:bw-ui-arm-cache - - # Push to PackageCloud - push-linux: - needs: tests - runs-on: [self-hosted, X64] - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - - name: Set variables - run: | - VER=$(cat VERSION | tr -d '\n') - echo "VERSION=$VER" >> $GITHUB_ENV - - - name: Remove Ubuntu DEB from packagecloud - run: package_cloud yank bunkerity/bunkerweb/ubuntu/jammy bunkerweb_${{ env.VERSION }}_amd64.deb - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Ubuntu DEB to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/prod/ubuntu/bunkerweb_${{ env.VERSION }}-1_amd64.deb - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb - PACKAGECLOUD-DISTRIB: ubuntu/jammy - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove Debian DEB from packagecloud - run: package_cloud yank bunkerity/bunkerweb/debian/bullseye bunkerweb_${{ env.VERSION }}_amd64.deb - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Debian DEB to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/prod/debian/bunkerweb_${{ env.VERSION }}-1_amd64.deb - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb - PACKAGECLOUD-DISTRIB: debian/bullseye - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove CentOS RPM from packagecloud - run: package_cloud yank bunkerity/bunkerweb/el/8 bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push CentOS RPM to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/prod/centos/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb - PACKAGECLOUD-DISTRIB: el/8 - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Remove Fedora RPM from packagecloud - run: package_cloud yank bunkerity/bunkerweb/fedora/36 bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - continue-on-error: true - env: - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - - name: Push Fedora RPM to packagecloud - uses: danielmundi/upload-packagecloud@v1 - with: - PACKAGE-NAME: /opt/packages/prod/fedora/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm - PACKAGECLOUD-USERNAME: bunkerity - PACKAGECLOUD-REPO: bunkerweb - PACKAGECLOUD-DISTRIB: fedora/36 - PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - - # Deploy to staging infrastructure - # deploy: - # needs: push-docker - # runs-on: ubuntu-latest - # steps: - - # - name: Checkout source code - # uses: actions/checkout@v3 - - # - name: k8s login (staging) - # uses: azure/k8s-set-context@v2 - # with: - # method: kubeconfig - # kubeconfig: ${{ secrets.KUBE_CONFIG_STAGING }} - - # - name: k8s deploy (staging) - # run: kubectl rollout restart deployment bunkerweb-controller && kubectl rollout restart daemonset bunkerweb +# TODO \ No newline at end of file diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 000000000..f87f5c14c --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1 @@ +# TODO \ No newline at end of file