From 9a3b4cd36550a433d3f6d1f73a3e4e4e41ca4da9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Fri, 1 Sep 2023 12:25:17 -0300 Subject: [PATCH] Attempt to stabilize the broken integration.yml workflow (#13653) #13547 This is an attempt to stabilize this workflow that has been broken for 4-6 months. # Issue and proposed solution Github runner VMs re-use UUIDs, which is not supported by Orbit (this causes a host to be enrolled as two hosts in Fleet), thus, until that is fixed in https://github.com/fleetdm/fleet/issues/8021 I propose we stabilize this workflow by testing all `stable` channels only (which is better than having the build broken all the time IMO). Once https://github.com/fleetdm/fleet/issues/8021 is fixed we can re-add the edge channels. --- .github/workflows/integration.yml | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fa179c5d3f..6bf42b01a8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -75,10 +75,7 @@ jobs: check_artifacts: true - name: Run Fleet server - timeout-minutes: 15 - env: - # Use instance identifier to allow for duplicate UUIDs - FLEET_OSQUERY_HOST_IDENTIFIER: instance + timeout-minutes: 10 run: | chmod +x ./build/fleetctl ./build/fleetctl preview --no-hosts @@ -86,16 +83,20 @@ jobs: ./build/fleetctl get enroll-secret docker compose -f ~/.fleet/preview/docker-compose.yml logs --follow fleet01 fleet02 & # Wait for all of the hosts to be enrolled - EXPECTED=12 + EXPECTED=3 until [ $(./build/fleetctl get hosts --json | wc -l | tee hostcount) -ge $EXPECTED ]; do echo -n "Waiting for hosts to enroll: " cat hostcount | xargs echo -n echo " / $EXPECTED" - sleep 10 + sleep 20 done - ./build/fleetctl get hosts --json echo "Success! $EXPECTED hosts enrolled." + - name: Show enrolled hosts + if: always() + run: | + ./build/fleetctl get hosts --json + - name: Slack Notification if: failure() uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 @@ -163,8 +164,17 @@ jobs: timeout-minutes: 10 strategy: matrix: - orbit-channel: [ 'stable', 'edge' ] - osqueryd-channel: ['stable', 'edge' ] + # To run multiple VMs that have the same UUID we need to implement + # https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd + # in the same host are enrolled as two hosts in Fleet). + # Until then we will just test the `stable` channel in all components. + # + # Alternatively, we can bring back the `edge` channel when we decide to upgrade + # our worker to macOS 13 in the future, as they changed the virtualization + # layer for 13 and now it has random UUIDs (https://github.com/actions/runner-images/issues/7591). + orbit-channel: [ 'stable' ] + osqueryd-channel: [ 'stable' ] + desktop-channel: [ 'stable' ] runs-on: macos-latest needs: [gen, login] steps: @@ -184,7 +194,7 @@ jobs: SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret') echo "Secret: $SECRET" echo "Hostname: $(hostname -s)" - fleetctl package --type pkg --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --fleet-desktop + fleetctl package --type pkg --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug sudo installer -pkg fleet-osquery.pkg -target / until fleetctl get hosts | grep -iF $(hostname -s); do @@ -202,7 +212,7 @@ jobs: if: always() uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2 with: - name: orbit-macos-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-logs + name: orbit-macos-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs path: | orbit-logs @@ -214,8 +224,13 @@ jobs: timeout-minutes: 10 strategy: matrix: - orbit-channel: [ 'stable', 'edge' ] - osqueryd-channel: ['stable', 'edge' ] + # To run multiple VMs that have the same UUID we need to implement + # https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd + # in the same host are enrolled as two hosts in Fleet). + # Until then we will just test the `stable` channel in all components. + orbit-channel: [ 'stable' ] + osqueryd-channel: [ 'stable' ] + desktop-channel: [ 'stable' ] runs-on: ubuntu-latest needs: [gen, login] steps: @@ -244,7 +259,7 @@ jobs: SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret') echo "Secret: $SECRET" echo "Hostname: $(hostname -s)" - ./build/fleetctl package --type deb --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} + ./build/fleetctl package --type deb --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug sudo dpkg -i fleet-osquery* until fleetctl get hosts | grep -iF $(hostname -s); do @@ -262,7 +277,7 @@ jobs: if: always() uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2 with: - name: orbit-ubuntu-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-logs + name: orbit-ubuntu-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs path: | orbit-logs @@ -274,8 +289,13 @@ jobs: timeout-minutes: 10 strategy: matrix: - orbit-channel: [ 'stable', 'edge' ] - osqueryd-channel: ['stable', 'edge' ] + # To run multiple VMs that have the same UUID we need to implement + # https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd + # in the same host are enrolled as two hosts in Fleet). + # Until then we will just test the `stable` channel in all components. + orbit-channel: [ 'stable' ] + osqueryd-channel: [ 'stable' ] + desktop-channel: [ 'stable' ] runs-on: ubuntu-latest needs: [gen, login] steps: @@ -292,21 +312,26 @@ jobs: SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret') echo "Secret: $SECRET" echo "Hostname: $(hostname -s)" - fleetctl package --type msi --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --fleet-desktop - mv fleet-osquery.msi orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}.msi + fleetctl package --type msi --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug + mv fleet-osquery.msi orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi - name: Upload MSI uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2 with: - name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}.msi - path: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}.msi + name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi + path: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi orbit-windows: timeout-minutes: 10 strategy: matrix: - orbit-channel: [ 'stable', 'edge' ] - osqueryd-channel: ['stable', 'edge' ] + # To run multiple VMs that have the same UUID we need to implement + # https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd + # in the same host are enrolled as two hosts in Fleet). + # Until then we will just test the `stable` channel in all components. + orbit-channel: [ 'stable' ] + osqueryd-channel: [ 'stable' ] + desktop-channel: [ 'stable' ] needs: [gen, login, orbit-windows-build] runs-on: windows-latest steps: @@ -320,12 +345,12 @@ jobs: id: download uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v2 with: - name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}.msi + name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi - name: Install Orbit shell: cmd run: | - msiexec /i ${{steps.download.outputs.download-path}}\orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}.msi /quiet /passive /lv log.txt + msiexec /i ${{steps.download.outputs.download-path}}\orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi /quiet /passive /lv log.txt sleep 30 # We can't very accurately check the install on these Windows hosts since the hostnames tend to @@ -343,5 +368,5 @@ jobs: if: always() uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2 with: - name: orbit-windows-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-logs + name: orbit-windows-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs path: C:\Windows\system32\config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log