diff --git a/.vscode/extensions.json b/.vscode/extensions.json index a8597204ad..8725e4126f 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,6 +7,7 @@ "redhat.vscode-yaml", "dbaeumer.vscode-eslint", "firefox-devtools.vscode-firefox-debug", - "editorconfig.editorconfig" + "editorconfig.editorconfig", + "timonwong.shellcheck" ] } \ No newline at end of file diff --git a/Makefile b/Makefile index fa4f92572c..03b20a73b1 100644 --- a/Makefile +++ b/Makefile @@ -318,7 +318,9 @@ changelog: sh -c "git rm changes/*" changelog-orbit: - sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {}; echo' > new-CHANGELOG.md" + $(eval TODAY_DATE := $(shell date "+%b %d, %Y")) + @echo -e "## Orbit $(version) ($(TODAY_DATE))\n" > new-CHANGELOG.md + sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {} | sed -E "s/^-/*/"; echo' >> new-CHANGELOG.md" sh -c "cat new-CHANGELOG.md orbit/CHANGELOG.md > tmp-CHANGELOG.md && rm new-CHANGELOG.md && mv tmp-CHANGELOG.md orbit/CHANGELOG.md" sh -c "git rm orbit/changes/*" @@ -394,7 +396,7 @@ ifneq ($(shell uname), Darwin) @exit 1 endif # locking the version of swiftDialog to 2.2.1-4591 as newer versions - # migth have layout issues. + # might have layout issues. ifneq ($(version), 2.2.1) @echo "Version is locked at 2.1.0, see comments in Makefile target for details" @exit 1 diff --git a/orbit/docs/TUF-Update-Guide.md b/orbit/docs/TUF-Update-Guide.md deleted file mode 100644 index e2f4946ecf..0000000000 --- a/orbit/docs/TUF-Update-Guide.md +++ /dev/null @@ -1,392 +0,0 @@ -# Pushing new releases to TUF - -This document is a walkthrough guide for: -- A Fleet member to publish new updates to [Fleet's TUF service](tuf.fleetctl.com). See [Pushing updates](#pushing-updates). -- A Fleet member to delete targets from [Fleet's TUF service](tuf.fleetctl.com). See [Removing unused targets](#removing-unused-targets). -- A Fleet member to become a publisher of updates for [Fleet's TUF service](tuf.fleetctl.com). See [Becoming a new Fleet publisher](#becoming-a-new-fleet-publisher). - -Video walkthrough related to this process for additional [context](https://drive.google.com/file/d/1c_iukFEMne12Cxx9WVTt_j1Wp0sC1kQU/view?usp=drive_link). - -## Security - -- The TUF keys for `targets`, `snapshot` and `timestamp` should be stored on a USB stick (used solely for this purpose). Whenever you need to push updates to Fleet's TUF repository you can temporarily copy the encrypted keys to your workstation (under the `keys/` folder, more on this below). -- The keys should be stored encrypted with its passphrase stored in 1Password (on a private vault). -- Every `fleetctl updates` command will prompt for the passphrases to decrypt the encrypted keys. You can input the passphrases every time or can alternatively set the following environment variables: `FLEET_TIMESTAMP_PASSPHRASE`, `FLEET_SNAPSHOT_PASSPHRASE` and `FLEET_TARGETS_PASSPHRASE`. Make sure to not leave traces of the passphrases (scripts, history and/or environment) when you are done. - -## Syncing Fleet's TUF repository - -> The `fleetctl updates` commands assume the folders `keys/`, `staged/` and `repository/` exist on the current working directory. - -> IMPORTANT: When syncing the repository make sure to use `--exact-timestamps`. Otherwise `aws s3 sync` may not sync files that do not change in size, like `timestamp.json`. - -- The `keys/` folder contains the encrypted private keys. -- The `staged/` folder contains uncommitted changes (usually empty because `fleetctl updates` commands automatically commit the changes). -- The `repository/` folder contains the full TUF repository. - -Following are the commands to initialize the repository on your workstation: -```sh -mkdir /path/to/tuf.fleetctl.com - -cd /path/to/tuf.fleetctl.com -mkdir -p ./repository -cp /Volumes/YOUR-USB-NAME/keys ./keys -mkdir -p ./staged - -export AWS_PROFILE=tuf -aws sso login - -aws s3 sync s3://fleet-tuf-repo ./repository --exact-timestamps -``` - -## Building the components for releasing to `edge` - -### fleetd - -> Assuming we are releasing version 1.21.0 of fleetd. - -1. Create the fleetd changelog for the new release: -```sh -git checkout main -git pull origin main -git checkout -b release-fleetd-v1.21.0 -make changelog-orbit -``` -2. Edit `orbit/CHANGELOG.md` accordingly -3. Bump Fleet Desktop version in https://github.com/fleetdm/fleet/blob/9ca85411a16c504087d2793f8b9099f98054c93f/.github/workflows/generate-desktop-targets.yml#L27. This will trigger a github action to build the Fleet Desktop executables: https://github.com/fleetdm/fleet/actions/workflows/generate-desktop-targets.yml. -4. Commit the changes, push the branch and create a PR. -5. Add the following git tag with the following format: `orbit-v1.21.0`. Once pushed this will trigger a github action to build the orbit executables: https://github.com/fleetdm/fleet/blob/main/.github/workflows/goreleaser-orbit.yaml. -```sh -git tag orbit-v1.21.0 -git push origin --tags -``` -6. Once the two github actions finish their runs, use the following scripts that will download the artifacts to a folder in your workstation (on this guide we assume you are using `$HOME/release-friday`). -NOTE: The `goreleaser-macos` job is unstable and may need several re-runs until it works. -```sh -go run ./tools/tuf/download-artifacts desktop \ - --git-branch release-fleetd-v1.21.0 \ - --output-directory $HOME/release-friday/desktop \ - --github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN -go run ./tools/tuf/download-artifacts orbit \ - --git-tag orbit-v1.21.0 \ - --output-directory $HOME/release-friday/orbit \ - --github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN -tree $HOME/release-friday -$HOME/release-friday -├── desktop -│   ├── linux -│   │   └── desktop.tar.gz -│   ├── macos -│   │   └── desktop.app.tar.gz -│   └── windows -│   └── fleet-desktop.exe -└── orbit - ├── linux - │   └── orbit - ├── macos - │   └── orbit - └── windows - └── orbit.exe -``` -7. With the executables on your workstation, proceed to [Pushing updates](#pushing-updates) (`edge`). -8. Manually run (`Run workflow`) this action that will update the released versions on our doc: https://github.com/fleetdm/fleet/actions/workflows/fleetd-tuf.yml. - -### osqueryd - -> Assuming we are releasing version 5.12.0 of osqueryd. - -1. Bump osquery version in https://github.com/fleetdm/fleet/blob/30a36b0b3a1fd50e48d98a4c3c955595022f5277/.github/workflows/generate-osqueryd-targets.yml#L27. -2. Commit the changes, push the branch (assuming branch name is `bump-osqueryd-5.12.0`) and create a PR. -3. Once the Github action completes run the following (the [GitHub API token](https://github.com/settings/tokens?type=beta) does not need any special permissions -- public repository access is sufficient): -```sh -go run ./tools/tuf/download-artifacts osqueryd \ - --git-branch bump-osqueryd-5.12.0 \ - --output-directory $HOME/release-friday/osqueryd \ - --github-username $GITHUB_USERNAME \ - --github-api-token $GITHUB_TOKEN -tree $HOME/release-friday/osqueryd -$HOME/release-friday/osqueryd -├── linux -│   └── osqueryd -├── macos -│   └── osqueryd.app.tar.gz -└── windows - └── osqueryd.exe -``` -4. With the executables on your workstation, proceed to [Pushing updates](#pushing-updates) (`edge`). -5. Manually run (`Run workflow`) this action that will update the released versions on our docs: https://github.com/fleetdm/fleet/actions/workflows/fleetd-tuf.yml. - -## Pushing updates - -> Before performing any actions on Fleet's TUF repository you must: -> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository). -> 2. Create a local backup in case we mess up with the repository: -> ```sh -> mkdir ~/tuf.fleetctl.com/backup -> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup -> ``` -> 3. Install fleetd on macOS, Linux and Windows VMs using the channel (`stable` or `edge`) you are about to release. -> You can do this using the following flags in `fleetctl package`: `--orbit-channel`, `--desktop-channel`, `--osqueryd-channel`. - -### Releasing to the `edge` channel - -The commands shown here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com). - -#### Setup - -Make sure to install fleetd components using the `edge` channels in the three supported OSs (this is useful to smoke test the update). -Here's how to generate the packages: -```sh -# (The same for --type=deb and --type=msi.) -fleetctl package --type=pkg \ - --enable-scripts \ - --fleet-desktop \ - --fleet-url=... --enroll-secret=... \ - --update-interval 10s \ - --orbit-channel edge --desktop-channel edge --osqueryd-channel edge -``` - -#### orbit - -The `orbit` executables are downloaded from the [GoReleaser Orbit action](https://github.com/fleetdm/fleet/actions/workflows/goreleaser-orbit.yaml). -Such action is triggered when git tagging a new orbit version with a tag of the form: `orbit-v1.21.0`. - -> IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it). -> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`. -> Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes. - -> The following commands assume you are pushing version `1.21.0`. - -```sh -# macOS -fleetctl updates add --target $HOME/release-friday/orbit/macos/orbit --platform macos --name orbit --version 1.21.0 -t edge -# Linux -fleetctl updates add --target $HOME/release-friday/orbit/linux/orbit --platform linux --name orbit --version 1.21.0 -t edge -# Windows -fleetctl updates add --target $HOME/release-friday/orbit/windows/orbit.exe --platform windows --name orbit --version 1.21.0 -t edge -``` - -#### desktop - -The Fleet Desktop executables are downloaded from the [Generate Fleet Desktop targets for Orbit action](https://github.com/fleetdm/fleet/actions/workflows/generate-desktop-targets.yml). -Such action is triggered by submitting a PR with the [following version string](https://github.com/fleetdm/fleet/blob/4a6bf0d447a2080f994da1e2f36ce6d51db88109/.github/workflows/generate-desktop-targets.yml#L27) changed. - -> The following commands assume you are pushing version `1.21.0`. - -```sh -# macOS -fleetctl updates add --target $HOME/release-friday/desktop/macos/desktop.app.tar.gz --platform macos --name desktop --version 1.21.0 -t edge -# Linux -fleetctl updates add --target $HOME/release-friday/desktop/linux/desktop.tar.gz --platform linux --name desktop --version 1.21.0 -t edge -# Windows -fleetctl updates add --target $HOME/release-friday/desktop/windows/fleet-desktop.exe --platform windows --name desktop --version 1.21.0 -t edge -``` - -#### swiftDialog - -> macOS only component - -The `swiftDialog` executable can be generated from a macOS host by running: -```sh -make swift-dialog-app-tar-gz version=2.2.1 build=4591 out-path=. -``` - -```sh -fleetctl updates add --target /path/to/macos/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version 2.2.1 -t edge -``` - -#### nudge - -> macOS only component - -The `nudge` executable can be generated from a macOS host by running: -```sh -make nudge-app-tar-gz version=1.1.10.81462 out-path=. -``` - -```sh -fleetctl updates add --target /path/to/macos/nudge.app.tar.gz --platform macos --name nudge --version 1.1.10.81462 -t edge -``` - -#### osqueryd - -Osquery executables are downloaded from the [Generate osqueryd targets for Fleetd action](https://github.com/fleetdm/fleet/blob/main/.github/workflows/generate-osqueryd-targets.yml). -Such action is triggered by submitting a PR with the [following version string](https://github.com/fleetdm/fleet/blob/7067ca586a4aa1a0377b387d4b4478a5958193ff/.github/workflows/generate-osqueryd-targets.yml#L27) changed. - -> The following commands assume you are pushing version `5.9.1`. - -```sh -# macOS -fleetctl updates add --target $HOME/release-friday/osqueryd/macos/osqueryd.app.tar.gz --platform macos-app --name osqueryd --version 5.9.1 -t edge -# Linux -fleetctl updates add --target $HOME/release-friday/osqueryd/linux/osqueryd --platform linux --name osqueryd --version 5.9.1 -t edge -# Windows -fleetctl updates add --target $HOME/release-friday/osqueryd/windows/osqueryd.exe --platform windows --name osqueryd --version 5.9.1 -t edge -``` - -#### Push updates - -Once all components are updated in your local repository we need to push the changes to the remote repository. -See [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository). - -### Promoting `edge` to the `stable` channel - -> Make sure to install fleetd components using the `stable` channels in the three supported OSs (this is useful to smoke test the update). - -Following is the list of components and each command for each operating system. - -The commands show here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com). - -#### orbit - -> IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it). -> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`. -> Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes. - -> The following command assumes you are pushing version `1.21.0`: -```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh orbit 1.21.0 -``` - -#### desktop - -> The following command assumes you are pushing version `1.21.0`: -```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh desktop 1.21.0 -``` - -#### swiftDialog - -> The following command assumes you are pushing version `2.2.1`: -```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh swiftDialog 2.2.1 -``` - -#### nudge - -> The following command assumes you are pushing version `1.1.10.81462`: -```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh nudge 1.1.10.81462 -``` - -#### osqueryd - -> The following command assumes you are pushing version `5.9.1`. -```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh osqueryd 5.9.1 -``` - -#### Push updates - -Once all components are updated in your local repository we need to push the changes to the remote repository. -See [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository). - -### Pushing releases to Fleet's TUF repository - -Once you are done with the changes on your local repository, you can use the following command to review the changes before pushing (`--dryrun` allows us to verify the upgrade before pushing): -```sh -AWS_PROFILE=tuf aws s3 sync ./repository s3://fleet-tuf-repo --dryrun -(dryrun) upload: repository/snapshot.json to s3://fleet-tuf-repo/snapshot.json -(dryrun) upload: repository/targets.json to s3://fleet-tuf-repo/targets.json -[...] -(dryrun) upload: repository/timestamp.json to s3://fleet-tuf-repo/timestamp.json -``` - -If all looks good, run the same command without the `--dryrun` flag. - -> NOTE: Some things to note after the changes are pushed: -> - Once pushed you might see some clients failing to upgrade due to some sha256 mismatches. These temporary failures are expected because it takes some time for caches to be invalidated (these errors should go away after a few minutes). -> - The auto-update routines in orbit runs every 15 minutes, so you might need to wait up to 15 minutes to verify online hosts are auto-updating properly. - -## Removing Unused Targets - -If you've inadvertently published a target that is no longer in use, follow these steps to remove it. - -> Before performing any actions on Fleet's TUF repository you must: -> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository). -> 2. Create a local backup in case we mess up with the repository: -> ```sh -> mkdir ~/tuf.fleetctl.com/backup -> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup -> ``` - -1. You'll need the [`go-tuf`](https://github.com/theupdateframework/go-tuf) binary. The removal operations aren't integrated into `fleetctl` at the moment. -2. Use `tuf remove` to remove the target and update `targets.json`. Substitute `desktop/windows/stable/desktop.exe` with the target you intend to delete. -```sh -tuf remove desktop/windows/stable/desktop.exe -``` -3. Snapshot, timestamp, and commit the changes. -```sh -tuf snapshot -tuf timestamp -tuf commit -``` -4. Run the following command to generate a timestamp that expires in two weeks (otherwise the default expiration when using `go-tuf` commands is 1 day) -```sh -fleetctl updates timestamp -``` -5. Confirm that the version of the local `timestamp.json` file is more recent than that of the remote server. -6. Verify the changes that will be synced by running a dry sync. Include the `--delete` flag as you're removing targets. -```sh -aws s3 sync ./repository s3://fleet-tuf-repo --delete --dryrun -``` -7. `diff` the local `targets.json` file with its remote version. -8. To upload the changes, perform a sync without the `--dryrun`: -```sh -aws s3 sync ./repository s3://fleet-tuf-repo --delete -``` - -## Becoming a New Fleet Publisher - -> Before performing any actions on Fleet's TUF repository you must: -> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository). -> 2. Create a local backup in case we mess up with the repository: -> ```sh -> mkdir ~/tuf.fleetctl.com/backup -> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup -> ``` - -### Generate targets+snapshot+timestamp keys - -All commands shown in this guide are executed from `/path/to/tuf.fleetctl.com`: -```sh -cd /path/to/tuf.fleetctl.com -``` - -```sh -tuf gen-key targets -Enter targets keys passphrase: -Repeat targets keys passphrase: -Generated targets key with ID ae943cb8be8a849b37c66ed46bdd7e905ba3118c0c051a6ee3cd30625855a076 -``` -```sh -tuf gen-key snapshot -Enter snapshot keys passphrase: -Repeat snapshot keys passphrase: -Generated snapshot key with ID 1a4d9beb826d1ff4e036d757cfcd6e36d0f041e58d25f99ef3a20ae3f8dd71e3 -``` -```sh -tuf gen-key timestamp -Enter timestamp keys passphrase: -Repeat timestamp keys passphrase: -Generated timestamp key with ID d940df08b59b12c30f95622a05cc40164b78a11dd7d408395ee4f79773331b30 -``` - -Share `staged/root.json` with Fleet member with the `root` role, who will sign with its root key and push to the repository. - -### Root role signs the `staged/root.json` - -Essentially the following commands are executed to sign the new keys: -- `tuf sign` -- `tuf snapshot` -- `tuf timestamp` -- `tuf commit` - -## Misc issues - -### Invalid timestamp.json version - -The following issue was solved by resigning the timestamp metadata `fleetctl updates timestamp` (executed three times to increase the version to `4175`) -```sh -2022-08-23T13:44:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174" -2022-08-23T13:59:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174" -``` diff --git a/tools/tuf/README.md b/tools/tuf/README.md new file mode 100644 index 0000000000..126b2610bc --- /dev/null +++ b/tools/tuf/README.md @@ -0,0 +1,292 @@ +# Releasing updates to Fleet's TUF repository + +The `releaser.sh` script automates the building and releasing of fleetd and osquery updates on [Fleet's TUF repository](https://tuf.fleetctl.com). + +> - The script was developed and tested on macOS Intel. +> - It currently supports pushing new `fleetd` and `osqueryd` versions. +> - By storing credentials encrypted in a USB flash drive and storing their decryption passphrase on 1Password we are enforcing a form of 2FA. + +```mermaid +graph LR; + subgraph Workstation; + releaser[releaser.sh]; + 1password("
1Password"); + usb("
USB flash drive"); + repository[(./repository)]; + end; + s3("
s3://fleet-tuf-repo"); + github("
Github Action\n(signing and notarization)"); + + usb--(1) copy encrypted signing keys-->releaser; + 1password--(2) get passphrases to decrypt encrypted signing keys-->releaser; + 1password--(3) get Github API token-->releaser; + s3--(4) pull TUF repository-->releaser; + releaser--(5) build components (new updates)\n(osqueryd, orbit, Fleet Desktop)-->github; + github--(6) download built components-->releaser; + releaser--(7) push updates and signed metadata-->s3; +``` + +## Permissions and configuration + +Following is the checklist for all credentials and configuration needed to run the script. + +### Dependencies + +- `make` +- `git` +- 1Password 8 application. +- Install and configure 1Password's `op` cli to connect to the application: https://developer.1password.com/docs/cli/get-started/ +- `aws` cli :`brew install awscli`. +- `fleetctl`: Either built from source or installed by npm. +- `tuf`: Download the release from https://github.com/theupdateframework/go-tuf/releases/download/v0.7.0/tuf_0.7.0_darwin_amd64.tar.gz and place the `tuf` executable in `/usr/local/bin/tuf`. You will need to make an exception in "Privacy & Security" because the executable is not signed. + +### 1Password + +You need to create three passphrases on your private 1Password vault for encrypting the signing keys (more on signing keys below). +Create three private "passwords" with the following names: `TUF TARGETS`, `TUF SNAPSHOT` and `TUF TIMESTAMP`. +The resulting credentials will have the following "path" within 1Password (these paths will be provided to the `releaser.sh` script) +```sh +Private/TUF TARGETS/password +Private/TUF SNAPSHOT/password +Private/TUF TIMESTAMP/password +``` + +### AWS + +The following is required to be able to run `aws` cli commands. + +1. You will need to request the infrastructure team to add the "TUFAdministrators" role to your Google account. +2. Configure AWS SSO with the following steps: https://github.com/fleetdm/confidential/tree/main/infrastructure/sso#how-to-use-sso. +Set the profile name as `tuf` (the profile name will be provided to the `releaser.sh` script). +3. Test the access by running: +```sh +AWS_PROFILE=tuf aws sso login +``` + +### TUF signing keys + +> You can skip this step if you already have authorized keys to sign and publish updates. + +To release updates to our TUF repository you need the `root` role (ask in Slack who has such `root` role) to sign your signing keys. +First, run the following script +```sh +AWS_PROFILE=tuf \ +ACTION=generate-signing-keys \ +TUF_DIRECTORY=/Users/luk/tuf3.fleetctl.com \ +TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \ +SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \ +TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \ +./tools/tuf/releaser.sh +``` + +The human with the `root` role will run the following commands to sign the provided `staged/root.json`: +```sh +tuf sign +tuf snapshot +tuf timestamp +tuf commit +``` +And push the newly signed `root.json` to the remote repository. + +### Encrypted keys in USB + +For releasing fleetd you need to plug in the USB that contains encrypted signing keys. +In this guide we assume the USB device will be mounted in `/Volumes/FLEET-TUF/` and it ONLY contains a `keys/` directory. + +### Github + +#### Personal access token + +> A personal access token is required to download artifacts from Github Actions using the Github API. + +1. Create a fine-grained personal access token at https://github.com/settings/tokens?type=beta +2. Store the token on 1Password as a "password" with name "Github Token" +The resulting credential will have the following "path" within 1Password (this path will be provided to the script) +```sh +Private/Github Token/password +``` + +#### Github session + +You need to log in to your Github account with your default browser. +It will be used to open your browser and allow you to create the PR needed to build artifacts (this can be improved later, see TODOs). + +## Samples + +Following are samples of the script execution to release components to `edge` and `stable`. + +> When releasing fleetd you need to checkout the branch (e.g. `main`) you want to release. + +> NOTE: When releasing fleetd: +> If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped +> (even if there are no changes in it). This is due to the fact that we want users to see the new version in the tray icon, +> e.g. `"Fleet Desktop v1.21.0"`. Technical debt: We could improve this process to reduce the complexity of releasing +> fleetd when there are no Fleet Desktop changes. + +### Releasing to `edge` + +#### Releasing fleetd `1.23.0` to `edge` + +```sh +AWS_PROFILE=tuf \ +TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \ +COMPONENT=fleetd \ +ACTION=release-to-edge \ +VERSION=1.23.0 \ +KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \ +TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \ +SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \ +TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \ +GITHUB_USERNAME=foobar \ +GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \ +PUSH_TO_REMOTE=1 \ +./tools/tuf/releaser.sh +``` + +#### Releasing osquery `5.12.1` to `edge` + +```sh +AWS_PROFILE=tuf \ +TUF_DIRECTORY=/Users/luk/tuf.fleetctl.com \ +COMPONENT=osqueryd \ +ACTION=release-to-edge \ +VERSION=5.12.1 \ +KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \ +TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \ +SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \ +TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \ +GITHUB_USERNAME=foobar \ +GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \ +PUSH_TO_REMOTE=1 \ +./tools/tuf/releaser.sh +``` + +### Promoting from `edge` to `stable` + +#### Promoting fleetd `1.23.0` from `edge` to `stable` + +```sh +AWS_PROFILE=tuf \ +TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \ +COMPONENT=fleetd \ +ACTION=promote-edge-to-stable \ +VERSION=1.23.0 \ +KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \ +TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \ +SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \ +TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \ +GITHUB_USERNAME=foobar \ +GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \ +PUSH_TO_REMOTE=1 \ +./tools/tuf/releaser.sh +``` + +#### Promoting osqueryd `5.12.1` from `edge` to `stable` + +```sh +AWS_PROFILE=tuf \ +TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \ +COMPONENT=osqueryd \ +ACTION=promote-edge-to-stable \ +VERSION=5.12.1 \ +KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \ +TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \ +SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \ +TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \ +GITHUB_USERNAME=foobar \ +GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \ +PUSH_TO_REMOTE=1 \ +./tools/tuf/releaser.sh +``` + +#### Releasing `swiftDialog` to `stable` + +> `releaser.sh` doesn't support `swiftDialog` yet. +> macOS only component + +The `swiftDialog` executable can be generated from a macOS host by running: +```sh +make swift-dialog-app-tar-gz version=2.2.1 build=4591 out-path=. +``` +```sh +fleetctl updates add --target /path/to/macos/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version 2.2.1 -t edge +``` + +#### Releasing `nudge` to `stable` + +> `releaser.sh` doesn't support `nudge` yet. +> macOS only component + +The `nudge` executable can be generated from a macOS host by running: +```sh +make nudge-app-tar-gz version=1.1.10.81462 out-path=. +``` +```sh +fleetctl updates add --target /path/to/macos/nudge.app.tar.gz --platform macos --name nudge --version 1.1.10.81462 -t edge +``` + +## Testing and improving the script + +- You can specify `GIT_REPOSITORY_DIRECTORY` to set a separate path for the Fleet repository (it uses the current by default). +This is sometimes necessary if the tooling the script uses is not present in the branch we are trying to release from. +```sh +git clone git@github.com:fleetdm/fleet.git +GIT_REPOSITORY_DIRECTORY= +``` + +- If the PR and orbit tag were already generated but you need to run the script again you can set `SKIP_PR_AND_TAG_PUSH=1` to skip that part. + +- While developing you can run with `PUSH_TO_REMOTE=0` to prevent pushing invalid metadata/components to the production repository. + +## TODOs to improve releaser.sh + +- Create the pull requests automatically using `gh` or the Github API. +- Support releasing `nudge` and `swiftDialog`. + +## Troubleshooting + +### Removing Unused Targets + +If you've inadvertently published a target that is no longer in use, follow these steps to remove it. + +> Before performing any actions on Fleet's TUF repository you must: +> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository). +> 2. Create a local backup in case we mess up with the repository: +> ```sh +> mkdir ~/tuf.fleetctl.com/backup +> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup +> ``` + +1. You'll need the [`go-tuf`](https://github.com/theupdateframework/go-tuf) binary. The removal operations aren't integrated into `fleetctl` at the moment. +2. Use `tuf remove` to remove the target and update `targets.json`. Substitute `desktop/windows/stable/desktop.exe` with the target you intend to delete. +```sh +tuf remove desktop/windows/stable/desktop.exe +``` +3. Snapshot, timestamp, and commit the changes. +```sh +tuf snapshot +tuf timestamp +tuf commit +``` +4. Run the following command to generate a timestamp that expires in two weeks (otherwise the default expiration when using `go-tuf` commands is 1 day) +```sh +fleetctl updates timestamp +``` +5. Confirm that the version of the local `timestamp.json` file is more recent than that of the remote server. +6. Verify the changes that will be synced by running a dry sync. Include the `--delete` flag as you're removing targets. +```sh +aws s3 sync ./repository s3://fleet-tuf-repo --delete --dryrun +``` +7. `diff` the local `targets.json` file with its remote version. +8. To upload the changes, perform a sync without the `--dryrun`: +```sh +aws s3 sync ./repository s3://fleet-tuf-repo --delete +``` + +### Invalid timestamp.json version + +The following issue was solved by resigning the timestamp metadata `fleetctl updates timestamp` (executed three times to increase the version to `4175`) +```sh +2022-08-23T13:44:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174" +2022-08-23T13:59:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174" +``` diff --git a/tools/tuf/download-artifacts/download-artifacts.go b/tools/tuf/download-artifacts/download-artifacts.go index e0df2e3dec..2f0d333cff 100644 --- a/tools/tuf/download-artifacts/download-artifacts.go +++ b/tools/tuf/download-artifacts/download-artifacts.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/fleetdm/fleet/v4/orbit/pkg/constant" "github.com/fleetdm/fleet/v4/pkg/fleethttp" @@ -37,6 +38,7 @@ func orbitCommand() *cli.Command { outputDirectory string githubUsername string githubAPIToken string + retry bool ) return &cli.Command{ Name: "orbit", @@ -70,13 +72,19 @@ func orbitCommand() *cli.Command { Destination: &githubAPIToken, Usage: "Github API token (https://github.com/settings/tokens)", }, + &cli.BoolFlag{ + Name: "retry", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"}, + Destination: &retry, + Usage: "Whether to retry if the artifact doesn't exist yet", + }, }, Action: func(c *cli.Context) error { return downloadComponents("goreleaser-orbit.yaml", gitTag, map[string]string{ "macos": "orbit-macos", "linux": "orbit-linux", "windows": "orbit-windows", - }, outputDirectory, githubUsername, githubAPIToken) + }, outputDirectory, githubUsername, githubAPIToken, retry) }, } } @@ -87,6 +95,7 @@ func desktopCommand() *cli.Command { outputDirectory string githubUsername string githubAPIToken string + retry bool ) return &cli.Command{ Name: "desktop", @@ -120,13 +129,19 @@ func desktopCommand() *cli.Command { Destination: &githubAPIToken, Usage: "Github API token (https://github.com/settings/tokens)", }, + &cli.BoolFlag{ + Name: "retry", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"}, + Destination: &retry, + Usage: "Whether to retry if the artifact doesn't exist yet", + }, }, Action: func(c *cli.Context) error { return downloadComponents("generate-desktop-targets.yml", gitBranch, map[string]string{ "macos": "desktop.app.tar.gz", "linux": "desktop.tar.gz", "windows": "fleet-desktop.exe", - }, outputDirectory, githubUsername, githubAPIToken) + }, outputDirectory, githubUsername, githubAPIToken, retry) }, } } @@ -231,7 +246,7 @@ func extractZipFile(archiveReader *zip.File, destPath string) error { return nil } -func downloadComponents(workflowName string, headBranch string, artifactNames map[string]string, outputDirectory string, githubUsername string, githubAPIToken string) error { +func downloadComponents(workflowName string, headBranch string, artifactNames map[string]string, outputDirectory string, githubUsername string, githubAPIToken string, retry bool) error { if err := os.RemoveAll(outputDirectory); err != nil { return err } @@ -241,40 +256,55 @@ func downloadComponents(workflowName string, headBranch string, artifactNames ma } } ctx := context.Background() - gc := github.NewClient(fleethttp.NewClient()) - workflow, _, err := gc.Actions.GetWorkflowByFileName(ctx, "fleetdm", "fleet", workflowName) - if err != nil { - return err - } - workflowRuns, _, err := gc.Actions.ListWorkflowRunsByID(ctx, "fleetdm", "fleet", *workflow.ID, nil) - if err != nil { - return err - } var workflowRun *github.WorkflowRun - for _, wr := range workflowRuns.WorkflowRuns { - if headBranch == *wr.HeadBranch { - workflowRun = wr + gc := github.NewClient(fleethttp.NewClient()) + for { + workflow, _, err := gc.Actions.GetWorkflowByFileName(ctx, "fleetdm", "fleet", workflowName) + if err != nil { + return err + } + workflowRuns, _, err := gc.Actions.ListWorkflowRunsByID(ctx, "fleetdm", "fleet", *workflow.ID, nil) + if err != nil { + return err + } + for _, wr := range workflowRuns.WorkflowRuns { + if headBranch == *wr.HeadBranch { + workflowRun = wr + break + } + } + if workflowRun != nil || !retry { break } + fmt.Printf("Workflow not available yet, it might be queued, retrying in 60s...\n") + time.Sleep(60 * time.Second) } if workflowRun == nil { return fmt.Errorf("workflow with tag %s not found", headBranch) } - artifactList, _, err := gc.Actions.ListWorkflowRunArtifacts(ctx, "fleetdm", "fleet", *workflowRun.ID, nil) - if err != nil { - return err - } - urls := make(map[string]string) - for _, artifact := range artifactList.Artifacts { - if *artifact.Name == artifactNames["linux"] { - urls["linux"] = *artifact.ArchiveDownloadURL - } else if *artifact.Name == artifactNames["macos"] { - urls["macos"] = *artifact.ArchiveDownloadURL - } else if *artifact.Name == artifactNames["windows"] { - urls["windows"] = *artifact.ArchiveDownloadURL - } else { - return fmt.Errorf("unknown artifact name: %s", *artifact.Name) + var urls map[string]string + for { + artifactList, _, err := gc.Actions.ListWorkflowRunArtifacts(ctx, "fleetdm", "fleet", *workflowRun.ID, nil) + if err != nil { + return err } + urls = make(map[string]string) + for _, artifact := range artifactList.Artifacts { + if *artifact.Name == artifactNames["linux"] { + urls["linux"] = *artifact.ArchiveDownloadURL + } else if *artifact.Name == artifactNames["macos"] { + urls["macos"] = *artifact.ArchiveDownloadURL + } else if *artifact.Name == artifactNames["windows"] { + urls["windows"] = *artifact.ArchiveDownloadURL + } else { + return fmt.Errorf("unknown artifact name: %s", *artifact.Name) + } + } + if len(urls) == 3 || !retry { + break + } + fmt.Printf("All artifacts are not available yet, the workflow might still be running, retrying in 60s...\n") + time.Sleep(60 * time.Second) } if len(urls) != 3 { return fmt.Errorf("missing some artifact: %+v", urls) @@ -295,6 +325,7 @@ func osquerydCommand() *cli.Command { outputDirectory string githubUsername string githubAPIToken string + retry bool ) return &cli.Command{ Name: "osqueryd", @@ -328,13 +359,19 @@ func osquerydCommand() *cli.Command { Destination: &githubAPIToken, Usage: "Github API token (https://github.com/settings/tokens)", }, + &cli.BoolFlag{ + Name: "retry", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"}, + Destination: &retry, + Usage: "Whether to retry if the artifact doesn't exist yet", + }, }, Action: func(c *cli.Context) error { return downloadComponents("generate-osqueryd-targets.yml", gitBranch, map[string]string{ "macos": "osqueryd.app.tar.gz", "linux": "osqueryd", "windows": "osqueryd.exe", - }, outputDirectory, githubUsername, githubAPIToken) + }, outputDirectory, githubUsername, githubAPIToken, retry) }, } } diff --git a/tools/tuf/promote_edge_to_stable.sh b/tools/tuf/promote_edge_to_stable.sh deleted file mode 100755 index 32829deff1..0000000000 --- a/tools/tuf/promote_edge_to_stable.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - - -component=$1 -version=$2 - -if [[ -z $component || -z $version ]]; then - echo "Usage: $0 " - exit 1 -fi - -if [[ ! -d "./repository" ]]; then - echo "Directory ./repository doesn't exist" - exit 1 -fi - -version_parts=(${version//./ }) -major=${version_parts[0]} -minor=${version_parts[1]} - -echo "Promoting $component from edge to stable, version='$version'" -echo "Press any key to continue..." -read -s -n 1 - -case $1 in - orbit) - fleetctl updates add --target ./repository/targets/orbit/macos/edge/orbit --platform macos --name orbit --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/orbit/linux/edge/orbit --platform linux --name orbit --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/orbit/windows/edge/orbit.exe --platform windows --name orbit --version $version -t $major.$minor -t $major -t stable - ;; - desktop) - fleetctl updates add --target ./repository/targets/desktop/macos/edge/desktop.app.tar.gz --platform macos --name desktop --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/desktop/linux/edge/desktop.tar.gz --platform linux --name desktop --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/desktop/windows/edge/fleet-desktop.exe --platform windows --name desktop --version $version -t $major.$minor -t $major -t stable - ;; - osqueryd) - fleetctl updates add --target ./repository/targets/osqueryd/macos-app/edge/osqueryd.app.tar.gz --platform macos-app --name osqueryd --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/osqueryd/linux/edge/osqueryd --platform linux --name osqueryd --version $version -t $major.$minor -t $major -t stable - fleetctl updates add --target ./repository/targets/osqueryd/windows/edge/osqueryd.exe --platform windows --name osqueryd --version $version -t $major.$minor -t $major -t stable - ;; - nudge) - fleetctl updates add --target ./repository/targets/nudge/macos/edge/nudge.app.tar.gz --platform macos --name nudge --version $version -t stable - ;; - swiftDialog) - fleetctl updates add --target ./repository/targets/swiftDialog/macos/edge/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version $version -t stable - ;; - *) - echo Unknown component $1 - exit 1 - ;; -esac diff --git a/tools/tuf/releaser.sh b/tools/tuf/releaser.sh new file mode 100755 index 0000000000..527314bd01 --- /dev/null +++ b/tools/tuf/releaser.sh @@ -0,0 +1,305 @@ +#!/bin/bash + +# +# For usage documentation, see the README.md. +# + +set -e + +# +# Input environment variables: +# +# AWS_PROFILE +# TUF_DIRECTORY +# COMPONENT +# ACTION +# VERSION +# KEYS_SOURCE_DIRECTORY +# TARGETS_PASSPHRASE_1PASSWORD_PATH +# SNAPSHOT_PASSPHRASE_1PASSWORD_PATH +# TIMESTAMP_PASSPHRASE_1PASSWORD_PATH +# GITHUB_USERNAME +# GITHUB_TOKEN_1PASSWORD_PATH +# SKIP_PR_AND_TAG_PUSH +# + +# +# Dev environment variables: +# PUSH_TO_REMOTE +# GIT_REPOSITORY_DIRECTORY +# + +clean_up () { + echo "Cleaning up directories..." + + # Make sure (best effort) to remove the keys after we are done. + rm -rf "$KEYS_DIRECTORY" + rm -rf "$ARTIFACTS_DOWNLOAD_DIRECTORY" + rm -rf "$GO_TOOLS_DIRECTORY" + ARG=$? + exit $ARG +} + +setup () { + echo "Running setup..." + + GO_TOOLS_DIRECTORY=$(mktemp -d) + ARTIFACTS_DOWNLOAD_DIRECTORY=$(mktemp -d) + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + REPOSITORY_DIRECTORY=$TUF_DIRECTORY/repository + STAGED_DIRECTORY=$TUF_DIRECTORY/staged + KEYS_DIRECTORY=$TUF_DIRECTORY/keys + if [[ -z $GIT_REPOSITORY_DIRECTORY ]]; then + GIT_REPOSITORY_DIRECTORY=$( realpath "$SCRIPT_DIR/../.." ) + fi + + mkdir -p "$REPOSITORY_DIRECTORY" + mkdir -p "$STAGED_DIRECTORY" + cp -r "$KEYS_SOURCE_DIRECTORY" "$KEYS_DIRECTORY" + + if ! aws sts get-caller-identity &> /dev/null; then + aws sso login + prompt "AWS SSO login was successful, press any key to continue..." + fi + + GITHUB_TOKEN=$(op read "op://$GITHUB_TOKEN_1PASSWORD_PATH") + + # These need to be exported for use by `fleetctl updates` commands. + FLEET_TARGETS_PASSPHRASE=$(op read "op://$TARGETS_PASSPHRASE_1PASSWORD_PATH") + export FLEET_TARGETS_PASSPHRASE + FLEET_SNAPSHOT_PASSPHRASE=$(op read "op://$SNAPSHOT_PASSPHRASE_1PASSWORD_PATH") + export FLEET_SNAPSHOT_PASSPHRASE + FLEET_TIMESTAMP_PASSPHRASE=$(op read "op://$TIMESTAMP_PASSPHRASE_1PASSWORD_PATH") + export FLEET_TIMESTAMP_PASSPHRASE + + go build -o "$GO_TOOLS_DIRECTORY/replace" "$SCRIPT_DIR/../../tools/tuf/replace" + go build -o "$GO_TOOLS_DIRECTORY/download-artifacts" "$SCRIPT_DIR/../../tools/tuf/download-artifacts" +} + +pull_from_remote () { + echo "Pulling repository from tuf.fleetctl.com... (--dryrun first)" + aws s3 sync s3://fleet-tuf-repo "$REPOSITORY_DIRECTORY" --exact-timestamps --dryrun + prompt "If the --dryrun looks good, press any key to continue... (no output means nothing to update)" + aws s3 sync s3://fleet-tuf-repo "$REPOSITORY_DIRECTORY" --exact-timestamps +} + +promote_component_edge_to_stable () { + component_name=$1 + component_version=$2 + + version_parts=("${component_version//./ }") + major=${version_parts[0]} + minor=${version_parts[1]} + + case $component_name in + orbit) + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/macos/edge/orbit" --platform macos --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/linux/edge/orbit" --platform linux --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/windows/edge/orbit.exe" --platform windows --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable + ;; + desktop) + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/macos/edge/desktop.app.tar.gz" --platform macos --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/linux/edge/desktop.tar.gz" --platform linux --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/windows/edge/fleet-desktop.exe" --platform windows --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable + ;; + osqueryd) + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/macos-app/edge/osqueryd.app.tar.gz" --platform macos-app --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/linux/edge/osqueryd" --platform linux --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable + fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/windows/edge/osqueryd.exe" --platform windows --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable + ;; + *) + echo "Unknown component $component_name" + exit 1 + ;; + esac +} + +promote_edge_to_stable () { + cd "$REPOSITORY_DIRECTORY" + if [[ $COMPONENT == "fleetd" ]]; then + echo "Promoting fleetd from edge to stable..." + promote_component_edge_to_stable orbit "$VERSION" + promote_component_edge_to_stable desktop "$VERSION" + elif [[ $COMPONENT == "osqueryd" ]]; then + echo "Promoting osqueryd from edge to stable..." + promote_component_edge_to_stable osqueryd "$VERSION" + else + echo "Unsupported component: $COMPONENT" + exit 1 + fi +} + +release_fleetd_to_edge () { + echo "Releasing fleetd to edge..." + BRANCH_NAME="release-fleetd-v$VERSION" + ORBIT_TAG="orbit-v$VERSION" + if [[ "$SKIP_PR_AND_TAG_PUSH" != "1" ]]; then + prompt "A PR for bumping the fleetd version will be created to trigger a Github Action that will build 'Fleet Desktop'. Press any key to continue..." + pushd "$GIT_REPOSITORY_DIRECTORY" + git checkout -b "$BRANCH_NAME" + make changelog-orbit version="$VERSION" + ORBIT_CHANGELOG=orbit/CHANGELOG.md + "$GO_TOOLS_DIRECTORY/replace" .github/workflows/generate-desktop-targets.yml "FLEET_DESKTOP_VERSION: .+\n" "FLEET_DESKTOP_VERSION: $VERSION\n" + git add .github/workflows/generate-desktop-targets.yml "$ORBIT_CHANGELOG" + git commit -m "Release fleetd $VERSION" + git push origin "$BRANCH_NAME" + open "https://github.com/fleetdm/fleet/pull/new/$BRANCH_NAME" + prompt "Press any key to continue after the PR is created..." + prompt "A 'git tag' will be created to trigger a Github Action to build orbit, press any key to continue..." + git tag "$ORBIT_TAG" + git push origin "$ORBIT_TAG" + popd + fi + DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/desktop" + mkdir -p "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY" + "$GO_TOOLS_DIRECTORY/download-artifacts" desktop \ + --git-branch "$BRANCH_NAME" \ + --output-directory "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY" \ + --github-username "$GITHUB_USERNAME" --github-api-token "$GITHUB_TOKEN" \ + --retry + ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/orbit" + mkdir -p "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY" + "$GO_TOOLS_DIRECTORY/download-artifacts" orbit \ + --git-tag "$ORBIT_TAG" \ + --output-directory "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY" \ + --github-username "$GITHUB_USERNAME" --github-api-token "$GITHUB_TOKEN" \ + --retry + pushd "$TUF_DIRECTORY" + fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/macos/orbit" --platform macos --name orbit --version "$VERSION" -t edge + fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/linux/orbit" --platform linux --name orbit --version "$VERSION" -t edge + fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/windows/orbit.exe" --platform windows --name orbit --version "$VERSION" -t edge + fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/macos/desktop.app.tar.gz" --platform macos --name desktop --version "$VERSION" -t edge + fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/linux/desktop.tar.gz" --platform linux --name desktop --version "$VERSION" -t edge + fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/windows/fleet-desktop.exe" --platform windows --name desktop --version "$VERSION" -t edge + popd +} + +release_osqueryd_to_edge () { + echo "Releasing osqueryd to edge..." + prompt "A branch and PR for bumping the osquery version will be created. Press any key to continue..." + BRANCH_NAME=release-osqueryd-v$VERSION + if [[ "$SKIP_PR_AND_TAG_PUSH" != "1" ]]; then + pushd "$GIT_REPOSITORY_DIRECTORY" + git checkout -b "$BRANCH_NAME" + "$GO_TOOLS_DIRECTORY/replace" .github/workflows/generate-osqueryd-targets.yml "OSQUERY_VERSION: .+\n" "OSQUERY_VERSION: $VERSION\n" + git add .github/workflows/generate-osqueryd-targets.yml + git commit -m "Bump osqueryd version to $VERSION" + git push origin "$BRANCH_NAME" + open "https://github.com/fleetdm/fleet/pull/new/$BRANCH_NAME" + prompt "Press any key to continue after the PR is created..." + popd + fi + OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/osqueryd" + mkdir -p "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY" + "$GO_TOOLS_DIRECTORY/download-artifacts" osqueryd \ + --git-branch "$BRANCH_NAME" \ + --output-directory "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY" \ + --github-username "$GITHUB_USERNAME" \ + --github-api-token "$GITHUB_TOKEN" \ + --retry + pushd "$TUF_DIRECTORY" + fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/macos/osqueryd.app.tar.gz" --platform macos-app --name osqueryd --version "$VERSION" -t edge + fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/linux/osqueryd" --platform linux --name osqueryd --version "$VERSION" -t edge + fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/windows/osqueryd.exe" --platform windows --name osqueryd --version "$VERSION" -t edge + popd +} + +release_to_edge () { + if [[ $COMPONENT == "fleetd" ]]; then + release_fleetd_to_edge + elif [[ $COMPONENT == "osqueryd" ]]; then + release_osqueryd_to_edge + else + echo "Unsupported component: $COMPONENT" + exit 1 + fi +} + +push_to_remote () { + echo "Running --dryrun push of repository to tuf.fleetctl.com..." + aws s3 sync "$REPOSITORY_DIRECTORY" s3://fleet-tuf-repo --dryrun + if [[ $PUSH_TO_REMOTE == "1" ]]; then + echo "WARNING: This step will push the release to tuf.fleetctl.com (production)..." + prompt "If the --dryrun looks good, press any key to continue..." + aws s3 sync "$REPOSITORY_DIRECTORY" s3://fleet-tuf-repo + echo "Release has been pushed!" + echo "NOTE: You might see some clients failing to upgrade due to some sha256 mismatches." + echo "These temporary failures are expected because it takes some time for caches to be invalidated (these errors should go away after ~15-30 minutes)." + else + echo "PUSH_TO_REMOTE not set to 1, so not pushing." + fi +} + +prompt () { + printf "%s\n" "$1" + read -r -s -n 1 +} + +setup_to_become_publisher () { + echo "Running setup to become publisher..." + + REPOSITORY_DIRECTORY=$TUF_DIRECTORY/repository + STAGED_DIRECTORY=$TUF_DIRECTORY/staged + KEYS_DIRECTORY=$TUF_DIRECTORY/keys + mkdir -p "$REPOSITORY_DIRECTORY" + mkdir -p "$STAGED_DIRECTORY" + mkdir -p "$KEYS_DIRECTORY" + if ! aws sts get-caller-identity &> /dev/null; then + aws sso login + prompt "AWS SSO login was successful, press any key to continue..." + fi + # These need to be exported for use by `tuf` commands. + FLEET_TARGETS_PASSPHRASE=$(op read "op://$TARGETS_PASSPHRASE_1PASSWORD_PATH") + export TUF_TARGETS_PASSPHRASE=$FLEET_TARGETS_PASSPHRASE + FLEET_SNAPSHOT_PASSPHRASE=$(op read "op://$SNAPSHOT_PASSPHRASE_1PASSWORD_PATH") + export TUF_SNAPSHOT_PASSPHRASE=$FLEET_SNAPSHOT_PASSPHRASE + FLEET_TIMESTAMP_PASSPHRASE=$(op read "op://$TIMESTAMP_PASSPHRASE_1PASSWORD_PATH") + export TUF_TIMESTAMP_PASSPHRASE=$FLEET_TIMESTAMP_PASSPHRASE +} + +if [[ $ACTION == "generate-signing-keys" ]]; then + setup_to_become_publisher + pull_from_remote + cd "$TUF_DIRECTORY" + tuf gen-key targets && echo + tuf gen-key snapshot && echo + tuf gen-key timestamp && echo + echo "Keys have been generated, now do the following actions:" + echo "- Share '$TUF_DIRECTORY/staged/root.json' with Fleet member with the 'root' role, who will sign with its root key and push it to the remote repository." + echo "- Store the '$TUF_DIRECTORY/keys' folder (that contains the encrypted keys) on a USB flash drive that you will ONLY use for releasing fleetd updates." + exit 0 +fi + +print_reminder () { + if [[ $ACTION == "release-to-edge" ]]; then + if [[ $COMPONENT == "fleetd" ]]; then + prompt "Make sure to install fleetd with '--orbit-channel=edge --desktop-channel=edge' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..." + elif [[ $COMPONENT == "osqueryd" ]]; then + prompt "Make sure to install fleetd with '--osqueryd-channel=edge' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..." + fi + elif [[ $ACTION == "promote-edge-to-stable" ]]; then + if [[ $COMPONENT == "fleetd" ]]; then + prompt "Make sure to install fleetd with '--orbit-channel=stable --desktop-channel=stable' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..." + elif [[ $COMPONENT == "osqueryd" ]]; then + prompt "Make sure to install fleetd with '--osqueryd-channel=stable' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..." + fi + else + echo "Unsupported action: $ACTION" + fi +} + +trap clean_up EXIT +print_reminder +setup +pull_from_remote + +if [[ $ACTION == "release-to-edge" ]]; then + release_to_edge +elif [[ $ACTION == "promote-edge-to-stable" ]]; then + promote_edge_to_stable +else + echo "Unsupported action: $ACTION" + exit 1 +fi + +push_to_remote \ No newline at end of file diff --git a/tools/tuf/replace/main.go b/tools/tuf/replace/main.go new file mode 100644 index 0000000000..ba742f09e7 --- /dev/null +++ b/tools/tuf/replace/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "regexp" + "strings" +) + +// This tool was created to prevent issues between GNU's sed and OSX's sed. + +func main() { + inputPath := os.Args[1] + expression := os.Args[2] + replace := os.Args[3] + r := regexp.MustCompile(expression) + stat, err := os.Stat(inputPath) + if err != nil { + panic(err) + } + input, err := os.ReadFile(inputPath) + if err != nil { + panic(err) + } + if strings.HasSuffix(replace, `\n`) { + replace = strings.TrimSuffix(replace, `\n`) + "\n" + } + output := r.ReplaceAllString(string(input), replace) + if err := os.WriteFile(inputPath, []byte(output), stat.Mode()); err != nil { + panic(err) + } +} diff --git a/tools/tuf/test/create_repository.sh b/tools/tuf/test/create_repository.sh index 8772db5b37..9131852d5b 100755 --- a/tools/tuf/test/create_repository.sh +++ b/tools/tuf/test/create_repository.sh @@ -82,7 +82,7 @@ for system in $SYSTEMS; do ORBIT_BINARY_PATH=$orbit_target \ go run ./orbit/tools/build/build.go else - GOOS=$goose_value GOARCH=$goarch_value go build -ldflags="-X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=42" -o $orbit_target ./orbit/cmd/orbit + CGO_ENABLED=0 GOOS=$goose_value GOARCH=$goarch_value go build -ldflags="-X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=42" -o $orbit_target ./orbit/cmd/orbit fi ./build/fleetctl updates add \