Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-04-21 13:37:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

Create releaser.sh to release fleetd updates (#18134)

Browse source 
#16131

`releaser.sh` was used to release fleetd 1.23.0.
This commit is contained in:
Lucas Manuel Rodriguez 2024-04-09 18:37:55 -03:00 committed by GitHub
parent 3367b7e036
commit 23772c69b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 702 additions and 477 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.vscode/extensions.json vendored
View file

@ -7,6 +7,7 @@
"redhat.vscode-yaml",
"dbaeumer.vscode-eslint",
"firefox-devtools.vscode-firefox-debug",
"editorconfig.editorconfig"
"editorconfig.editorconfig",
"timonwong.shellcheck"
]
}

6
Makefile
View file

@ -318,7 +318,9 @@ changelog:
sh -c "git rm changes/*"
changelog-orbit:
sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {}; echo' > new-CHANGELOG.md"
$(eval TODAY_DATE := $(shell date "+%b %d, %Y"))
@echo -e "## Orbit $(version) ($(TODAY_DATE))\n" > new-CHANGELOG.md
sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {} | sed -E "s/^-/*/"; echo' >> new-CHANGELOG.md"
sh -c "cat new-CHANGELOG.md orbit/CHANGELOG.md > tmp-CHANGELOG.md && rm new-CHANGELOG.md && mv tmp-CHANGELOG.md orbit/CHANGELOG.md"
sh -c "git rm orbit/changes/*"
@ -394,7 +396,7 @@ ifneq ($(shell uname), Darwin)
@exit 1
endif
# locking the version of swiftDialog to 2.2.1-4591 as newer versions
# migth have layout issues.
# might have layout issues.
ifneq ($(version), 2.2.1)
@echo "Version is locked at 2.1.0, see comments in Makefile target for details"
@exit 1

392
orbit/docs/TUF-Update-Guide.md
View file

@ -1,392 +0,0 @@
# Pushing new releases to TUF
This document is a walkthrough guide for:
- A Fleet member to publish new updates to [Fleet's TUF service](tuf.fleetctl.com). See [Pushing updates](#pushing-updates).
- A Fleet member to delete targets from [Fleet's TUF service](tuf.fleetctl.com). See [Removing unused targets](#removing-unused-targets).
- A Fleet member to become a publisher of updates for [Fleet's TUF service](tuf.fleetctl.com). See [Becoming a new Fleet publisher](#becoming-a-new-fleet-publisher).
Video walkthrough related to this process for additional [context](https://drive.google.com/file/d/1c_iukFEMne12Cxx9WVTt_j1Wp0sC1kQU/view?usp=drive_link).
## Security
- The TUF keys for `targets`, `snapshot` and `timestamp` should be stored on a USB stick (used solely for this purpose). Whenever you need to push updates to Fleet's TUF repository you can temporarily copy the encrypted keys to your workstation (under the `keys/` folder, more on this below).
- The keys should be stored encrypted with its passphrase stored in 1Password (on a private vault).
- Every `fleetctl updates` command will prompt for the passphrases to decrypt the encrypted keys. You can input the passphrases every time or can alternatively set the following environment variables: `FLEET_TIMESTAMP_PASSPHRASE`, `FLEET_SNAPSHOT_PASSPHRASE` and `FLEET_TARGETS_PASSPHRASE`. Make sure to not leave traces of the passphrases (scripts, history and/or environment) when you are done.
## Syncing Fleet's TUF repository
> The `fleetctl updates` commands assume the folders `keys/`, `staged/` and `repository/` exist on the current working directory.
> IMPORTANT: When syncing the repository make sure to use `--exact-timestamps`. Otherwise `aws s3 sync` may not sync files that do not change in size, like `timestamp.json`.
- The `keys/` folder contains the encrypted private keys.
- The `staged/` folder contains uncommitted changes (usually empty because `fleetctl updates` commands automatically commit the changes).
- The `repository/` folder contains the full TUF repository.
Following are the commands to initialize the repository on your workstation:
```sh
mkdir /path/to/tuf.fleetctl.com
cd /path/to/tuf.fleetctl.com
mkdir -p ./repository
cp /Volumes/YOUR-USB-NAME/keys ./keys
mkdir -p ./staged
export AWS_PROFILE=tuf
aws sso login
aws s3 sync s3://fleet-tuf-repo ./repository --exact-timestamps
```
## Building the components for releasing to `edge`
### fleetd
> Assuming we are releasing version 1.21.0 of fleetd.
1. Create the fleetd changelog for the new release:
```sh
git checkout main
git pull origin main
git checkout -b release-fleetd-v1.21.0
make changelog-orbit
```
2. Edit `orbit/CHANGELOG.md` accordingly
3. Bump Fleet Desktop version in https://github.com/fleetdm/fleet/blob/9ca85411a16c504087d2793f8b9099f98054c93f/.github/workflows/generate-desktop-targets.yml#L27. This will trigger a github action to build the Fleet Desktop executables: https://github.com/fleetdm/fleet/actions/workflows/generate-desktop-targets.yml.
4. Commit the changes, push the branch and create a PR.
5. Add the following git tag with the following format: `orbit-v1.21.0`. Once pushed this will trigger a github action to build the orbit executables: https://github.com/fleetdm/fleet/blob/main/.github/workflows/goreleaser-orbit.yaml.
```sh
git tag orbit-v1.21.0
git push origin --tags
```
6. Once the two github actions finish their runs, use the following scripts that will download the artifacts to a folder in your workstation (on this guide we assume you are using `$HOME/release-friday`).
NOTE: The `goreleaser-macos` job is unstable and may need several re-runs until it works.
```sh
go run ./tools/tuf/download-artifacts desktop \
--git-branch release-fleetd-v1.21.0 \
--output-directory $HOME/release-friday/desktop \
--github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN
go run ./tools/tuf/download-artifacts orbit \
--git-tag orbit-v1.21.0 \
--output-directory $HOME/release-friday/orbit \
--github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN
tree $HOME/release-friday
$HOME/release-friday
├── desktop
│   ├── linux
│   │   └── desktop.tar.gz
│   ├── macos
│   │   └── desktop.app.tar.gz
│   └── windows
│   └── fleet-desktop.exe
└── orbit
├── linux
│   └── orbit
├── macos
│   └── orbit
└── windows
└── orbit.exe
```
7. With the executables on your workstation, proceed to [Pushing updates](#pushing-updates) (`edge`).
8. Manually run (`Run workflow`) this action that will update the released versions on our doc: https://github.com/fleetdm/fleet/actions/workflows/fleetd-tuf.yml.
### osqueryd
> Assuming we are releasing version 5.12.0 of osqueryd.
1. Bump osquery version in https://github.com/fleetdm/fleet/blob/30a36b0b3a1fd50e48d98a4c3c955595022f5277/.github/workflows/generate-osqueryd-targets.yml#L27.
2. Commit the changes, push the branch (assuming branch name is `bump-osqueryd-5.12.0`) and create a PR.
3. Once the Github action completes run the following (the [GitHub API token](https://github.com/settings/tokens?type=beta) does not need any special permissions -- public repository access is sufficient):
```sh
go run ./tools/tuf/download-artifacts osqueryd \
--git-branch bump-osqueryd-5.12.0 \
--output-directory $HOME/release-friday/osqueryd \
--github-username $GITHUB_USERNAME \
--github-api-token $GITHUB_TOKEN
tree $HOME/release-friday/osqueryd
$HOME/release-friday/osqueryd
├── linux
│   └── osqueryd
├── macos
│   └── osqueryd.app.tar.gz
└── windows
└── osqueryd.exe
```
4. With the executables on your workstation, proceed to [Pushing updates](#pushing-updates) (`edge`).
5. Manually run (`Run workflow`) this action that will update the released versions on our docs: https://github.com/fleetdm/fleet/actions/workflows/fleetd-tuf.yml.
## Pushing updates
> Before performing any actions on Fleet's TUF repository you must:
> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository).
> 2. Create a local backup in case we mess up with the repository:
> ```sh
> mkdir ~/tuf.fleetctl.com/backup
> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup
> ```
> 3. Install fleetd on macOS, Linux and Windows VMs using the channel (`stable` or `edge`) you are about to release.
> You can do this using the following flags in `fleetctl package`: `--orbit-channel`, `--desktop-channel`, `--osqueryd-channel`.
### Releasing to the `edge` channel
The commands shown here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com).
#### Setup
Make sure to install fleetd components using the `edge` channels in the three supported OSs (this is useful to smoke test the update).
Here's how to generate the packages:
```sh
# (The same for --type=deb and --type=msi.)
fleetctl package --type=pkg \
--enable-scripts \
--fleet-desktop \
--fleet-url=... --enroll-secret=... \
--update-interval 10s \
--orbit-channel edge --desktop-channel edge --osqueryd-channel edge
```
#### orbit
The `orbit` executables are downloaded from the [GoReleaser Orbit action](https://github.com/fleetdm/fleet/actions/workflows/goreleaser-orbit.yaml).
Such action is triggered when git tagging a new orbit version with a tag of the form: `orbit-v1.21.0`.
> IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it).
> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`.
> Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes.
> The following commands assume you are pushing version `1.21.0`.
```sh
# macOS
fleetctl updates add --target $HOME/release-friday/orbit/macos/orbit --platform macos --name orbit --version 1.21.0 -t edge
# Linux
fleetctl updates add --target $HOME/release-friday/orbit/linux/orbit --platform linux --name orbit --version 1.21.0 -t edge
# Windows
fleetctl updates add --target $HOME/release-friday/orbit/windows/orbit.exe --platform windows --name orbit --version 1.21.0 -t edge
```
#### desktop
The Fleet Desktop executables are downloaded from the [Generate Fleet Desktop targets for Orbit action](https://github.com/fleetdm/fleet/actions/workflows/generate-desktop-targets.yml).
Such action is triggered by submitting a PR with the [following version string](https://github.com/fleetdm/fleet/blob/4a6bf0d447a2080f994da1e2f36ce6d51db88109/.github/workflows/generate-desktop-targets.yml#L27) changed.
> The following commands assume you are pushing version `1.21.0`.
```sh
# macOS
fleetctl updates add --target $HOME/release-friday/desktop/macos/desktop.app.tar.gz --platform macos --name desktop --version 1.21.0 -t edge
# Linux
fleetctl updates add --target $HOME/release-friday/desktop/linux/desktop.tar.gz --platform linux --name desktop --version 1.21.0 -t edge
# Windows
fleetctl updates add --target $HOME/release-friday/desktop/windows/fleet-desktop.exe --platform windows --name desktop --version 1.21.0 -t edge
```
#### swiftDialog
> macOS only component
The `swiftDialog` executable can be generated from a macOS host by running:
```sh
make swift-dialog-app-tar-gz version=2.2.1 build=4591 out-path=.
```
```sh
fleetctl updates add --target /path/to/macos/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version 2.2.1 -t edge
```
#### nudge
> macOS only component
The `nudge` executable can be generated from a macOS host by running:
```sh
make nudge-app-tar-gz version=1.1.10.81462 out-path=.
```
```sh
fleetctl updates add --target /path/to/macos/nudge.app.tar.gz --platform macos --name nudge --version 1.1.10.81462 -t edge
```
#### osqueryd
Osquery executables are downloaded from the [Generate osqueryd targets for Fleetd action](https://github.com/fleetdm/fleet/blob/main/.github/workflows/generate-osqueryd-targets.yml).
Such action is triggered by submitting a PR with the [following version string](https://github.com/fleetdm/fleet/blob/7067ca586a4aa1a0377b387d4b4478a5958193ff/.github/workflows/generate-osqueryd-targets.yml#L27) changed.
> The following commands assume you are pushing version `5.9.1`.
```sh
# macOS
fleetctl updates add --target $HOME/release-friday/osqueryd/macos/osqueryd.app.tar.gz --platform macos-app --name osqueryd --version 5.9.1 -t edge
# Linux
fleetctl updates add --target $HOME/release-friday/osqueryd/linux/osqueryd --platform linux --name osqueryd --version 5.9.1 -t edge
# Windows
fleetctl updates add --target $HOME/release-friday/osqueryd/windows/osqueryd.exe --platform windows --name osqueryd --version 5.9.1 -t edge
```
#### Push updates
Once all components are updated in your local repository we need to push the changes to the remote repository.
See [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository).
### Promoting `edge` to the `stable` channel
> Make sure to install fleetd components using the `stable` channels in the three supported OSs (this is useful to smoke test the update).
Following is the list of components and each command for each operating system.
The commands show here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com).
#### orbit
> IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it).
> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`.
> Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes.
> The following command assumes you are pushing version `1.21.0`:
```sh
/fleet/repo/tools/tuf/promote_edge_to_stable.sh orbit 1.21.0
```
#### desktop
> The following command assumes you are pushing version `1.21.0`:
```sh
/fleet/repo/tools/tuf/promote_edge_to_stable.sh desktop 1.21.0
```
#### swiftDialog
> The following command assumes you are pushing version `2.2.1`:
```sh
/fleet/repo/tools/tuf/promote_edge_to_stable.sh swiftDialog 2.2.1
```
#### nudge
> The following command assumes you are pushing version `1.1.10.81462`:
```sh
/fleet/repo/tools/tuf/promote_edge_to_stable.sh nudge 1.1.10.81462
```
#### osqueryd
> The following command assumes you are pushing version `5.9.1`.
```sh
/fleet/repo/tools/tuf/promote_edge_to_stable.sh osqueryd 5.9.1
```
#### Push updates
Once all components are updated in your local repository we need to push the changes to the remote repository.
See [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository).
### Pushing releases to Fleet's TUF repository
Once you are done with the changes on your local repository, you can use the following command to review the changes before pushing (`--dryrun` allows us to verify the upgrade before pushing):
```sh
AWS_PROFILE=tuf aws s3 sync ./repository s3://fleet-tuf-repo --dryrun
(dryrun) upload: repository/snapshot.json to s3://fleet-tuf-repo/snapshot.json
(dryrun) upload: repository/targets.json to s3://fleet-tuf-repo/targets.json
[...]
(dryrun) upload: repository/timestamp.json to s3://fleet-tuf-repo/timestamp.json
```
If all looks good, run the same command without the `--dryrun` flag.
> NOTE: Some things to note after the changes are pushed:
> - Once pushed you might see some clients failing to upgrade due to some sha256 mismatches. These temporary failures are expected because it takes some time for caches to be invalidated (these errors should go away after a few minutes).
> - The auto-update routines in orbit runs every 15 minutes, so you might need to wait up to 15 minutes to verify online hosts are auto-updating properly.
## Removing Unused Targets
If you've inadvertently published a target that is no longer in use, follow these steps to remove it.
> Before performing any actions on Fleet's TUF repository you must:
> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository).
> 2. Create a local backup in case we mess up with the repository:
> ```sh
> mkdir ~/tuf.fleetctl.com/backup
> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup
> ```
1. You'll need the [`go-tuf`](https://github.com/theupdateframework/go-tuf) binary. The removal operations aren't integrated into `fleetctl` at the moment.
2. Use `tuf remove` to remove the target and update `targets.json`. Substitute `desktop/windows/stable/desktop.exe` with the target you intend to delete.
```sh
tuf remove desktop/windows/stable/desktop.exe
```
3. Snapshot, timestamp, and commit the changes.
```sh
tuf snapshot
tuf timestamp
tuf commit
```
4. Run the following command to generate a timestamp that expires in two weeks (otherwise the default expiration when using `go-tuf` commands is 1 day)
```sh
fleetctl updates timestamp
```
5. Confirm that the version of the local `timestamp.json` file is more recent than that of the remote server.
6. Verify the changes that will be synced by running a dry sync. Include the `--delete` flag as you're removing targets.
```sh
aws s3 sync ./repository s3://fleet-tuf-repo --delete --dryrun
```
7. `diff` the local `targets.json` file with its remote version.
8. To upload the changes, perform a sync without the `--dryrun`:
```sh
aws s3 sync ./repository s3://fleet-tuf-repo --delete
```
## Becoming a New Fleet Publisher
> Before performing any actions on Fleet's TUF repository you must:
> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository).
> 2. Create a local backup in case we mess up with the repository:
> ```sh
> mkdir ~/tuf.fleetctl.com/backup
> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup
> ```
### Generate targets+snapshot+timestamp keys
All commands shown in this guide are executed from `/path/to/tuf.fleetctl.com`:
```sh
cd /path/to/tuf.fleetctl.com
```
```sh
tuf gen-key targets
Enter targets keys passphrase:
Repeat targets keys passphrase:
Generated targets key with ID ae943cb8be8a849b37c66ed46bdd7e905ba3118c0c051a6ee3cd30625855a076
```
```sh
tuf gen-key snapshot
Enter snapshot keys passphrase:
Repeat snapshot keys passphrase:
Generated snapshot key with ID 1a4d9beb826d1ff4e036d757cfcd6e36d0f041e58d25f99ef3a20ae3f8dd71e3
```
```sh
tuf gen-key timestamp
Enter timestamp keys passphrase:
Repeat timestamp keys passphrase:
Generated timestamp key with ID d940df08b59b12c30f95622a05cc40164b78a11dd7d408395ee4f79773331b30
```
Share `staged/root.json` with Fleet member with the `root` role, who will sign with its root key and push to the repository.
### Root role signs the `staged/root.json`
Essentially the following commands are executed to sign the new keys:
- `tuf sign`
- `tuf snapshot`
- `tuf timestamp`
- `tuf commit`
## Misc issues
### Invalid timestamp.json version
The following issue was solved by resigning the timestamp metadata `fleetctl updates timestamp` (executed three times to increase the version to `4175`)
```sh
2022-08-23T13:44:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174"
2022-08-23T13:59:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174"
```

292
tools/tuf/README.md Normal file
View file

@ -0,0 +1,292 @@
# Releasing updates to Fleet's TUF repository
The `releaser.sh` script automates the building and releasing of fleetd and osquery updates on [Fleet's TUF repository](https://tuf.fleetctl.com).
> - The script was developed and tested on macOS Intel.
> - It currently supports pushing new `fleetd` and `osqueryd` versions.
> - By storing credentials encrypted in a USB flash drive and storing their decryption passphrase on 1Password we are enforcing a form of 2FA.
```mermaid
graph LR;
subgraph Workstation;
releaser[releaser.sh];
1password("<div><img src='../../website/assets/images/articles/tales-from-fleet-security-securing-1password-cover-1600x900@2x.jpg' /></div>1Password");
usb("<div><img src='../../website/assets/images/articles/config-less-fleetd-agent-deployment-1600x900@2x.png' /></div>USB flash drive");
repository[(./repository)];
end;
s3("<div><img src='../../website/assets/images/icon-aws-60x36@2x.png' /></div>s3://fleet-tuf-repo");
github("<div><img src='../../website/assets/images/github-mark-white-24x24@2x.png' /></div>Github Action\n(signing and notarization)");
usb--(1) copy encrypted signing keys-->releaser;
1password--(2) get passphrases to decrypt encrypted signing keys-->releaser;
1password--(3) get Github API token-->releaser;
s3--(4) pull TUF repository-->releaser;
releaser--(5) build components (new updates)\n(osqueryd, orbit, Fleet Desktop)-->github;
github--(6) download built components-->releaser;
releaser--(7) push updates and signed metadata-->s3;
```
## Permissions and configuration
Following is the checklist for all credentials and configuration needed to run the script.
### Dependencies
- `make`
- `git`
- 1Password 8 application.
- Install and configure 1Password's `op` cli to connect to the application: https://developer.1password.com/docs/cli/get-started/
- `aws` cli :`brew install awscli`.
- `fleetctl`: Either built from source or installed by npm.
- `tuf`: Download the release from https://github.com/theupdateframework/go-tuf/releases/download/v0.7.0/tuf_0.7.0_darwin_amd64.tar.gz and place the `tuf` executable in `/usr/local/bin/tuf`. You will need to make an exception in "Privacy & Security" because the executable is not signed.
### 1Password
You need to create three passphrases on your private 1Password vault for encrypting the signing keys (more on signing keys below).
Create three private "passwords" with the following names: `TUF TARGETS`, `TUF SNAPSHOT` and `TUF TIMESTAMP`.
The resulting credentials will have the following "path" within 1Password (these paths will be provided to the `releaser.sh` script)
```sh
Private/TUF TARGETS/password
Private/TUF SNAPSHOT/password
Private/TUF TIMESTAMP/password
```
### AWS
The following is required to be able to run `aws` cli commands.
1. You will need to request the infrastructure team to add the "TUFAdministrators" role to your Google account.
2. Configure AWS SSO with the following steps: https://github.com/fleetdm/confidential/tree/main/infrastructure/sso#how-to-use-sso.
Set the profile name as `tuf` (the profile name will be provided to the `releaser.sh` script).
3. Test the access by running:
```sh
AWS_PROFILE=tuf aws sso login
```
### TUF signing keys
> You can skip this step if you already have authorized keys to sign and publish updates.
To release updates to our TUF repository you need the `root` role (ask in Slack who has such `root` role) to sign your signing keys.
First, run the following script
```sh
AWS_PROFILE=tuf \
ACTION=generate-signing-keys \
TUF_DIRECTORY=/Users/luk/tuf3.fleetctl.com \
TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \
SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \
TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \
./tools/tuf/releaser.sh
```
The human with the `root` role will run the following commands to sign the provided `staged/root.json`:
```sh
tuf sign
tuf snapshot
tuf timestamp
tuf commit
```
And push the newly signed `root.json` to the remote repository.
### Encrypted keys in USB
For releasing fleetd you need to plug in the USB that contains encrypted signing keys.
In this guide we assume the USB device will be mounted in `/Volumes/FLEET-TUF/` and it ONLY contains a `keys/` directory.
### Github
#### Personal access token
> A personal access token is required to download artifacts from Github Actions using the Github API.
1. Create a fine-grained personal access token at https://github.com/settings/tokens?type=beta
2. Store the token on 1Password as a "password" with name "Github Token"
The resulting credential will have the following "path" within 1Password (this path will be provided to the script)
```sh
Private/Github Token/password
```
#### Github session
You need to log in to your Github account with your default browser.
It will be used to open your browser and allow you to create the PR needed to build artifacts (this can be improved later, see TODOs).
## Samples
Following are samples of the script execution to release components to `edge` and `stable`.
> When releasing fleetd you need to checkout the branch (e.g. `main`) you want to release.
> NOTE: When releasing fleetd:
> If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped
> (even if there are no changes in it). This is due to the fact that we want users to see the new version in the tray icon,
> e.g. `"Fleet Desktop v1.21.0"`. Technical debt: We could improve this process to reduce the complexity of releasing
> fleetd when there are no Fleet Desktop changes.
### Releasing to `edge`
#### Releasing fleetd `1.23.0` to `edge`
```sh
AWS_PROFILE=tuf \
TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \
COMPONENT=fleetd \
ACTION=release-to-edge \
VERSION=1.23.0 \
KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \
TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \
SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \
TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \
GITHUB_USERNAME=foobar \
GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \
PUSH_TO_REMOTE=1 \
./tools/tuf/releaser.sh
```
#### Releasing osquery `5.12.1` to `edge`
```sh
AWS_PROFILE=tuf \
TUF_DIRECTORY=/Users/luk/tuf.fleetctl.com \
COMPONENT=osqueryd \
ACTION=release-to-edge \
VERSION=5.12.1 \
KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \
TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \
SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \
TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \
GITHUB_USERNAME=foobar \
GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \
PUSH_TO_REMOTE=1 \
./tools/tuf/releaser.sh
```
### Promoting from `edge` to `stable`
#### Promoting fleetd `1.23.0` from `edge` to `stable`
```sh
AWS_PROFILE=tuf \
TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \
COMPONENT=fleetd \
ACTION=promote-edge-to-stable \
VERSION=1.23.0 \
KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \
TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \
SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \
TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \
GITHUB_USERNAME=foobar \
GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \
PUSH_TO_REMOTE=1 \
./tools/tuf/releaser.sh
```
#### Promoting osqueryd `5.12.1` from `edge` to `stable`
```sh
AWS_PROFILE=tuf \
TUF_DIRECTORY=/Users/foobar/tuf.fleetctl.com \
COMPONENT=osqueryd \
ACTION=promote-edge-to-stable \
VERSION=5.12.1 \
KEYS_SOURCE_DIRECTORY=/Volumes/FLEET-TUF/keys \
TARGETS_PASSPHRASE_1PASSWORD_PATH="Private/TUF TARGETS/password" \
SNAPSHOT_PASSPHRASE_1PASSWORD_PATH="Private/TUF SNAPSHOT/password" \
TIMESTAMP_PASSPHRASE_1PASSWORD_PATH="Private/TUF TIMESTAMP/password" \
GITHUB_USERNAME=foobar \
GITHUB_TOKEN_1PASSWORD_PATH="Private/Github Token/password" \
PUSH_TO_REMOTE=1 \
./tools/tuf/releaser.sh
```
#### Releasing `swiftDialog` to `stable`
> `releaser.sh` doesn't support `swiftDialog` yet.
> macOS only component
The `swiftDialog` executable can be generated from a macOS host by running:
```sh
make swift-dialog-app-tar-gz version=2.2.1 build=4591 out-path=.
```
```sh
fleetctl updates add --target /path/to/macos/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version 2.2.1 -t edge
```
#### Releasing `nudge` to `stable`
> `releaser.sh` doesn't support `nudge` yet.
> macOS only component
The `nudge` executable can be generated from a macOS host by running:
```sh
make nudge-app-tar-gz version=1.1.10.81462 out-path=.
```
```sh
fleetctl updates add --target /path/to/macos/nudge.app.tar.gz --platform macos --name nudge --version 1.1.10.81462 -t edge
```
## Testing and improving the script
- You can specify `GIT_REPOSITORY_DIRECTORY` to set a separate path for the Fleet repository (it uses the current by default).
This is sometimes necessary if the tooling the script uses is not present in the branch we are trying to release from.
```sh
git clone git@github.com:fleetdm/fleet.git <SOME_DIRECTORY>
GIT_REPOSITORY_DIRECTORY=<SOME_DIRECTORY>
```
- If the PR and orbit tag were already generated but you need to run the script again you can set `SKIP_PR_AND_TAG_PUSH=1` to skip that part.
- While developing you can run with `PUSH_TO_REMOTE=0` to prevent pushing invalid metadata/components to the production repository.
## TODOs to improve releaser.sh
- Create the pull requests automatically using `gh` or the Github API.
- Support releasing `nudge` and `swiftDialog`.
## Troubleshooting
### Removing Unused Targets
If you've inadvertently published a target that is no longer in use, follow these steps to remove it.
> Before performing any actions on Fleet's TUF repository you must:
> 1. Make sure your local copy of the repository is up-to-date. See [Syncing Fleet's TUF repository](#syncing-fleets-tuf-repository).
> 2. Create a local backup in case we mess up with the repository:
> ```sh
> mkdir ~/tuf.fleetctl.com/backup
> cp -r ~/tuf.fleetctl.com ~/tuf.fleetctl.com-backup
> ```
1. You'll need the [`go-tuf`](https://github.com/theupdateframework/go-tuf) binary. The removal operations aren't integrated into `fleetctl` at the moment.
2. Use `tuf remove` to remove the target and update `targets.json`. Substitute `desktop/windows/stable/desktop.exe` with the target you intend to delete.
```sh
tuf remove desktop/windows/stable/desktop.exe
```
3. Snapshot, timestamp, and commit the changes.
```sh
tuf snapshot
tuf timestamp
tuf commit
```
4. Run the following command to generate a timestamp that expires in two weeks (otherwise the default expiration when using `go-tuf` commands is 1 day)
```sh
fleetctl updates timestamp
```
5. Confirm that the version of the local `timestamp.json` file is more recent than that of the remote server.
6. Verify the changes that will be synced by running a dry sync. Include the `--delete` flag as you're removing targets.
```sh
aws s3 sync ./repository s3://fleet-tuf-repo --delete --dryrun
```
7. `diff` the local `targets.json` file with its remote version.
8. To upload the changes, perform a sync without the `--dryrun`:
```sh
aws s3 sync ./repository s3://fleet-tuf-repo --delete
```
### Invalid timestamp.json version
The following issue was solved by resigning the timestamp metadata `fleetctl updates timestamp` (executed three times to increase the version to `4175`)
```sh
2022-08-23T13:44:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174"
2022-08-23T13:59:48-03:00 INF update failed error="update metadata: update metadata: tuf: failed to decode timestamp.json: version 4172 is lower than current version 4174"
```

97
tools/tuf/download-artifacts/download-artifacts.go
View file

@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/fleetdm/fleet/v4/orbit/pkg/constant"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
@ -37,6 +38,7 @@ func orbitCommand() *cli.Command {
outputDirectory string
githubUsername string
githubAPIToken string
retry bool
)
return &cli.Command{
Name: "orbit",
@ -70,13 +72,19 @@ func orbitCommand() *cli.Command {
Destination: &githubAPIToken,
Usage: "Github API token (https://github.com/settings/tokens)",
},
&cli.BoolFlag{
Name: "retry",
EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"},
Destination: &retry,
Usage: "Whether to retry if the artifact doesn't exist yet",
},
},
Action: func(c *cli.Context) error {
return downloadComponents("goreleaser-orbit.yaml", gitTag, map[string]string{
"macos": "orbit-macos",
"linux": "orbit-linux",
"windows": "orbit-windows",
}, outputDirectory, githubUsername, githubAPIToken)
}, outputDirectory, githubUsername, githubAPIToken, retry)
},
}
}
@ -87,6 +95,7 @@ func desktopCommand() *cli.Command {
outputDirectory string
githubUsername string
githubAPIToken string
retry bool
)
return &cli.Command{
Name: "desktop",
@ -120,13 +129,19 @@ func desktopCommand() *cli.Command {
Destination: &githubAPIToken,
Usage: "Github API token (https://github.com/settings/tokens)",
},
&cli.BoolFlag{
Name: "retry",
EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"},
Destination: &retry,
Usage: "Whether to retry if the artifact doesn't exist yet",
},
},
Action: func(c *cli.Context) error {
return downloadComponents("generate-desktop-targets.yml", gitBranch, map[string]string{
"macos": "desktop.app.tar.gz",
"linux": "desktop.tar.gz",
"windows": "fleet-desktop.exe",
}, outputDirectory, githubUsername, githubAPIToken)
}, outputDirectory, githubUsername, githubAPIToken, retry)
},
}
}
@ -231,7 +246,7 @@ func extractZipFile(archiveReader *zip.File, destPath string) error {
return nil
}
func downloadComponents(workflowName string, headBranch string, artifactNames map[string]string, outputDirectory string, githubUsername string, githubAPIToken string) error {
func downloadComponents(workflowName string, headBranch string, artifactNames map[string]string, outputDirectory string, githubUsername string, githubAPIToken string, retry bool) error {
if err := os.RemoveAll(outputDirectory); err != nil {
return err
}
@ -241,40 +256,55 @@ func downloadComponents(workflowName string, headBranch string, artifactNames ma
}
}
ctx := context.Background()
gc := github.NewClient(fleethttp.NewClient())
workflow, _, err := gc.Actions.GetWorkflowByFileName(ctx, "fleetdm", "fleet", workflowName)
if err != nil {
return err
}
workflowRuns, _, err := gc.Actions.ListWorkflowRunsByID(ctx, "fleetdm", "fleet", *workflow.ID, nil)
if err != nil {
return err
}
var workflowRun *github.WorkflowRun
for _, wr := range workflowRuns.WorkflowRuns {
if headBranch == *wr.HeadBranch {
workflowRun = wr
gc := github.NewClient(fleethttp.NewClient())
for {
workflow, _, err := gc.Actions.GetWorkflowByFileName(ctx, "fleetdm", "fleet", workflowName)
if err != nil {
return err
}
workflowRuns, _, err := gc.Actions.ListWorkflowRunsByID(ctx, "fleetdm", "fleet", *workflow.ID, nil)
if err != nil {
return err
}
for _, wr := range workflowRuns.WorkflowRuns {
if headBranch == *wr.HeadBranch {
workflowRun = wr
break
}
}
if workflowRun != nil || !retry {
break
}
fmt.Printf("Workflow not available yet, it might be queued, retrying in 60s...\n")
time.Sleep(60 * time.Second)
}
if workflowRun == nil {
return fmt.Errorf("workflow with tag %s not found", headBranch)
}
artifactList, _, err := gc.Actions.ListWorkflowRunArtifacts(ctx, "fleetdm", "fleet", *workflowRun.ID, nil)
if err != nil {
return err
}
urls := make(map[string]string)
for _, artifact := range artifactList.Artifacts {
if *artifact.Name == artifactNames["linux"] {
urls["linux"] = *artifact.ArchiveDownloadURL
} else if *artifact.Name == artifactNames["macos"] {
urls["macos"] = *artifact.ArchiveDownloadURL
} else if *artifact.Name == artifactNames["windows"] {
urls["windows"] = *artifact.ArchiveDownloadURL
} else {
return fmt.Errorf("unknown artifact name: %s", *artifact.Name)
var urls map[string]string
for {
artifactList, _, err := gc.Actions.ListWorkflowRunArtifacts(ctx, "fleetdm", "fleet", *workflowRun.ID, nil)
if err != nil {
return err
}
urls = make(map[string]string)
for _, artifact := range artifactList.Artifacts {
if *artifact.Name == artifactNames["linux"] {
urls["linux"] = *artifact.ArchiveDownloadURL
} else if *artifact.Name == artifactNames["macos"] {
urls["macos"] = *artifact.ArchiveDownloadURL
} else if *artifact.Name == artifactNames["windows"] {
urls["windows"] = *artifact.ArchiveDownloadURL
} else {
return fmt.Errorf("unknown artifact name: %s", *artifact.Name)
}
}
if len(urls) == 3 || !retry {
break
}
fmt.Printf("All artifacts are not available yet, the workflow might still be running, retrying in 60s...\n")
time.Sleep(60 * time.Second)
}
if len(urls) != 3 {
return fmt.Errorf("missing some artifact: %+v", urls)
@ -295,6 +325,7 @@ func osquerydCommand() *cli.Command {
outputDirectory string
githubUsername string
githubAPIToken string
retry bool
)
return &cli.Command{
Name: "osqueryd",
@ -328,13 +359,19 @@ func osquerydCommand() *cli.Command {
Destination: &githubAPIToken,
Usage: "Github API token (https://github.com/settings/tokens)",
},
&cli.BoolFlag{
Name: "retry",
EnvVars: []string{"DOWNLOAD_ARTIFACTS_RETRY"},
Destination: &retry,
Usage: "Whether to retry if the artifact doesn't exist yet",
},
},
Action: func(c *cli.Context) error {
return downloadComponents("generate-osqueryd-targets.yml", gitBranch, map[string]string{
"macos": "osqueryd.app.tar.gz",
"linux": "osqueryd",
"windows": "osqueryd.exe",
}, outputDirectory, githubUsername, githubAPIToken)
}, outputDirectory, githubUsername, githubAPIToken, retry)
},
}
}

51
tools/tuf/promote_edge_to_stable.sh
View file

@ -1,51 +0,0 @@
#!/bin/bash
component=$1
version=$2
if [[ -z $component || -z $version ]]; then
echo "Usage: $0 <component> <version>"
exit 1
fi
if [[ ! -d "./repository" ]]; then
echo "Directory ./repository doesn't exist"
exit 1
fi
version_parts=(${version//./ })
major=${version_parts[0]}
minor=${version_parts[1]}
echo "Promoting $component from edge to stable, version='$version'"
echo "Press any key to continue..."
read -s -n 1
case $1 in
orbit)
fleetctl updates add --target ./repository/targets/orbit/macos/edge/orbit --platform macos --name orbit --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/orbit/linux/edge/orbit --platform linux --name orbit --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/orbit/windows/edge/orbit.exe --platform windows --name orbit --version $version -t $major.$minor -t $major -t stable
;;
desktop)
fleetctl updates add --target ./repository/targets/desktop/macos/edge/desktop.app.tar.gz --platform macos --name desktop --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/desktop/linux/edge/desktop.tar.gz --platform linux --name desktop --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/desktop/windows/edge/fleet-desktop.exe --platform windows --name desktop --version $version -t $major.$minor -t $major -t stable
;;
osqueryd)
fleetctl updates add --target ./repository/targets/osqueryd/macos-app/edge/osqueryd.app.tar.gz --platform macos-app --name osqueryd --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/osqueryd/linux/edge/osqueryd --platform linux --name osqueryd --version $version -t $major.$minor -t $major -t stable
fleetctl updates add --target ./repository/targets/osqueryd/windows/edge/osqueryd.exe --platform windows --name osqueryd --version $version -t $major.$minor -t $major -t stable
;;
nudge)
fleetctl updates add --target ./repository/targets/nudge/macos/edge/nudge.app.tar.gz --platform macos --name nudge --version $version -t stable
;;
swiftDialog)
fleetctl updates add --target ./repository/targets/swiftDialog/macos/edge/swiftDialog.app.tar.gz --platform macos --name swiftDialog --version $version -t stable
;;
*)
echo Unknown component $1
exit 1
;;
esac

305
tools/tuf/releaser.sh Executable file
View file

@ -0,0 +1,305 @@
#!/bin/bash
#
# For usage documentation, see the README.md.
#
set -e
#
# Input environment variables:
#
# AWS_PROFILE
# TUF_DIRECTORY
# COMPONENT
# ACTION
# VERSION
# KEYS_SOURCE_DIRECTORY
# TARGETS_PASSPHRASE_1PASSWORD_PATH
# SNAPSHOT_PASSPHRASE_1PASSWORD_PATH
# TIMESTAMP_PASSPHRASE_1PASSWORD_PATH
# GITHUB_USERNAME
# GITHUB_TOKEN_1PASSWORD_PATH
# SKIP_PR_AND_TAG_PUSH
#
#
# Dev environment variables:
# PUSH_TO_REMOTE
# GIT_REPOSITORY_DIRECTORY
#
clean_up () {
echo "Cleaning up directories..."
# Make sure (best effort) to remove the keys after we are done.
rm -rf "$KEYS_DIRECTORY"
rm -rf "$ARTIFACTS_DOWNLOAD_DIRECTORY"
rm -rf "$GO_TOOLS_DIRECTORY"
ARG=$?
exit $ARG
}
setup () {
echo "Running setup..."
GO_TOOLS_DIRECTORY=$(mktemp -d)
ARTIFACTS_DOWNLOAD_DIRECTORY=$(mktemp -d)
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
REPOSITORY_DIRECTORY=$TUF_DIRECTORY/repository
STAGED_DIRECTORY=$TUF_DIRECTORY/staged
KEYS_DIRECTORY=$TUF_DIRECTORY/keys
if [[ -z $GIT_REPOSITORY_DIRECTORY ]]; then
GIT_REPOSITORY_DIRECTORY=$( realpath "$SCRIPT_DIR/../.." )
fi
mkdir -p "$REPOSITORY_DIRECTORY"
mkdir -p "$STAGED_DIRECTORY"
cp -r "$KEYS_SOURCE_DIRECTORY" "$KEYS_DIRECTORY"
if ! aws sts get-caller-identity &> /dev/null; then
aws sso login
prompt "AWS SSO login was successful, press any key to continue..."
fi
GITHUB_TOKEN=$(op read "op://$GITHUB_TOKEN_1PASSWORD_PATH")
# These need to be exported for use by `fleetctl updates` commands.
FLEET_TARGETS_PASSPHRASE=$(op read "op://$TARGETS_PASSPHRASE_1PASSWORD_PATH")
export FLEET_TARGETS_PASSPHRASE
FLEET_SNAPSHOT_PASSPHRASE=$(op read "op://$SNAPSHOT_PASSPHRASE_1PASSWORD_PATH")
export FLEET_SNAPSHOT_PASSPHRASE
FLEET_TIMESTAMP_PASSPHRASE=$(op read "op://$TIMESTAMP_PASSPHRASE_1PASSWORD_PATH")
export FLEET_TIMESTAMP_PASSPHRASE
go build -o "$GO_TOOLS_DIRECTORY/replace" "$SCRIPT_DIR/../../tools/tuf/replace"
go build -o "$GO_TOOLS_DIRECTORY/download-artifacts" "$SCRIPT_DIR/../../tools/tuf/download-artifacts"
}
pull_from_remote () {
echo "Pulling repository from tuf.fleetctl.com... (--dryrun first)"
aws s3 sync s3://fleet-tuf-repo "$REPOSITORY_DIRECTORY" --exact-timestamps --dryrun
prompt "If the --dryrun looks good, press any key to continue... (no output means nothing to update)"
aws s3 sync s3://fleet-tuf-repo "$REPOSITORY_DIRECTORY" --exact-timestamps
}
promote_component_edge_to_stable () {
component_name=$1
component_version=$2
version_parts=("${component_version//./ }")
major=${version_parts[0]}
minor=${version_parts[1]}
case $component_name in
orbit)
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/macos/edge/orbit" --platform macos --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/linux/edge/orbit" --platform linux --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/orbit/windows/edge/orbit.exe" --platform windows --name orbit --version "$component_version" -t "$major.$minor" -t "$major" -t stable
;;
desktop)
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/macos/edge/desktop.app.tar.gz" --platform macos --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/linux/edge/desktop.tar.gz" --platform linux --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/desktop/windows/edge/fleet-desktop.exe" --platform windows --name desktop --version "$component_version" -t "$major.$minor" -t "$major" -t stable
;;
osqueryd)
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/macos-app/edge/osqueryd.app.tar.gz" --platform macos-app --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/linux/edge/osqueryd" --platform linux --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable
fleetctl updates add --target "$REPOSITORY_DIRECTORY/targets/osqueryd/windows/edge/osqueryd.exe" --platform windows --name osqueryd --version "$component_version" -t "$major.$minor" -t "$major" -t stable
;;
*)
echo "Unknown component $component_name"
exit 1
;;
esac
}
promote_edge_to_stable () {
cd "$REPOSITORY_DIRECTORY"
if [[ $COMPONENT == "fleetd" ]]; then
echo "Promoting fleetd from edge to stable..."
promote_component_edge_to_stable orbit "$VERSION"
promote_component_edge_to_stable desktop "$VERSION"
elif [[ $COMPONENT == "osqueryd" ]]; then
echo "Promoting osqueryd from edge to stable..."
promote_component_edge_to_stable osqueryd "$VERSION"
else
echo "Unsupported component: $COMPONENT"
exit 1
fi
}
release_fleetd_to_edge () {
echo "Releasing fleetd to edge..."
BRANCH_NAME="release-fleetd-v$VERSION"
ORBIT_TAG="orbit-v$VERSION"
if [[ "$SKIP_PR_AND_TAG_PUSH" != "1" ]]; then
prompt "A PR for bumping the fleetd version will be created to trigger a Github Action that will build 'Fleet Desktop'. Press any key to continue..."
pushd "$GIT_REPOSITORY_DIRECTORY"
git checkout -b "$BRANCH_NAME"
make changelog-orbit version="$VERSION"
ORBIT_CHANGELOG=orbit/CHANGELOG.md
"$GO_TOOLS_DIRECTORY/replace" .github/workflows/generate-desktop-targets.yml "FLEET_DESKTOP_VERSION: .+\n" "FLEET_DESKTOP_VERSION: $VERSION\n"
git add .github/workflows/generate-desktop-targets.yml "$ORBIT_CHANGELOG"
git commit -m "Release fleetd $VERSION"
git push origin "$BRANCH_NAME"
open "https://github.com/fleetdm/fleet/pull/new/$BRANCH_NAME"
prompt "Press any key to continue after the PR is created..."
prompt "A 'git tag' will be created to trigger a Github Action to build orbit, press any key to continue..."
git tag "$ORBIT_TAG"
git push origin "$ORBIT_TAG"
popd
fi
DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/desktop"
mkdir -p "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY"
"$GO_TOOLS_DIRECTORY/download-artifacts" desktop \
--git-branch "$BRANCH_NAME" \
--output-directory "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY" \
--github-username "$GITHUB_USERNAME" --github-api-token "$GITHUB_TOKEN" \
--retry
ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/orbit"
mkdir -p "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY"
"$GO_TOOLS_DIRECTORY/download-artifacts" orbit \
--git-tag "$ORBIT_TAG" \
--output-directory "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY" \
--github-username "$GITHUB_USERNAME" --github-api-token "$GITHUB_TOKEN" \
--retry
pushd "$TUF_DIRECTORY"
fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/macos/orbit" --platform macos --name orbit --version "$VERSION" -t edge
fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/linux/orbit" --platform linux --name orbit --version "$VERSION" -t edge
fleetctl updates add --target "$ORBIT_ARTIFACT_DOWNLOAD_DIRECTORY/windows/orbit.exe" --platform windows --name orbit --version "$VERSION" -t edge
fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/macos/desktop.app.tar.gz" --platform macos --name desktop --version "$VERSION" -t edge
fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/linux/desktop.tar.gz" --platform linux --name desktop --version "$VERSION" -t edge
fleetctl updates add --target "$DESKTOP_ARTIFACT_DOWNLOAD_DIRECTORY/windows/fleet-desktop.exe" --platform windows --name desktop --version "$VERSION" -t edge
popd
}
release_osqueryd_to_edge () {
echo "Releasing osqueryd to edge..."
prompt "A branch and PR for bumping the osquery version will be created. Press any key to continue..."
BRANCH_NAME=release-osqueryd-v$VERSION
if [[ "$SKIP_PR_AND_TAG_PUSH" != "1" ]]; then
pushd "$GIT_REPOSITORY_DIRECTORY"
git checkout -b "$BRANCH_NAME"
"$GO_TOOLS_DIRECTORY/replace" .github/workflows/generate-osqueryd-targets.yml "OSQUERY_VERSION: .+\n" "OSQUERY_VERSION: $VERSION\n"
git add .github/workflows/generate-osqueryd-targets.yml
git commit -m "Bump osqueryd version to $VERSION"
git push origin "$BRANCH_NAME"
open "https://github.com/fleetdm/fleet/pull/new/$BRANCH_NAME"
prompt "Press any key to continue after the PR is created..."
popd
fi
OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY="$ARTIFACTS_DOWNLOAD_DIRECTORY/osqueryd"
mkdir -p "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY"
"$GO_TOOLS_DIRECTORY/download-artifacts" osqueryd \
--git-branch "$BRANCH_NAME" \
--output-directory "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY" \
--github-username "$GITHUB_USERNAME" \
--github-api-token "$GITHUB_TOKEN" \
--retry
pushd "$TUF_DIRECTORY"
fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/macos/osqueryd.app.tar.gz" --platform macos-app --name osqueryd --version "$VERSION" -t edge
fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/linux/osqueryd" --platform linux --name osqueryd --version "$VERSION" -t edge
fleetctl updates add --target "$OSQUERYD_ARTIFACT_DOWNLOAD_DIRECTORY/windows/osqueryd.exe" --platform windows --name osqueryd --version "$VERSION" -t edge
popd
}
release_to_edge () {
if [[ $COMPONENT == "fleetd" ]]; then
release_fleetd_to_edge
elif [[ $COMPONENT == "osqueryd" ]]; then
release_osqueryd_to_edge
else
echo "Unsupported component: $COMPONENT"
exit 1
fi
}
push_to_remote () {
echo "Running --dryrun push of repository to tuf.fleetctl.com..."
aws s3 sync "$REPOSITORY_DIRECTORY" s3://fleet-tuf-repo --dryrun
if [[ $PUSH_TO_REMOTE == "1" ]]; then
echo "WARNING: This step will push the release to tuf.fleetctl.com (production)..."
prompt "If the --dryrun looks good, press any key to continue..."
aws s3 sync "$REPOSITORY_DIRECTORY" s3://fleet-tuf-repo
echo "Release has been pushed!"
echo "NOTE: You might see some clients failing to upgrade due to some sha256 mismatches."
echo "These temporary failures are expected because it takes some time for caches to be invalidated (these errors should go away after ~15-30 minutes)."
else
echo "PUSH_TO_REMOTE not set to 1, so not pushing."
fi
}
prompt () {
printf "%s\n" "$1"
read -r -s -n 1
}
setup_to_become_publisher () {
echo "Running setup to become publisher..."
REPOSITORY_DIRECTORY=$TUF_DIRECTORY/repository
STAGED_DIRECTORY=$TUF_DIRECTORY/staged
KEYS_DIRECTORY=$TUF_DIRECTORY/keys
mkdir -p "$REPOSITORY_DIRECTORY"
mkdir -p "$STAGED_DIRECTORY"
mkdir -p "$KEYS_DIRECTORY"
if ! aws sts get-caller-identity &> /dev/null; then
aws sso login
prompt "AWS SSO login was successful, press any key to continue..."
fi
# These need to be exported for use by `tuf` commands.
FLEET_TARGETS_PASSPHRASE=$(op read "op://$TARGETS_PASSPHRASE_1PASSWORD_PATH")
export TUF_TARGETS_PASSPHRASE=$FLEET_TARGETS_PASSPHRASE
FLEET_SNAPSHOT_PASSPHRASE=$(op read "op://$SNAPSHOT_PASSPHRASE_1PASSWORD_PATH")
export TUF_SNAPSHOT_PASSPHRASE=$FLEET_SNAPSHOT_PASSPHRASE
FLEET_TIMESTAMP_PASSPHRASE=$(op read "op://$TIMESTAMP_PASSPHRASE_1PASSWORD_PATH")
export TUF_TIMESTAMP_PASSPHRASE=$FLEET_TIMESTAMP_PASSPHRASE
}
if [[ $ACTION == "generate-signing-keys" ]]; then
setup_to_become_publisher
pull_from_remote
cd "$TUF_DIRECTORY"
tuf gen-key targets && echo
tuf gen-key snapshot && echo
tuf gen-key timestamp && echo
echo "Keys have been generated, now do the following actions:"
echo "- Share '$TUF_DIRECTORY/staged/root.json' with Fleet member with the 'root' role, who will sign with its root key and push it to the remote repository."
echo "- Store the '$TUF_DIRECTORY/keys' folder (that contains the encrypted keys) on a USB flash drive that you will ONLY use for releasing fleetd updates."
exit 0
fi
print_reminder () {
if [[ $ACTION == "release-to-edge" ]]; then
if [[ $COMPONENT == "fleetd" ]]; then
prompt "Make sure to install fleetd with '--orbit-channel=edge --desktop-channel=edge' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..."
elif [[ $COMPONENT == "osqueryd" ]]; then
prompt "Make sure to install fleetd with '--osqueryd-channel=edge' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..."
fi
elif [[ $ACTION == "promote-edge-to-stable" ]]; then
if [[ $COMPONENT == "fleetd" ]]; then
prompt "Make sure to install fleetd with '--orbit-channel=stable --desktop-channel=stable' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..."
elif [[ $COMPONENT == "osqueryd" ]]; then
prompt "Make sure to install fleetd with '--osqueryd-channel=stable' on a Linux, Windows and macOS VM. (To smoke test the release.) Press any key to continue..."
fi
else
echo "Unsupported action: $ACTION"
fi
}
trap clean_up EXIT
print_reminder
setup
pull_from_remote
if [[ $ACTION == "release-to-edge" ]]; then
release_to_edge
elif [[ $ACTION == "promote-edge-to-stable" ]]; then
promote_edge_to_stable
else
echo "Unsupported action: $ACTION"
exit 1
fi
push_to_remote

31
tools/tuf/replace/main.go Normal file
View file

@ -0,0 +1,31 @@
package main
import (
"os"
"regexp"
"strings"
)
// This tool was created to prevent issues between GNU's sed and OSX's sed.
func main() {
inputPath := os.Args[1]
expression := os.Args[2]
replace := os.Args[3]
r := regexp.MustCompile(expression)
stat, err := os.Stat(inputPath)
if err != nil {
panic(err)
}
input, err := os.ReadFile(inputPath)
if err != nil {
panic(err)
}
if strings.HasSuffix(replace, `\n`) {
replace = strings.TrimSuffix(replace, `\n`) + "\n"
}
output := r.ReplaceAllString(string(input), replace)
if err := os.WriteFile(inputPath, []byte(output), stat.Mode()); err != nil {
panic(err)
}
}

2
tools/tuf/test/create_repository.sh
View file

@ -82,7 +82,7 @@ for system in $SYSTEMS; do
ORBIT_BINARY_PATH=$orbit_target \
go run ./orbit/tools/build/build.go
else
GOOS=$goose_value GOARCH=$goarch_value go build -ldflags="-X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=42" -o $orbit_target ./orbit/cmd/orbit
CGO_ENABLED=0 GOOS=$goose_value GOARCH=$goarch_value go build -ldflags="-X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=42" -o $orbit_target ./orbit/cmd/orbit
fi
./build/fleetctl updates add \