Update TUF status generation to use new TUF repository (#26099)

For #25853.
This commit is contained in:
Lucas Manuel Rodriguez 2025-02-07 08:30:07 -03:00 committed by GitHub
parent 329c45488c
commit 9114d0405b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 188 additions and 253 deletions

View file

@ -40,6 +40,10 @@ jobs:
with:
go-version-file: 'go.mod'
- name: Update orbit/old-TUF.md
run: |
make fleetd-old-tuf
- name: Update orbit/TUF.md
run: |
make fleetd-tuf

View file

@ -395,9 +395,17 @@ changelog-chrome:
sh -c "cat new-CHANGELOG.md ee/fleetd-chrome/CHANGELOG.md > tmp-CHANGELOG.md && rm new-CHANGELOG.md && mv tmp-CHANGELOG.md ee/fleetd-chrome/CHANGELOG.md"
sh -c "git rm ee/fleetd-chrome/changes/*"
# Updates the documentation for the currently released versions of fleetd components in Fleet's TUF.
# Updates the documentation for the currently released versions of fleetd components in old Fleet's TUF (tuf.fleetctl.com).
fleetd-old-tuf:
sh -c 'echo "<!-- DO NOT EDIT. This document is automatically generated by running \`make fleetd-old-tuf\`. -->\n# tuf.fleetctl.com\n\nFollowing are the currently deployed versions of fleetd components on the \`stable\` and \`edge\` channel.\n" > orbit/old-TUF.md'
sh -c 'echo "## \`stable\`\n" >> orbit/old-TUF.md'
sh -c 'go run tools/tuf/status/tuf-status.go channel-version -s3-vendor amazon -url https://tuf.fleetctl.com -channel stable -format markdown >> orbit/old-TUF.md'
sh -c 'echo "\n## \`edge\`\n" >> orbit/old-TUF.md'
sh -c 'go run tools/tuf/status/tuf-status.go channel-version -s3-vendor amazon -url https://tuf.fleetctl.com -channel edge -format markdown >> orbit/old-TUF.md'
# Updates the documentation for the currently released versions of fleetd components in Fleet's TUF (updates.fleetdm.com).
fleetd-tuf:
sh -c 'echo "<!-- DO NOT EDIT. This document is automatically generated by running \`make fleetd-tuf\`. -->\n# tuf.fleetctl.com\n\nFollowing are the currently deployed versions of fleetd components on the \`stable\` and \`edge\` channel.\n" > orbit/TUF.md'
sh -c 'echo "<!-- DO NOT EDIT. This document is automatically generated by running \`make fleetd-tuf\`. -->\n# updates.fleetdm.com\n\nFollowing are the currently deployed versions of fleetd components on the \`stable\` and \`edge\` channel.\n" > orbit/TUF.md'
sh -c 'echo "## \`stable\`\n" >> orbit/TUF.md'
sh -c 'go run tools/tuf/status/tuf-status.go channel-version -channel stable -format markdown >> orbit/TUF.md'
sh -c 'echo "\n## \`edge\`\n" >> orbit/TUF.md'

View file

@ -1,5 +1,5 @@
<!-- DO NOT EDIT. This document is automatically generated by running `make fleetd-tuf`. -->
# tuf.fleetctl.com
# updates.fleetdm.com
Following are the currently deployed versions of fleetd components on the `stable` and `edge` channel.

26
orbit/old-TUF.md Normal file
View file

@ -0,0 +1,26 @@
<!-- DO NOT EDIT. This document is automatically generated by running `make fleetd-old-tuf`. -->
# tuf.fleetctl.com
Following are the currently deployed versions of fleetd components on the `stable` and `edge` channel.
## `stable`
| Component\OS | macOS | Linux | Windows | Linux (arm64) |
|--------------|--------------|--------|---------|---------------|
| orbit | 1.38.1 | 1.38.1 | 1.38.1 | 1.38.1 |
| desktop | 1.38.1 | 1.38.1 | 1.38.1 | 1.38.1 |
| osqueryd | 5.15.0 | 5.15.0 | 5.15.0 | 5.15.0 |
| nudge | 1.1.10.81462 | - | - | - |
| swiftDialog | 2.1.0 | - | - | - |
| escrowBuddy | 1.0.0 | - | - | - |
## `edge`
| Component\OS | macOS | Linux | Windows | Linux (arm64) |
|--------------|--------|--------|---------|---------------|
| orbit | 1.38.1 | 1.38.1 | 1.38.1 | 1.38.1 |
| desktop | 1.38.1 | 1.38.1 | 1.38.1 | 1.38.1 |
| osqueryd | 5.15.0 | 5.15.0 | 5.15.0 | 5.15.0 |
| nudge | - | - | - | - |
| swiftDialog | - | - | - | - |
| escrowBuddy | - | - | - | - |

View file

@ -1,34 +1,11 @@
# TUF status
The TUF status tool can be used to process information of a Fleet TUF repository hosted on AWS S3.
The default URL is Fleet's TUF: https://tuf.fleetctl.com.
# Fetch and filter targets
To get information of targets you can use the `key-filter` command.
E.g. to get all targets filtering by the `edge` channel:
```sh
go run tools/tuf/status/tuf-status.go key-filter -filter edge
Results filtered by "edge" and sorted by version, platform and key.
VERSION PLATFORM KEY LAST MODIFIED SIZE ETAG
edge linux targets/desktop/linux/edge/desktop.tar.gz 2024-01-09T20:51:49.000Z 16.3 MB "da05e73b8b351299f1d7063afb538529-2"
edge linux targets/orbit/linux/edge/orbit 2024-01-19T21:35:09.000Z 40.7 MB "a38ff2a2e47b73fe1456563126a8db6d-5"
edge linux targets/osqueryd/linux/edge/osqueryd 2024-01-03T22:19:35.000Z 86.5 MB "8d7e48d9e9883013bfc493d44b96b4e7-11"
edge macos targets/desktop/macos/edge/desktop.app.tar.gz 2024-01-09T20:52:04.000Z 31.9 MB "37e3048387d1f2724fb90417126f8444-4"
edge macos targets/orbit/macos/edge/orbit 2024-01-19T21:36:58.000Z 83.9 MB "7d9bc91b9ce6b5234195650c082d9b9b-11"
edge macos-app targets/osqueryd/macos-app/edge/osqueryd.app.tar.gz 2024-01-03T22:19:47.000Z 24.4 MB "653a3f86b2607798592de3c73a88b1f0-3"
edge windows targets/desktop/windows/edge/fleet-desktop.exe 2024-01-09T20:52:13.000Z 36.8 MB "a482a0e4f0b57e89e6846bd65b8d8ab1-5"
edge windows targets/orbit/windows/edge/orbit.exe 2024-01-19T21:38:37.000Z 40.7 MB "b90014b53abf013fc1bdaec39ab03683-5"
edge windows targets/osqueryd/windows/edge/osqueryd.exe 2024-01-03T22:19:52.000Z 24.8 MB "2887ba627688255d9ec009fbe7b02fbf-3"
```
The TUF status tool can be used to process information of a Fleet TUF repository hosted on AWS S3 or Cloudflare R2.
The default URL is Fleet's TUF: https://updates.fleetctl.com.
# Get the version numbers of a channel
To get the version numbers of components in a given channel you can use the `channel-version` command.
```sh
go run tools/tuf/status/tuf-status.go channel-version -channel stable
{

View file

@ -2,14 +2,11 @@ package main
import (
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
@ -17,33 +14,11 @@ import (
"github.com/urfave/cli/v2"
)
type listBucketResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Text string `xml:",chardata"`
Xmlns string `xml:"xmlns,attr"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
MaxKeys string `xml:"MaxKeys"`
IsTruncated string `xml:"IsTruncated"`
Contents []content `xml:"Contents"`
}
type content struct {
Text string `xml:",chardata"`
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
}
func main() {
app := cli.NewApp()
app.Name = "tuf-status"
app.Usage = "CLI to query a Fleet TUF repository hosted on AWS S3"
app.Usage = "CLI to query a Fleet TUF repository"
app.Commands = []*cli.Command{
keyFilterCommand(),
channelVersionCommand(),
}
if err := app.Run(os.Args); err != nil {
@ -52,144 +27,37 @@ func main() {
}
}
func keyFilterCommand() *cli.Command {
var (
filter string
tufURL string
)
return &cli.Command{
Name: "key-filter",
Usage: "Fetch and filter the entries by the given value",
UsageText: `- To filter all items on the edge channel use -filter="edge"
- To filter all items on version 1.3 including patches that run on Linux use -filter="linux/1.3.*"
- To filter Fleet Desktop items on 1.3.*, stable and edge that run on macOS use -filter="desktop/*.*/macos/(1.3.*|stable|edge)"
`,
Flags: []cli.Flag{
urlFlag(&tufURL),
&cli.StringFlag{
Name: "filter",
EnvVars: []string{"VALUE"},
Value: "stable",
Destination: &filter,
Usage: "Filter string value",
},
},
Action: func(c *cli.Context) error {
res, err := http.Get(tufURL) //nolint
if err != nil {
return err
}
body, err := io.ReadAll(res.Body)
if err != nil {
return err
}
defer res.Body.Close()
var list listBucketResult
if err := xml.Unmarshal(body, &list); err != nil {
return err
}
if err := printTable(list.Contents, filter); err != nil {
return err
}
return nil
},
}
}
func printTable(contents []content, filter string) error {
data := [][]string{}
regFilter, err := regexp.Compile(filter)
if err != nil {
return err
}
for _, content := range contents {
if regFilter.MatchString(content.Key) {
r := strings.Split(content.Key, "/")
platform, version := r[2], r[3]
data = append(data, []string{version, platform, content.Key, content.LastModified, byteCountSI(content.Size), content.ETag})
}
}
// sort by version, platform, key
sort.Slice(data, func(i, j int) bool {
if data[i][0] != data[j][0] {
return data[i][0] < data[j][0]
}
if data[i][1] != data[j][1] {
return data[i][1] < data[j][1]
}
return data[i][2] < data[j][2]
})
fmt.Printf("\nResults filtered by \"%s\" and sorted by version, platform and key.\n\n", filter)
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"version", "platform", "key", "last modified", "size", "etag"})
table.SetAutoWrapText(false)
table.SetAutoFormatHeaders(true)
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
table.SetAlignment(tablewriter.ALIGN_LEFT)
table.SetCenterSeparator("")
table.SetColumnSeparator("")
table.SetRowSeparator("")
table.SetHeaderLine(false)
table.SetBorder(false)
table.SetTablePadding("\t")
table.SetNoWhiteSpace(true)
table.AppendBulk(data)
table.Render()
return nil
}
func byteCountSI(b int64) string {
const unit = 1000
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB",
float64(b)/float64(div), "kMGTPE"[exp])
var componentFileMap = map[string]map[string]string{
"orbit": {
"linux": "orbit",
"linux-arm64": "orbit",
"macos": "orbit",
"windows": "orbit.exe",
},
"desktop": {
"linux": "desktop.tar.gz",
"linux-arm64": "desktop.tar.gz",
"macos": "desktop.app.tar.gz",
"windows": "fleet-desktop.exe",
},
"osqueryd": {
"linux": "osqueryd",
"linux-arm64": "osqueryd",
"macos-app": "osqueryd.app.tar.gz",
"windows": "osqueryd.exe",
},
"nudge": {
"macos": "nudge.app.tar.gz",
},
"swiftDialog": {
"macos": "swiftDialog.app.tar.gz",
},
"escrowBuddy": {
"macos": "escrowBuddy.pkg",
},
}
func channelVersionCommand() *cli.Command {
componentFileMap := map[string]map[string]string{
"orbit": {
"linux": "orbit",
"linux-arm64": "orbit",
"macos": "orbit",
"windows": "orbit.exe",
},
"desktop": {
"linux": "desktop.tar.gz",
"linux-arm64": "desktop.tar.gz",
"macos": "desktop.app.tar.gz",
"windows": "fleet-desktop.exe",
},
"osqueryd": {
"linux": "osqueryd",
"linux-arm64": "osqueryd",
"macos-app": "osqueryd.app.tar.gz",
"windows": "osqueryd.exe",
},
"nudge": {
"macos": "nudge.app.tar.gz",
},
"swiftDialog": {
"macos": "swiftDialog.app.tar.gz",
},
"escrowBuddy": {
"macos": "escrowBuddy.pkg",
},
}
var (
channel string
tufURL string
@ -200,7 +68,13 @@ func channelVersionCommand() *cli.Command {
Name: "channel-version",
Usage: "Fetch display the version of components on a channel (JSON output)",
Flags: []cli.Flag{
urlFlag(&tufURL),
&cli.StringFlag{
Name: "url",
EnvVars: []string{"URL"},
Value: "https://updates.fleetdm.com",
Destination: &tufURL,
Usage: "URL of the TUF repository",
},
&cli.StringFlag{
Name: "channel",
EnvVars: []string{"TUF_STATUS_CHANNEL"},
@ -228,68 +102,21 @@ func channelVersionCommand() *cli.Command {
return errors.New("supported formats are: json, markdown")
}
res, err := http.Get(tufURL) //nolint
var (
foundComponents map[string]map[string]string // component -> OS -> sha512
sha512Map map[string][]string
err error
)
foundComponents, sha512Map, err = getComponents(tufURL, components.Value(), channel)
if err != nil {
return err
}
body, err := io.ReadAll(res.Body)
if err != nil {
return err
}
defer res.Body.Close()
selectedComponents := make(map[string]struct{})
for _, component := range components.Value() {
selectedComponents[component] = struct{}{}
}
var list listBucketResult
if err := xml.Unmarshal(body, &list); err != nil {
return err
}
eTagMap := make(map[string][]string)
foundComponents := make(map[string]map[string]string) // component -> OS -> eTag
for _, content := range list.Contents {
parts := strings.Split(content.Key, "/")
if len(parts) != 5 {
continue
}
componentPart := parts[1]
if _, ok := selectedComponents[componentPart]; !ok {
continue
}
osPart := parts[2]
channelPart := parts[3]
itemPart := parts[4]
if validVersion(channelPart) {
eTagMap[content.ETag] = append(eTagMap[content.ETag], channelPart)
}
if channelPart != channel {
continue
}
m, ok := componentFileMap[componentPart]
if !ok {
continue
}
if v, ok := m[osPart]; !ok || v != itemPart {
continue
}
osMap := foundComponents[componentPart]
if osMap == nil {
osMap = make(map[string]string)
}
osMap[osPart] = content.ETag
foundComponents[componentPart] = osMap
return fmt.Errorf("get components: %w", err)
}
outputMap := make(map[string]map[string]string) // component -> OS -> version
for component, osMap := range foundComponents {
outputMap[component] = make(map[string]string)
for os, eTag := range osMap {
versions := eTagMap[eTag]
for os, sha512 := range osMap {
versions := sha512Map[sha512]
var maxPartsVersion string
for _, version := range versions {
if len(strings.Split(version, ".")) > len(strings.Split(maxPartsVersion, ".")) {
@ -364,12 +191,105 @@ func validVersion(version string) bool {
return true
}
func urlFlag(url *string) *cli.StringFlag {
return &cli.StringFlag{
Name: "url",
EnvVars: []string{"URL"},
Value: "https://tuf.fleetctl.com",
Destination: url,
Usage: "URL of the TUF repository",
func getComponents(tufURL string, components []string, channel string) (foundComponents map[string]map[string]string, sha512Map map[string][]string, err error) {
res, err := http.Get(tufURL + "/targets.json") //nolint
if err != nil {
return nil, nil, fmt.Errorf("failed to get /targets.json: %w", err)
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
return nil, nil, fmt.Errorf("failed to read /targets.json response: %w", err)
}
selectedComponents := make(map[string]struct{})
for _, component := range components {
selectedComponents[component] = struct{}{}
}
var targetsJSON map[string]interface{}
if err := json.Unmarshal(body, &targetsJSON); err != nil {
return nil, nil, fmt.Errorf("failed to parse the source targets.json file: %w", err)
}
signed_ := targetsJSON["signed"]
if signed_ == nil {
return nil, nil, errors.New("missing signed key in targets.json file")
}
signed, ok := signed_.(map[string]interface{})
if !ok {
return nil, nil, fmt.Errorf("invalid signed key in targets.json file: %T, expected map", signed_)
}
targets_ := signed["targets"]
if targets_ == nil {
return nil, nil, errors.New("missing signed.targets key in targets.json file")
}
targets, ok := targets_.(map[string]interface{})
if !ok {
return nil, nil, fmt.Errorf("invalid signed.targets key in targets.json file: %T, expected map", targets_)
}
sha512Map = make(map[string][]string)
foundComponents = make(map[string]map[string]string) // component -> OS -> sha512
for target, metadata_ := range targets {
parts := strings.Split(target, "/")
if len(parts) != 4 {
return nil, nil, fmt.Errorf("target %q: invalid number of parts, expected 4", target)
}
targetName := parts[0]
platformPart := parts[1]
channelPart := parts[2]
executablePart := parts[3]
if _, ok := selectedComponents[targetName]; !ok {
continue
}
metadata, ok := metadata_.(map[string]interface{})
if !ok {
return nil, nil, fmt.Errorf("target: %q: invalid metadata field: %T, expected map", target, metadata_)
}
custom_ := metadata["custom"]
if custom_ == nil {
return nil, nil, fmt.Errorf("target: %q: missing custom field", target)
}
hashes_ := metadata["hashes"]
if hashes_ == nil {
return nil, nil, fmt.Errorf("target: %q: missing hashes field", target)
}
hashes, ok := hashes_.(map[string]interface{})
if !ok {
return nil, nil, fmt.Errorf("target: %q: invalid hashes field: %T", target, hashes_)
}
sha512_ := hashes["sha512"]
if sha512_ == nil {
return nil, nil, fmt.Errorf("target: %q: missing hashes.sha512 field", target)
}
hashSHA512, ok := sha512_.(string)
if !ok {
return nil, nil, fmt.Errorf("target: %q: invalid hashes.sha512 field: %T", target, sha512_)
}
if validVersion(channelPart) {
sha512Map[hashSHA512] = append(sha512Map[hashSHA512], channelPart)
}
if channelPart != channel {
continue
}
m, ok := componentFileMap[targetName]
if !ok {
continue
}
if v, ok := m[platformPart]; !ok || v != executablePart {
continue
}
osMap := foundComponents[targetName]
if osMap == nil {
osMap = make(map[string]string)
}
osMap[platformPart] = hashSHA512
foundComponents[targetName] = osMap
}
return foundComponents, sha512Map, nil
}