diff --git a/.github/ISSUE_TEMPLATE/smoke-tests.md b/.github/ISSUE_TEMPLATE/smoke-tests.md new file mode 100644 index 0000000000..fb9d454adf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/smoke-tests.md @@ -0,0 +1,67 @@ +--- +name: Release QA +about: Checklist of required tests prior to release +title: '' +labels: '' +assignees: '' + +--- + +# Goal: easy-to-follow test steps for sanity checking a release manually + +**Fleet version** (Head to the "My account" page in the Fleet UI or run `fleetctl version`): + +**Web browser** _(e.g. Chrome 88.0.4324)_: + +# Important reference data + +1. [fleetctl preview setup](https://fleetdm.com/get-started) +2. [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) + +# Smoke Tests +Smoke tests are limited to core functionality and serve as a sanity test. If smoke tests are failing, a release cannot proceed. + +## Prerequisites + +1. `fleetctrl preview` is set up and running the desired test version using `--tag` parameters. +2. Unless you are explicitly testing older browser versions, browser is up to date. +3. Certificate & flagfile are in place to create new host. + +## Instructions + + + + + + + + +
Test nameStep instructionsExpected resultpass/fail
$Name{what a tester should do}{what a tester should see when they do that}pass/fail
Update flow + +1. Prereq: previous version is set up with data. +2. Update fleet to testing version +No data should be lost in updatepass/fail
Login flow + +1. navigate to the login page and attempt to login with both valid and invalid credentials to verify some combination of expected results. +2. Login with SSO + + +1. text fields prompt when blank +2. correct error message is "authentication failed" +3. forget password link prompts for email +4. valid credentials result in a successful login. pass/fail
Query flowCreate, edit, run, and delete queries. + +1. permissions regarding creating/editing/deleting queries are up to date with documentation +2. syntax errors result in error messaging +3. queries can be run manually +pass/fail
Host FlowVerify a new host can be added and removed following modal instructions using your own device. + +1. Host is added via command line +2. Host serial number and date added are accurate +3. Host is not visible after it is deleted +4. Warning and informational modals show when expected and make sense +pass/fail
+ +# Notes + +* {any notes} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f38da8fefa..d8297eaafa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,9 @@ # Checklist for submitter -If some of the following don't apply, please write a short explanation why. +If some of the following don't apply, delete the relevant line. -- [ ] Changes file added (if needed) +- [ ] Changes file added (for user-visible changes) - [ ] Documented any API changes -- [ ] Added tests for all functionality -- [ ] Manual QA for all functionality +- [ ] Documented any permissions changes +- [ ] Added/updated tests +- [ ] Manual QA for all new/changed functionality diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ed33f31b81..313c7e37d2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,6 @@ jobs: - uses: actions/checkout@master - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: - check-modified-files-only: 'yes' use-quiet-mode: 'yes' config-file: .github/workflows/markdown-link-check-config.json base-branch: ${{ github.base_ref }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d1072848fc..49be7f82d4 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -17,4 +17,4 @@ jobs: # specified without patch version: we always use the latest patch # version. version: v1.42 - args: --timeout 3m + args: --timeout 10m diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml new file mode 100644 index 0000000000..95ca8232b1 --- /dev/null +++ b/.github/workflows/goreleaser-snapshot-fleet.yaml @@ -0,0 +1,77 @@ +name: Docker publish + +on: + push: + branches: + - main + - patch-* + pull_request: + +jobs: + # This check-secrets job is used to gate execution of the main job. External PRs will not have + # access to the necessary secrets for pushing an image, so we want to skip that job if the secret + # is unavailable. + check-secrets: + environment: Docker Hub + runs-on: ubuntu-latest + outputs: + available: ${{ steps.check-secrets.outputs.available }} + steps: + - name: Check Secrets availability + id: check-secrets + run: | + if [ ! -z "${{ secrets.DOCKERHUB_USERNAME }}" ]; then + echo "::set-output name=available::true" + fi + + # If the secrets are available, build and publish a Docker image. + publish: + needs: + - check-secrets + if: ${{ needs.check-secrets.outputs.available == 'true' }} + runs-on: ubuntu-latest + environment: Docker Hub + steps: + - name: Checkout (PR) + if: ${{ github.event.pull_request.head.sha }} + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout (main) + if: ${{ !github.event.pull_request.head.sha }} + uses: actions/checkout@v2 + + - name: Login to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # v1.10.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.5 + + - name: Install JS Dependencies + run: make deps-js + + - name: Install Go Dependencies + run: make deps-go + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@ac067437f516133269923265894e77920c3dce18 # v2.6.1 + with: + distribution: goreleaser-pro + version: latest + args: release --snapshot --rm-dist -f .goreleaser-snapshot.yml + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + - name: Tag image with branch name + if: ${{ !github.event.pull_request.head.sha }} + run: docker tag fleetdm/fleet:$(git rev-parse --short HEAD) fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) + + # Explicitly push the docker images as GoReleaser will not do so in snapshot mode + - name: Publish Docker images + run: docker push fleetdm/fleet --all-tags diff --git a/.github/workflows/markdown-link-check-config.json b/.github/workflows/markdown-link-check-config.json index 79ac5c0107..d8da76dcb6 100644 --- a/.github/workflows/markdown-link-check-config.json +++ b/.github/workflows/markdown-link-check-config.json @@ -12,6 +12,9 @@ { "pattern": "fleet.corp.example.com" }, + { + "pattern": "fleet.example.com" + }, { "pattern": "/server/datastore/mysql/migrations/" }, diff --git a/.github/workflows/push-osquery-perf-to-ecr.yml b/.github/workflows/push-osquery-perf-to-ecr.yml new file mode 100644 index 0000000000..197ec32864 --- /dev/null +++ b/.github/workflows/push-osquery-perf-to-ecr.yml @@ -0,0 +1,46 @@ +name: Build docker image and publish to ECR + +on: + workflow_dispatch: + inputs: + enroll_secret: + description: 'Enroll Secret' + required: true + url: + description: 'Fleet server URL' + required: true + host_count: + description: 'Amount of hosts to emulate' + required: true + default: 20 + tag: + description: 'docker image tag' + required: true + default: latest + +jobs: + build-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.LOADTEST_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.LOADTEST_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: osquery-perf + IMAGE_TAG: ${{ github.event.inputs.tag }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg ENROLL_SECRET=${{ github.event.inputs.enroll_secret }} --build-arg HOST_COUNT=${{ github.event.inputs.host_count }} --build-arg SERVER_URL=${{ github.event.inputs.url }} -f Dockerfile.osquery-perf . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 6488fc6dc1..d9a1e3d3fe 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -41,7 +41,7 @@ jobs: - name: Run Go Tests run: | - MYSQL_TEST=1 make test-go + NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 make test-go - name: Upload to Codecov uses: codecov/codecov-action@v2 diff --git a/.gitignore b/.gitignore index c1aa8fb7ba..b6049ee72b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,11 @@ backup.sql.gz # committing a package-lock.json. Fleet app uses Yarn with yarn.lock. package-lock.json +# infra +.terraform +.terraform.tfstate* +.terraform.lock* +terraform.tfstate* + # generated installers -orbit-osquery* +orbit-osquery* \ No newline at end of file diff --git a/.goreleaser-snapshot.yml b/.goreleaser-snapshot.yml new file mode 100644 index 0000000000..fa57b30145 --- /dev/null +++ b/.goreleaser-snapshot.yml @@ -0,0 +1,74 @@ +project_name: fleet + +monorepo: + tag_prefix: fleet- + dir: . + +before: + hooks: + - make deps + - make generate + +gomod: + proxy: true + +builds: + - id: fleet + dir: ./cmd/fleet/ + binary: fleet + env: + - CGO_ENABLED=1 + goos: + - linux + goarch: + - amd64 + flags: + - -tags=full,fts5,netgo + - -trimpath + ldflags: + - -extldflags "-static" + - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} + - -X github.com/kolide/kit/version.version={{ .Version }} + - -X github.com/kolide/kit/version.branch={{ .Branch }} + - -X github.com/kolide/kit/version.revision={{ .FullCommit }} + - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} + - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} + + - id: fleetctl + dir: ./cmd/fleetctl/ + binary: fleetctl + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + flags: + - -trimpath + ldflags: + - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} + - -X github.com/kolide/kit/version.version={{ .Version }} + - -X github.com/kolide/kit/version.branch={{ .Branch }} + - -X github.com/kolide/kit/version.revision={{ .FullCommit }} + - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} + - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} + + +dockers: + - goos: linux + goarch: amd64 + ids: + - fleet + - fleetctl + dockerfile: tools/docker/fleet.Dockerfile + image_templates: + - 'fleetdm/fleet:{{ .ShortCommit }}' + + - goos: linux + goarch: amd64 + ids: + - fleetctl + dockerfile: tools/docker/fleetctl.Dockerfile + image_templates: + - 'fleetdm/fleetctl:{{ .ShortCommit }}' + diff --git a/CHANGELOG.md b/CHANGELOG.md index 010e4736e1..31bb2f8748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,33 @@ +## Fleet 4.3.1 (Sept 21, 2021) + +* Add `fleetctl get software` command to list all software and the detected vulnerabilities. The Vulnerable software feature is currently in Beta. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://fleetdm.com/docs/using-fleet/vulnerability-processing). + +* Add `fleetctl vulnerability-data-stream` command to sync the vulnerabilities processing data streams by hand. + +* Add `disable_data_sync` vulnerabilities configuration option to avoid downloading the data streams. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#disable-data-sync). + +* Only show observers the queries they have permissions to run on the **Queries** page. In, Fleet 4.0.0, the Admin, Maintainer, and Observer user roles were introduced. Documentation for the permissions associated with each role can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/using-fleet/permissions). + +* Add `connect_retry_attempts` Redis configuration option to retry failed connections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-connect-retry-attempts). + +* Add `cluster_follow_redirections` Redis configuration option to follow cluster redirections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-cluster-follow-redirections). + +* Add `max_jitter_percent` osquery configuration option to prevent all hosts from returning data at roughly the same time. Note that this improves the Fleet server performance, but it will now take longer for new labels to populate. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#osquery-max-jitter-percent). + +* Improve the performance of database migrations. + +* Reduce database load for label membership recording. + +* Fail early if the process does not have permissions to write to the logging file. + +* Completely skip trying to save a host's users and software inventory if it's disabled to reduce database load. + +* Fix a bug in which team maintainers were unable to run live queries against the hosts assigned to their team(s). + +* Fix a bug in which a blank screen would intermittently appear on the **Hosts** page. + +* Fix a bug detecting disk space for hosts. + ## Fleet 4.3.0 (Sept 13, 2021) * Add Policies feature for detecting device compliance with organizational policies. @@ -138,7 +168,7 @@ * Add ability to create a Team schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). *Available for Fleet Basic customers*. -* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). +* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). * Add ability to see which logging destination is configured for Fleet in the Fleet UI. To see this information, head to the **Schedule** page and then select "Schedule a query." Configured logging destination information is also available in the `GET api/v1/fleet/config` API route. @@ -148,9 +178,9 @@ * Add ability to modify scheduled queries in your Schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). -* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/1-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. +* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/01-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. -* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/2-Configuration.md#feature-flags). +* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#feature-flags). * Improve performance of inserting `pack_stats` in the database. The `pack_stats` information is used to display "Frequency" and "Last run" information for a specific host's scheduled queries. You can find this information on the **Host details** page. @@ -329,9 +359,9 @@ There are currently no known issues in this release. However, we recommend only The primary additions in Fleet 4.0.0 are the new Role-based access control (RBAC) and Teams features. -RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/9-Permissions.md) for a breakdown of the new user roles and their respective capabilities. +RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/09-Permissions.md) for a breakdown of the new user roles and their respective capabilities. -Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/10-Teams.md). +Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md). There are several known issues that will be fixed for the stable release of Fleet 4.0.0. Therefore, we recommend only upgrading to Fleet 4.0.0 RC1 for testing purposes. Please file a GitHub issue for any issues discovered when testing Fleet 4.0.0! @@ -399,7 +429,7 @@ Fleet 4.0.0 is a major release and introduces several breaking changes and datab * Improve Fleet performance by batch updating host seen time instead of updating synchronously. This improvement reduces MySQL CPU usage by ~33% with 4,000 simulated hosts and MySQL running in Docker. -* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/2-Deploying/2-Configuration.md#software-inventory). +* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/02-Deploying/02-Configuration.md#software-inventory). * Add Windows support for `fleetctl` agent autoupdates. The `fleetctl updates` command provides the ability to self-manage an agent update server. Available for Fleet Basic customers. @@ -867,7 +897,7 @@ to 2.0.0. ## Kolide Fleet 2.0.0 (currently preparing for release) -The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md). +The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md). ### New Features diff --git a/Dockerfile.osquery-perf b/Dockerfile.osquery-perf new file mode 100644 index 0000000000..e78d967ef5 --- /dev/null +++ b/Dockerfile.osquery-perf @@ -0,0 +1,16 @@ +FROM golang:1.17.1-alpine + +ARG ENROLL_SECRET +ARG HOST_COUNT +ARG SERVER_URL + +ENV ENROLL_SECRET ${ENROLL_SECRET} +ENV HOST_COUNT ${HOST_COUNT} +ENV SERVER_URL ${SERVER_URL} + +COPY ./cmd/osquery-perf/agent.go ./go.mod ./go.sum ./cmd/osquery-perf/mac10.14.6.tmpl /osquery-perf/ +WORKDIR /osquery-perf/ +RUN go mod download +RUN go build -o osquery-perf + +CMD ./osquery-perf -enroll_secret $ENROLL_SECRET -host_count $HOST_COUNT -server_url $SERVER_URL \ No newline at end of file diff --git a/README.md b/README.md index 6b622672a9..c3f8ce93e4 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The Fleet UI is now available at http://localhost:1337. #### Now what? -Check out the [Ask questions about your devices tutorial](./docs/1-Using-Fleet/tutorials/Ask-questions-about-your-devices.md) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. +Check out the [Ask questions about your devices tutorial](./docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md#how-to-ask-questions-about-your-devices) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. ## Team Fleet is [independently backed](https://linkedin.com/company/fleetdm) and actively maintained with the help of many amazing [contributors](https://github.com/fleetdm/fleet/graphs/contributors). diff --git a/changes/add-policy-updated-at b/changes/add-policy-updated-at new file mode 100644 index 0000000000..5a3d257a7e --- /dev/null +++ b/changes/add-policy-updated-at @@ -0,0 +1 @@ +* Add interval to policies to prevent running them every time the host checks in. diff --git a/changes/ensure-only-one-row-disk-space b/changes/ensure-only-one-row-disk-space deleted file mode 100644 index 1e23bec84e..0000000000 --- a/changes/ensure-only-one-row-disk-space +++ /dev/null @@ -1 +0,0 @@ -* Ensure only one row is returned when checking for disk space in hosts. diff --git a/changes/issue-1894-surface-inherited-queries b/changes/issue-1894-surface-inherited-queries new file mode 100644 index 0000000000..7faae4910e --- /dev/null +++ b/changes/issue-1894-surface-inherited-queries @@ -0,0 +1 @@ +* Fleet premium teams schedules surfaces inherited queries from All teams (global) schedule \ No newline at end of file diff --git a/changes/issue-1963-vulnerabilities-no-sync b/changes/issue-1963-vulnerabilities-no-sync deleted file mode 100644 index c99dd2d4a7..0000000000 --- a/changes/issue-1963-vulnerabilities-no-sync +++ /dev/null @@ -1,2 +0,0 @@ -* Add fleetctl vulnerability-data-stream command to sync the vulnerabilities processing data streams by hand. -* Add vulnerabilities.disable_data_sync config to fleet serve to avoid downloading the data streams. diff --git a/changes/issue-1964-list-software b/changes/issue-1964-list-software deleted file mode 100644 index c6607ea255..0000000000 --- a/changes/issue-1964-list-software +++ /dev/null @@ -1 +0,0 @@ -* Add `fleetctl get software` to list all software and the detected vulnerabilities. diff --git a/changes/issue-1984-label-count-on-edit b/changes/issue-1984-label-count-on-edit new file mode 100644 index 0000000000..44e8a302d8 --- /dev/null +++ b/changes/issue-1984-label-count-on-edit @@ -0,0 +1 @@ +* Do not reset label counts when modifying them. diff --git a/changes/issue-2078-enter-submits-create-team b/changes/issue-2078-enter-submits-create-team new file mode 100644 index 0000000000..48aba2bfe5 --- /dev/null +++ b/changes/issue-2078-enter-submits-create-team @@ -0,0 +1 @@ +- Fix create team modal to submit with Enter key \ No newline at end of file diff --git a/changes/issue-2160-straighten-permissions-around-policies b/changes/issue-2160-straighten-permissions-around-policies new file mode 100644 index 0000000000..370c57ad9b --- /dev/null +++ b/changes/issue-2160-straighten-permissions-around-policies @@ -0,0 +1 @@ +* Expand and improve how roles interact with policies. diff --git a/changes/issue-2201-global-maintainer-create-query b/changes/issue-2201-global-maintainer-create-query new file mode 100644 index 0000000000..4510cd6c41 --- /dev/null +++ b/changes/issue-2201-global-maintainer-create-query @@ -0,0 +1 @@ +- Bug fix: Global maintainer can create and save a new query \ No newline at end of file diff --git a/changes/reduce-sql-queries-per-host b/changes/reduce-sql-queries-per-host new file mode 100644 index 0000000000..833ebadfca --- /dev/null +++ b/changes/reduce-sql-queries-per-host @@ -0,0 +1 @@ +* Reduce the amount of queries to the database when a host checks in. diff --git a/charts/fleet/Chart.yaml b/charts/fleet/Chart.yaml index 8f238ae4dd..66d0884e07 100644 --- a/charts/fleet/Chart.yaml +++ b/charts/fleet/Chart.yaml @@ -4,8 +4,8 @@ name: fleet keywords: - fleet - osquery -version: v4.3.0 +version: v4.3.1 home: https://github.com/fleetdm/fleet sources: - https://github.com/fleetdm/fleet.git -appVersion: v4.3.0 +appVersion: v4.3.1 diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 7dc1636ab2..3e9d3fdee8 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -2,7 +2,7 @@ # All settings related to how Fleet is deployed in Kubernetes hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy -imageTag: v4.3.0 # Version of Fleet to deploy +imageTag: v4.3.1 # Version of Fleet to deploy createIngress: true # Whether or not to automatically create an Ingress ingressAnnotations: {} # Additional annotation to add to the Ingress podAnnotations: {} # Additional annotations to add to the Fleet pod diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 13dbf29b6c..ab90d2bba6 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -200,21 +200,21 @@ the way that the Fleet server works. } } - redisPool, err := redis.NewRedisPool( - config.Redis.Address, - config.Redis.Password, - config.Redis.Database, - config.Redis.UseTLS, - config.Redis.ConnectTimeout, - config.Redis.KeepAlive, - ) + redisPool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: config.Redis.Address, + Password: config.Redis.Password, + Database: config.Redis.Database, + UseTLS: config.Redis.UseTLS, + ConnTimeout: config.Redis.ConnectTimeout, + KeepAlive: config.Redis.KeepAlive, + ConnectRetryAttempts: config.Redis.ConnectRetryAttempts, + ClusterFollowRedirections: config.Redis.ClusterFollowRedirections, + }) if err != nil { initFatal(err, "initialize Redis") } resultStore := pubsub.NewRedisQueryResults(redisPool, config.Redis.DuplicateResults) liveQueryStore := live_query.NewRedisLiveQuery(redisPool) - // TODO: should that only be done when a certain "migrate" flag is set, - // to prevent affecting every startup? if err := liveQueryStore.MigrateKeys(); err != nil { level.Info(logger).Log( "err", err, @@ -509,6 +509,10 @@ func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, if err != nil { level.Error(logger).Log("err", "cleaning scheduled query stats", "details", err) } + err = ds.CleanupOrphanLabelMembership(ctx) + if err != nil { + level.Error(logger).Log("err", "cleaning label_membership", "details", err) + } err = trySendStatistics(ctx, ds, fleet.StatisticsFrequency, "https://fleetdm.com/api/v1/webhooks/receive-usage-analytics") if err != nil { diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 64cd4bfc21..84ec2fa996 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -41,8 +41,7 @@ var userRoleSpecList = []*fleet.User{ } func TestApplyUserRoles(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil @@ -102,11 +101,10 @@ spec: func TestApplyTeamSpecs(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} - server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) teamsByName := map[string]*fleet.Team{ - "team1": &fleet.Team{ + "team1": { ID: 42, Name: "team1", Description: "team1 description", @@ -185,8 +183,7 @@ func writeTmpYml(t *testing.T, contents string) string { } func TestApplyAppConfig(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil diff --git a/cmd/fleetctl/convert_test.go b/cmd/fleetctl/convert_test.go index 1d0724494d..3b838aae1e 100644 --- a/cmd/fleetctl/convert_test.go +++ b/cmd/fleetctl/convert_test.go @@ -11,7 +11,6 @@ import ( ) func TestConvertFileOutput(t *testing.T) { - t.Parallel() // setup the cli and the convert command app := cli.NewApp() app.Commands = []*cli.Command{convertCommand()} @@ -41,7 +40,6 @@ func TestConvertFileOutput(t *testing.T) { } func TestConvertFileStdout(t *testing.T) { - t.Parallel() r, w, _ := os.Pipe() os.Stdout = w diff --git a/cmd/fleetctl/debug_test.go b/cmd/fleetctl/debug_test.go index f8a30a99b8..d71e72196e 100644 --- a/cmd/fleetctl/debug_test.go +++ b/cmd/fleetctl/debug_test.go @@ -44,8 +44,7 @@ oug6edBNpdhp8r2/4t6n3AouK0/zG2naAlmXV0JoFuEvy2bX0BbbbPg+v4WNZIsC func TestDebugConnectionCommand(t *testing.T) { t.Run("without certificate", func(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.VerifyEnrollSecretFunc = func(ctx context.Context, secret string) (*fleet.EnrollSecret, error) { return nil, errors.New("invalid") diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 8212b268fe..2e96b18ca6 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -55,8 +55,7 @@ var userRoleList = []*fleet.User{ } func TestGetUserRoles(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleList, nil @@ -114,8 +113,7 @@ func TestGetTeams(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { license := tt.license - server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { @@ -196,8 +194,7 @@ spec: } func TestGetHosts(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) // this func is called when no host is specified i.e. `fleetctl get hosts --json` ds.ListHostsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) { @@ -343,8 +340,7 @@ func TestGetHosts(t *testing.T) { } func TestGetConfig(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{ @@ -412,8 +408,7 @@ spec: } func TestGetSoftawre(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) foo001 := fleet.Software{ Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "somecpe", diff --git a/cmd/fleetctl/hosts_test.go b/cmd/fleetctl/hosts_test.go index 7e8f080605..936f0cae63 100644 --- a/cmd/fleetctl/hosts_test.go +++ b/cmd/fleetctl/hosts_test.go @@ -10,8 +10,7 @@ import ( ) func TestHostTransferFlagChecks(t *testing.T) { - server, _ := runServerWithMockedDS(t) - defer server.Close() + runServerWithMockedDS(t) runAppCheckErr(t, []string{"hosts", "transfer", "--team", "team1", "--hosts", "host1", "--label", "AAA"}, @@ -24,8 +23,7 @@ func TestHostTransferFlagChecks(t *testing.T) { } func TestHostsTransferByHosts(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -48,8 +46,7 @@ func TestHostsTransferByHosts(t *testing.T) { } func TestHostsTransferByLabel(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -83,8 +80,7 @@ func TestHostsTransferByLabel(t *testing.T) { } func TestHostsTransferByStatus(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -118,8 +114,7 @@ func TestHostsTransferByStatus(t *testing.T) { } func TestHostsTransferByStatusAndSearchQuery(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) diff --git a/cmd/fleetctl/package_test.go b/cmd/fleetctl/package_test.go index 72b3db3561..aa4a9c468b 100644 --- a/cmd/fleetctl/package_test.go +++ b/cmd/fleetctl/package_test.go @@ -1,12 +1,16 @@ package main import ( - "github.com/stretchr/testify/require" "os" "testing" + + "github.com/stretchr/testify/require" ) func TestPackage(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } // --type is required runAppCheckErr(t, []string{"package", "deb"}, "Required flag \"type\" not set") diff --git a/cmd/fleetctl/preview.go b/cmd/fleetctl/preview.go index 4ca9d3c1dd..7afc3d9bf2 100644 --- a/cmd/fleetctl/preview.go +++ b/cmd/fleetctl/preview.go @@ -22,8 +22,9 @@ import ( const ( downloadUrl = "https://github.com/fleetdm/osquery-in-a-box/archive/master.zip" - standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml" + standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml" licenseKeyFlagName = "license-key" + tagFlagName = "tag" ) func previewCommand() *cli.Command { @@ -45,6 +46,11 @@ Use the stop and reset subcommands to manage the server and dependencies once st Name: licenseKeyFlagName, Usage: "License key to enable Fleet Premium (optional)", }, + &cli.StringFlag{ + Name: tagFlagName, + Usage: "Run a specific version of Fleet", + Value: "latest", + }, }, Action: func(c *cli.Context) error { if err := checkDocker(); err != nil { @@ -72,6 +78,10 @@ Use the stop and reset subcommands to manage the server and dependencies once st return errors.Wrap(err, "make logs writable") } + if err := os.Setenv("FLEET_VERSION", c.String(tagFlagName)); err != nil { + return errors.Wrap(err, "failed to set Fleet version") + } + fmt.Println("Pulling Docker dependencies...") out, err := exec.Command("docker-compose", "pull").CombinedOutput() if err != nil { diff --git a/cmd/fleetctl/query_test.go b/cmd/fleetctl/query_test.go index 306ecf1734..145d5d0f6e 100644 --- a/cmd/fleetctl/query_test.go +++ b/cmd/fleetctl/query_test.go @@ -16,8 +16,7 @@ import ( func TestLiveQuery(t *testing.T) { rs := pubsub.NewInmemQueryResults() lq := new(live_query.MockLiveQuery) - server, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) ds.HostIDsByNameFunc = func(ctx context.Context, filter fleet.TeamFilter, hostnames []string) ([]uint, error) { return []uint{1234}, nil diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index 8f4906c363..aed67bf5e4 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -1 +1 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","policy_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index caf6b03972..2642bf768c 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -34,6 +34,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/testdata/expectedListHostsJson.json b/cmd/fleetctl/testdata/expectedListHostsJson.json index 9d90c3914f..1945360d46 100644 --- a/cmd/fleetctl/testdata/expectedListHostsJson.json +++ b/cmd/fleetctl/testdata/expectedListHostsJson.json @@ -1,2 +1,2 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedListHostsYaml.yml b/cmd/fleetctl/testdata/expectedListHostsYaml.yml index 2b75527a0f..b667063c25 100644 --- a/cmd/fleetctl/testdata/expectedListHostsYaml.yml +++ b/cmd/fleetctl/testdata/expectedListHostsYaml.yml @@ -32,6 +32,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false @@ -76,6 +77,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/users_test.go b/cmd/fleetctl/users_test.go index 663c6514bc..482a1d4e0f 100644 --- a/cmd/fleetctl/users_test.go +++ b/cmd/fleetctl/users_test.go @@ -9,8 +9,7 @@ import ( ) func TestUserDelete(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.UserByEmailFunc = func(ctx context.Context, email string) (*fleet.User, error) { return &fleet.User{ diff --git a/cmd/fleetctl/vulnerability_data_stream_test.go b/cmd/fleetctl/vulnerability_data_stream_test.go index d3c7469eec..8fb865447a 100644 --- a/cmd/fleetctl/vulnerability_data_stream_test.go +++ b/cmd/fleetctl/vulnerability_data_stream_test.go @@ -12,6 +12,10 @@ import ( ) func TestVulnerabilityDataStream(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + runAppCheckErr(t, []string{"vulnerability-data-stream"}, "No directory provided") vulnPath := t.TempDir() diff --git a/cmd/osquery-perf/README.md b/cmd/osquery-perf/README.md new file mode 100644 index 0000000000..9a809d34ae --- /dev/null +++ b/cmd/osquery-perf/README.md @@ -0,0 +1,82 @@ +# Osquery Server Performance Tester + +> **TODO: Archive this repo and move its contents inline into https://github.com/fleetdm/fleet** + +This repository provides a tool to generate realistic traffic to an osquery +management server (primarily, [Fleet](https://github.com/fleetdm/fleet)). With +this tool, many thousands of hosts can be simulated from a single host. + +## Requirements + +The only requirement for running this tool is a working installation of +[Go](https://golang.org/doc/install). + +## Usage + +Typically `go run` is used: + +``` +go run agent.go --help +Usage of agent.go: + -config_interval duration + Interval for config requests (default 1m0s) + -enroll_secret string + Enroll secret to authenticate enrollment + -host_count int + Number of hosts to start (default 10) (default 10) + -query_interval duration + Interval for live query requests (default 10s) + -seed int + Seed for random generator (default current time) (default 1586310930917739000) + -server_url string + URL (with protocol and port of osquery server) (default "https://localhost:8080") + -start_period duration + Duration to spread start of hosts over (default 10s) +exit status 2 +``` + +The tool should be invoked with the appropriate enroll secret. A typical +invocation looks like: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf +``` + +When starting many hosts, it is a good idea to extend the intervals, and also +the period over which the hosts are started: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf --host_count 5000 --start_period 5m --query_interval 60s --config_interval 5m +``` + +This will start 5,000 hosts over a period of 5 minutes. Each host will check in +for live queries at a 1 minute interval, and for configuration at a 5 minute +interval. Starting over a 5 minute period ensures that the configuration +requests are spread evenly over the 5 minute interval. + +It can be useful to start the "same" hosts. This can be achieved with the +`--seed` parameter: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf --seed 0 +``` + +By using the same seed, along with other values, we usually get hosts that look +the same to the server. This is not guaranteed, but it is a useful technique. + +### Resource Limits + +On many systems, trying to simulate a large number of hosts will result in hitting system resource limits (such as number of open file descriptors). + +If you see errors such as `dial tcp: lookup localhost: no such host` or `read: connection reset by peer`, try increasing these limits. + +#### macOS + +Run the following command in the shell before running the Fleet server _and_ before running `agent.go` (run it once in each shell): + +``` sh +ulimit -n 64000 +``` + +## Bugs +To report a bug, [click here](https://github.com/fleetdm/fleet). diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go new file mode 100644 index 0000000000..a0339bd8fa --- /dev/null +++ b/cmd/osquery-perf/agent.go @@ -0,0 +1,285 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "strings" + "text/template" + "time" + + "github.com/google/uuid" +) + +type Agent struct { + ServerAddress string + EnrollSecret string + NodeKey string + UUID string + Client http.Client + ConfigInterval time.Duration + QueryInterval time.Duration + Templates *template.Template + strings map[string]string +} + +func NewAgent(serverAddress, enrollSecret string, templates *template.Template, configInterval, queryInterval time.Duration) *Agent { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + transport.DisableCompression = true + return &Agent{ + ServerAddress: serverAddress, + EnrollSecret: enrollSecret, + Templates: templates, + ConfigInterval: configInterval, + QueryInterval: queryInterval, + UUID: uuid.New().String(), + Client: http.Client{Transport: transport}, + strings: make(map[string]string), + } +} + +type enrollResponse struct { + NodeKey string `json:"node_key"` +} + +type distributedReadResponse struct { + Queries map[string]string `json:"queries"` +} + +func (a *Agent) runLoop() { + a.Enroll() + + a.Config() + resp, err := a.DistributedRead() + if err != nil { + log.Println(err) + } else { + if len(resp.Queries) > 0 { + a.DistributedWrite(resp.Queries) + } + } + + configTicker := time.Tick(a.ConfigInterval) + liveQueryTicker := time.Tick(a.QueryInterval) + for { + select { + case <-configTicker: + a.Config() + case <-liveQueryTicker: + resp, err := a.DistributedRead() + if err != nil { + log.Println(err) + } else { + if len(resp.Queries) > 0 { + a.DistributedWrite(resp.Queries) + } + } + } + } +} + +const stringVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_." + +func (a *Agent) randomString(n int) string { + sb := strings.Builder{} + sb.Grow(n) + for i := 0; i < n; i++ { + sb.WriteByte(stringVals[rand.Int63()%int64(len(stringVals))]) + } + return sb.String() +} + +func (a *Agent) CachedString(key string) string { + if val, ok := a.strings[key]; ok { + return val + } + val := a.randomString(12) + a.strings[key] = val + return val +} + +func (a *Agent) Enroll() { + var body bytes.Buffer + if err := a.Templates.ExecuteTemplate(&body, "enroll", a); err != nil { + log.Println("execute template:", err) + return + } + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/enroll", &body) + if err != nil { + log.Println("create request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("status:", resp.Status) + return + } + + var parsedResp enrollResponse + if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { + log.Println("json parse:", err) + return + } + + a.NodeKey = parsedResp.NodeKey +} + +func (a *Agent) Config() { + body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/config", body) + if err != nil { + log.Println("create config request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do config request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("config status:", resp.Status) + return + } + + // No need to read the config body +} + +func (a *Agent) DistributedRead() (*distributedReadResponse, error) { + body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/read", body) + if err != nil { + return nil, fmt.Errorf("create distributed read request: %s", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("do distributed read request: %s", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("distributed read status: %s", resp.Status) + } + + var parsedResp distributedReadResponse + if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { + return nil, fmt.Errorf("json parse distributed read response: %s", err) + } + + return &parsedResp, nil +} + +type distributedWriteRequest struct { + Queries map[string]json.RawMessage `json:"queries"` + Statuses map[string]string `json:"statuses"` + NodeKey string `json:"node_key"` +} + +var defaultQueryResult = json.RawMessage(`[{"foo": "bar"}]`) + +const statusSuccess = "0" + +func (a *Agent) DistributedWrite(queries map[string]string) { + var body bytes.Buffer + + if _, ok := queries["fleet_detail_query_network_interface"]; ok { + // Respond to label/detail queries + a.Templates.ExecuteTemplate(&body, "distributed_write", a) + } else { + // Return a generic response for any other queries + req := distributedWriteRequest{ + Queries: make(map[string]json.RawMessage), + Statuses: make(map[string]string), + NodeKey: a.NodeKey, + } + + for name := range queries { + req.Queries[name] = defaultQueryResult + req.Statuses[name] = statusSuccess + } + json.NewEncoder(&body).Encode(req) + } + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/write", &body) + if err != nil { + log.Println("create distributed write request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do distributed write request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("distributed write status:", resp.Status) + return + } + + // No need to read the distributed write body +} + +func main() { + serverURL := flag.String("server_url", "https://localhost:8080", "URL (with protocol and port of osquery server)") + enrollSecret := flag.String("enroll_secret", "", "Enroll secret to authenticate enrollment") + hostCount := flag.Int("host_count", 10, "Number of hosts to start (default 10)") + randSeed := flag.Int64("seed", time.Now().UnixNano(), "Seed for random generator (default current time)") + startPeriod := flag.Duration("start_period", 10*time.Second, "Duration to spread start of hosts over") + configInterval := flag.Duration("config_interval", 1*time.Minute, "Interval for config requests") + queryInterval := flag.Duration("query_interval", 10*time.Second, "Interval for live query requests") + + flag.Parse() + + rand.Seed(*randSeed) + + tmpl, err := template.ParseGlob("*.tmpl") + if err != nil { + log.Fatal("parse templates: ", err) + } + + // Spread starts over the interval to prevent thunering herd + sleepTime := *startPeriod / time.Duration(*hostCount) + var agents []*Agent + for i := 0; i < *hostCount; i++ { + a := NewAgent(*serverURL, *enrollSecret, tmpl, *configInterval, *queryInterval) + agents = append(agents, a) + go a.runLoop() + time.Sleep(sleepTime) + } + + fmt.Println("Agents running. Kill with C-c.") + <-make(chan struct{}) +} diff --git a/cmd/osquery-perf/mac10.14.6.tmpl b/cmd/osquery-perf/mac10.14.6.tmpl new file mode 100644 index 0000000000..88b6f713a9 --- /dev/null +++ b/cmd/osquery-perf/mac10.14.6.tmpl @@ -0,0 +1,309 @@ +{{ define "enroll" -}} +{ + "enroll_secret": "{{ .EnrollSecret }}", + "host_details": { + "os_version": { + "build": "18G3020", + "major": "10", + "minor": "14", + "name": "Mac OS X", + "patch": "6", + "platform": "darwin", + "platform_like": "darwin", + "version": "10.14.6" + }, + "osquery_info": { + "build_distro": "10.12", + "build_platform": "darwin", + "config_hash": "", + "config_valid": "0", + "extensions": "inactive", + "instance_id": "{{ .UUID }}", + "pid": "12947", + "platform_mask": "21", + "start_time": "1580931224", + "uuid": "{{ .UUID }}", + "version": "4.6.0", + "watcher": "12946" + }, + "platform_info": { + "address": "0xff990000", + "date": "12/16/2019 ", + "extra": "MBP114; 196.0.0.0.0; root@xapp160; Mon Dec 16 15:55:18 PST 2019; 196 (B&I); F000_B00; Official Build, Release; Apple LLVM version 5.0 (clang-500.0.68) (based on LLVM 3.3svn)", + "revision": "196 (B&I)", + "size": "8388608", + "vendor": "Apple Inc. ", + "version": "196.0.0.0.0 ", + "volume_size": "1507328" + }, + "system_info": { + "computer_name": "{{ .CachedString "hostname" }}", + "cpu_brand": "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "cpu_logical_cores": "8", + "cpu_physical_cores": "4", + "cpu_subtype": "Intel x86-64h Haswell", + "cpu_type": "x86_64h", + "hardware_model": "MacBookPro11,4", + "hardware_serial": "D02R835DG8WK", + "hardware_vendor": "Apple Inc.", + "hardware_version": "1.0", + "hostname": "{{ .CachedString "hostname" }}", + "local_hostname": "{{ .CachedString "hostname" }}", + "physical_memory": "17179869184", + "uuid": "{{ .UUID }}" + } + }, + "host_identifier": "{{ .CachedString "hostname" }}", + "platform_type": "21" +} +{{- end }} + +{{ define "distributed_write" -}} +{ + "queries":{ + "fleet_detail_query_network_interface":[ + { + "point_to_point":"", + "address":"fe80::8cb:112d:ff51:1e5d%en0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"en0", + "mac":"f8:2d:88:93:56:5c", + "type":"6", + "mtu":"1500", + "metric":"0", + "ipackets":"278493", + "opackets":"206238", + "ibytes":"275799040", + "obytes":"37720064", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582848084" + +}, + { + "point_to_point":"", + "address":"192.168.1.3", + "mask":"255.255.255.0", + "broadcast":"192.168.1.255", + "interface":"en0", + "mac":"f5:5a:80:92:52:5b", + "type":"6", + "mtu":"1500", + "metric":"0", + "ipackets":"278493", + "opackets":"206238", + "ibytes":"275799040", + "obytes":"37720064", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582848084" + +}, + { + "point_to_point":"127.0.0.1", + "address":"127.0.0.1", + "mask":"255.0.0.0", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"::1", + "address":"::1", + "mask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"", + "address":"fe80::1%lo0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"", + "address":"fe80::3a:84ff:fe6b:bf75%awdl0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"awdl0", + "mac":"03:3b:94:5b:be:75", + "type":"6", + "mtu":"1484", + "metric":"0", + "ipackets":"0", + "opackets":"16", + "ibytes":"0", + "obytes":"3072", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582842892" + +}, + { + "point_to_point":"", + "address":"fe80::6eaf:9721:3476:b691%utun0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"utun0", + "mac":"00:00:00:00:00:00", + "type":"1", + "mtu":"2000", + "metric":"0", + "ipackets":"0", + "opackets":"2", + "ibytes":"0", + "obytes":"0", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840897" + +} + +], + "fleet_detail_query_os_version":[ + { + "name":"Mac OS X", + "version":"10.14.6", + "major":"10", + "minor":"14", + "patch":"6", + "build":"18G3020", + "platform":"darwin", + "platform_like":"darwin", + "codename":"" + +} + +], + "fleet_detail_query_osquery_flags":[ + { + "name":"config_refresh", + "value":"{{ printf "%.0f" .ConfigInterval.Seconds }}" + +}, + { + "name":"distributed_interval", + "value":"{{ printf "%.0f" .QueryInterval.Seconds }}" + +}, + { + "name":"logger_tls_period", + "value":"99999" + +} + +], + "fleet_detail_query_osquery_info":[ + { + "pid":"11287", + "uuid":"{{ .UUID }}", + "instance_id":"{{ .UUID }}", + "version":"4.1.2", + "config_hash":"b01efbf375ac6767f259ae98751154fef727ce35", + "config_valid":"1", + "extensions":"inactive", + "build_platform":"darwin", + "build_distro":"10.12", + "start_time":"1582857555", + "watcher":"11286", + "platform_mask":"21" + +} + +], + "fleet_detail_query_system_info":[ + { + "hostname":"{{ .CachedString "hostname" }}", + "uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB", + "cpu_type":"x86_64h", + "cpu_subtype":"Intel x86-64h Haswell", + "cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", + "cpu_physical_cores":"4", + "cpu_logical_cores":"8", + "cpu_microcode":"", + "physical_memory":"17179869184", + "hardware_vendor":"Apple Inc.", + "hardware_model":"MacBookPro11,4", + "hardware_version":"1.0", + "hardware_serial":"C02R262BM8LN", + "computer_name":"{{ .CachedString "hostname" }}", + "local_hostname":"{{ .CachedString "hostname" }}" + +} + +], + "fleet_detail_query_uptime":[ + { + "days":"0", + "hours":"4", + "minutes":"38", + "seconds":"11", + "total_seconds":"16691" + +} + +] + +}, + "statuses":{ + "fleet_detail_query_network_interface":0, + "fleet_detail_query_os_version":0, + "fleet_detail_query_osquery_flags":0, + "fleet_detail_query_osquery_info":0, + "fleet_detail_query_system_info":0, + "fleet_detail_query_uptime":0 +}, + "node_key":"{{ .NodeKey }}" +} +{{- end }} diff --git a/cypress-free.json b/cypress-free.json index d2b6814b8c..91f97117f0 100644 --- a/cypress-free.json +++ b/cypress-free.json @@ -1,5 +1,9 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,free}/**/*.spec.ts" + "testFiles": "{all,free}/**/*.spec.ts", + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress-premium.json b/cypress-premium.json index 86c1807f11..394960d126 100644 --- a/cypress-premium.json +++ b/cypress-premium.json @@ -1,5 +1,9 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,premium}/**/*.spec.ts" + "testFiles": "{all,premium}/**/*.spec.ts", + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress.json b/cypress.json index 73960449f2..b15968f2da 100644 --- a/cypress.json +++ b/cypress.json @@ -2,5 +2,9 @@ "baseUrl": "https://localhost:8642", "fixturesFolder": false, "testFiles": "**/*.ts", - "defaultCommandTimeout": 8000 + "defaultCommandTimeout": 8000, + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress/README.md b/cypress/README.md index c67e59da62..ab80dd2069 100644 --- a/cypress/README.md +++ b/cypress/README.md @@ -1,6 +1,6 @@ # Cypress Testing -Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) includes git instructions for test preparation and running tests. +Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) includes git instructions for test preparation and running tests. ## Fleet Cypress directories @@ -37,6 +37,6 @@ As much as possible, assert that the code is only selecting 1 item or that the f ## Resources -- [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) +- [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) - [Cypress documentation](https://docs.cypress.io/api/table-of-contents) - [React testing-library query documentation](https://testing-library.com/docs/queries/about) diff --git a/cypress/integration/all/app/hosts.spec.ts b/cypress/integration/all/app/hosts.spec.ts index 39ae559ab3..6131989072 100644 --- a/cypress/integration/all/app/hosts.spec.ts +++ b/cypress/integration/all/app/hosts.spec.ts @@ -10,6 +10,7 @@ describe( cy.setup(); cy.login(); cy.addDockerHost(); + cy.clearDownloads(); }); afterEach(() => { @@ -45,9 +46,6 @@ describe( cy.get("input[disabled]").should("have.value", contents); }); - // ensure load - cy.wait(5000); // eslint-disable-line cypress/no-unnecessary-waiting - // Wait until the host becomes available (usually immediate in local // testing, but may vary by environment). cy.waitUntil( diff --git a/cypress/integration/all/app/manageUsers.spec.ts b/cypress/integration/all/app/manageUsers.spec.ts index 674ee386fa..5c6f25dc71 100644 --- a/cypress/integration/all/app/manageUsers.spec.ts +++ b/cypress/integration/all/app/manageUsers.spec.ts @@ -6,15 +6,7 @@ describe("Manage Users", () => { }); it("Searching for a user", () => { - cy.intercept({ - method: "GET", - url: "/api/v1/fleet/users", - }).as("getUsers"); - cy.visit("/settings/users"); - cy.url().should("match", /\/settings\/users$/i); - - cy.wait("@getUsers"); cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("exist"); @@ -23,8 +15,6 @@ describe("Manage Users", () => { cy.findByPlaceholderText("Search").type("admin"); - cy.wait("@getUsers"); - cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("not.exist"); cy.findByText("observer@example.com").should("not.exist"); diff --git a/cypress/integration/all/app/queryflow.spec.ts b/cypress/integration/all/app/queryflow.spec.ts index c91dc70aea..bcc8804123 100644 --- a/cypress/integration/all/app/queryflow.spec.ts +++ b/cypress/integration/all/app/queryflow.spec.ts @@ -9,50 +9,43 @@ describe( cy.login(); }); - // TODO - Fix tests according to improved query experience - MP it("Create, check, edit, and delete a query successfully and create, edit, and delete a global scheduled query successfully", () => { cy.visit("/queries/manage"); - cy.findByRole("button", { name: /create new query/i }).should("exist"); - // cy.findByRole("button", { name: /create new query/i }).click(); + cy.findByRole("button", { name: /create new query/i }).click(); - // cy.findByLabelText(/query name/i) - // .click() - // .type("Query all window crashes"); + // Using class selector because third party element doesn't work with Cypress Testing Selector Library + cy.get(".ace_scroller") + .click({ force: true }) + .type("{selectall}SELECT * FROM windows_crashes;"); - // // Using class selector because third party element doesn't work with Cypress Testing Selector Library - // cy.get(".ace_scroller") - // .click({ force: true }) - // .type("{selectall}{backspace}SELECT * FROM windows_crashes;"); + cy.findByRole("button", { name: /save/i }).click(); - // cy.findByLabelText(/description/i) - // .click() - // .type("See all window crashes"); + cy.findByLabelText(/name/i).click().type("Query all window crashes"); - // cy.findByRole("button", { name: /save/i }).click(); + cy.findByLabelText(/description/i) + .click() + .type("See all window crashes"); - // cy.findByRole("button", { name: /save as new/i }).click(); + cy.findByRole("button", { name: /save query/i }).click(); - // Just refreshes to create new query, needs success alert to user that they created a query - // cy.visit("/queries/manage"); + cy.findByText(/query created/i).should("exist"); + cy.findByText(/back to queries/i).should("exist"); + cy.visit("/queries/manage"); - // cy.findByText(/query all/i).click(); + cy.findByText(/query all/i).click(); - // cy.findByText(/edit & run query/i).should("exist"); + cy.findByText(/run query/i).should("exist"); - // cy.get(".ace_scroller") - // .click({ force: true }) - // .type( - // "{selectall}{backspace}SELECT datetime, username FROM windows_crashes;" - // ); + cy.get(".ace_scroller") + .click({ force: true }) + .type("{selectall}SELECT datetime, username FROM windows_crashes;"); - // cy.findByRole("button", { name: /save/i }).click(); + cy.findByRole("button", { name: /^Save$/ }).click(); - // cy.findByRole("button", { name: /save changes/i }).click(); + cy.findByText(/query updated/i).should("be.visible"); - // cy.findByText(/query updated/i).should("be.visible"); - - // // Test Schedules + // // Start e2e test for schedules // cy.visit("/schedule/manage"); // cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting @@ -85,64 +78,67 @@ describe( // .contains("button", /schedule/i) // .click(); - // cy.visit("/schedule/manage"); + // cy.visit("/schedule/manage"); - // cy.wait(3000); // eslint-disable-line cypress/no-unnecessary-waiting - // cy.findByText(/query all window crashes/i).should("exist"); + // cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting + // cy.findByText(/query all window crashes/i).should("exist"); - // cy.findByText(/actions/i).click(); - // cy.findByText(/edit/i).click(); + // cy.findByText(/actions/i).click(); + // cy.findByText(/edit/i).click(); - // cy.get( - // ".schedule-editor-modal__form-field--frequency > .dropdown__select" - // ).click(); + // cy.get( + // ".schedule-editor-modal__form-field--frequency > .dropdown__select" + // ).click(); - // cy.findByText(/every 6 hours/i).click(); + // cy.findByText(/every 6 hours/i).click(); - // cy.findByText(/show advanced options/i).click(); + // cy.findByText(/show advanced options/i).click(); - // cy.findByText(/ignore removals/i).click(); - // cy.findByText(/snapshot/i).click(); + // cy.findByText(/ignore removals/i).click(); + // cy.findByText(/snapshot/i).click(); - // cy.get(".schedule-editor-modal__form-field--shard > .input-field") - // .click() - // .type("{selectall}{backspace}10"); + // cy.get(".schedule-editor-modal__form-field--shard > .input-field") + // .click() + // .type("{selectall}{backspace}10"); - // cy.get(".schedule-editor-modal__btn-wrap") - // .contains("button", /schedule/i) - // .click(); + // cy.get(".schedule-editor-modal__btn-wrap") + // .contains("button", /schedule/i) + // .click(); - // cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting - // cy.findByText(/actions/i).click(); - // cy.findByText(/remove/i).click(); + // cy.visit("/schedule/manage"); - // cy.get(".remove-scheduled-query-modal__btn-wrap") - // .contains("button", /remove/i) - // .click(); + // cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting + // cy.findByText(/actions/i).click(); + // cy.findByText(/remove/i).click(); - // cy.findByText(/query all window crashes/i).should("not.exist"); - // // End Test Schedules + // cy.get(".remove-scheduled-query-modal__btn-wrap") + // .contains("button", /remove/i) + // .click(); - // cy.visit("/queries/manage"); + // cy.findByText(/query all window crashes/i).should("not.exist"); - // cy.findByText(/query all window crashes/i) - // .parent() - // .parent() - // .within(() => { - // cy.get(".fleet-checkbox__input").check({ force: true }); - // }); + // // End e2e test for schedules - // cy.findByRole("button", { name: /delete/i }).click(); + cy.visit("/queries/manage"); - // // Can't figure out how attach findByRole onto modal button - // // Can't use findByText because delete button under modal - // cy.get(".remove-query-modal") - // .contains("button", /delete/i) - // .click(); + cy.findByText(/query all window crashes/i) + .parent() + .parent() + .within(() => { + cy.get(".fleet-checkbox__input").check({ force: true }); + }); - // cy.findByText(/successfully removed query/i).should("be.visible"); + cy.findByRole("button", { name: /delete/i }).click(); - // cy.findByText(/query all/i).should("not.exist"); + // Can't figure out how attach findByRole onto modal button + // Can't use findByText because delete button under modal + cy.get(".remove-query-modal") + .contains("button", /delete/i) + .click(); + + cy.findByText(/successfully removed query/i).should("be.visible"); + + cy.findByText(/query all/i).should("not.exist"); }); } ); diff --git a/cypress/integration/all/app/resetsessions.spec.ts b/cypress/integration/all/app/resetsessions.spec.ts index c9bba80710..0a9e1e9100 100644 --- a/cypress/integration/all/app/resetsessions.spec.ts +++ b/cypress/integration/all/app/resetsessions.spec.ts @@ -20,14 +20,11 @@ describe("Reset user sessions flow", () => { // first select the table cell with the user's email address then go back up to the containing row // so we can select reset sessions from actions dropdown - cy.get("tbody>tr>td") - .contains(/admin@example.com/i) - .parent() - .parent() - .within(() => { - cy.findByText(/actions/i).click(); - cy.findByText(/reset sessions/i).click(); - }); + cy.contains("div.Select-placeholder", /actions/i); + // eslint-disable-next-line cypress/no-unnecessary-waiting + cy.wait(500); + cy.contains("div.Select-placeholder", /actions/i).click(); + cy.contains(/reset sessions/i).click(); cy.get(".modal__modal_container").within(() => { cy.findByText(/reset sessions/i).should("exist"); diff --git a/cypress/integration/free/observer.spec.ts b/cypress/integration/free/observer.spec.ts index ab7e05af38..253ceb2cf1 100644 --- a/cypress/integration/free/observer.spec.ts +++ b/cypress/integration/free/observer.spec.ts @@ -65,20 +65,11 @@ describe("Free tier - Observer user", () => { cy.findByText(/show sql/i).click(); cy.findByRole("button", { name: /run query/i }).should("exist"); - cy.visit("/queries/manage"); - - cy.findByText(/get authorized/i).click(); - cy.findByText(/packs/i).should("not.exist"); - cy.findByLabelText(/query name/i).should("not.exist"); - cy.findByLabelText(/sql/i).should("not.exist"); - cy.findByLabelText(/description/i).should("not.exist"); - cy.findByLabelText(/observer can run/i).should("not.exist"); - cy.findByText(/show sql/i).click(); - cy.findByRole("button", { name: /run query/i }).should("not.exist"); - // On the Profile page, they should… // See Observer in Role section, and no Team section cy.visit("/profile"); + + cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByText(/teams/i).should("not.exist"); cy.findByText("Role") .next() diff --git a/cypress/integration/premium/teamflow.spec.ts b/cypress/integration/premium/teamflow.spec.ts index 2d26bcfdcb..ba60bb2589 100644 --- a/cypress/integration/premium/teamflow.spec.ts +++ b/cypress/integration/premium/teamflow.spec.ts @@ -5,12 +5,13 @@ describe("Teams flow", () => { cy.viewport(1200, 660); }); + /* TODO fix and reenable + This test is causing major flake issues due to the dropdown menu */ + it("Create, edit, and delete a team successfully", () => { cy.visit("/settings/teams"); - cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting - - cy.findByRole("button", { name: /create team/i }).click(); + cy.findByRole("button", { name: /create team/i }).click({ force: true }); cy.findByLabelText(/team name/i) .click() @@ -28,24 +29,92 @@ describe("Teams flow", () => { cy.findByText(/agent options/i).click(); - cy.get(".ace_content") - .click() - .type("{selectall}{backspace}apiVersion: v1{enter}kind: options"); + cy.contains(".ace_content", "config:"); + cy.get(".ace_text-input") + .first() + .focus() + .type("{selectall}{backspace}config:\n options:"); cy.findByRole("button", { name: /save options/i }).click(); + cy.contains("span", /successfully saved/i); + cy.visit("/settings/teams/1/options"); - cy.contains(/apiVersion: v1/i).should("be.visible"); - cy.contains(/kind: options/i).should("be.visible"); + cy.contains(/config:/i).should("be.visible"); + cy.contains(/options:/i).should("be.visible"); + // Check team in schedules + cy.visit("/queries/manage"); + + cy.findByRole("button", { name: /create new query/i }).click(); + + // Using class selector because third party element doesn't work with Cypress Testing Selector Library + cy.get(".ace_scroller") + .click({ force: true }) + .type("{selectall}SELECT * FROM windows_crashes;"); + + cy.findByRole("button", { name: /save/i }).click(); + + cy.findByLabelText(/name/i).click().type("Query all window crashes"); + + cy.findByLabelText(/description/i) + .click() + .type("See all window crashes"); + + cy.findByRole("button", { name: /save query/i }).click(); + + cy.visit("/schedule/manage"); + + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + + cy.findByRole("button", { name: /schedule a query/i }).click(); + + cy.findByText(/select query/i).click(); + + cy.findByText(/query all window crashes/i).click(); + + cy.get( + ".schedule-editor-modal__form-field--frequency > .dropdown__select" + ).click(); + + cy.findByText(/every week/i).click(); + + cy.findByText(/show advanced options/i).click(); + + cy.get( + ".schedule-editor-modal__form-field--logging > .dropdown__select" + ).click(); + + cy.findByText(/ignore removals/i).click(); + + cy.get(".schedule-editor-modal__form-field--shard > .input-field") + .click() + .type("50"); + + cy.get(".schedule-editor-modal__btn-wrap") + .contains("button", /schedule/i) + .click(); + + cy.visit("/schedule/manage"); + + cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.findByText(/all teams/i).click(); + cy.findByText(/valor/i).click(); + + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.findByText(/query all window crashes/i).should("not.exist"); + cy.findByText(/inherited query/i).click(); + cy.findByText(/query all window crashes/i).should("exist"); + + // Edit Team cy.visit("/settings/teams"); - cy.contains("Valor").get(".Select-arrow-zone").click(); - - // need force:true for dropdown - cy.findByText(/edit/i).click({ force: true }); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.findByText(/actions/i).click({ force: true }); + cy.findByText(/edit/i).click({ force: true }); // need force:true for dropdown + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByLabelText(/team name/i) .click() .type("{selectall}{backspace}Mystic"); @@ -61,6 +130,7 @@ describe("Teams flow", () => { cy.findByText(/delete/i).click({ force: true }); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByRole("button", { name: /delete/i }).click(); cy.findByText(/successfully deleted/i).should("be.visible"); diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index 9af750deac..b73c3ccae9 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -285,3 +285,7 @@ Cypress.Commands.add("stopDockerHost", () => { }, }); }); + +Cypress.Commands.add("clearDownloads", () => { + cy.exec(`rm -rf ${Cypress.config("downloadsFolder")}`); +}); diff --git a/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md b/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md similarity index 89% rename from docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md rename to docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md index aa7a894d83..c10538fcc9 100644 --- a/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md +++ b/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md @@ -28,19 +28,14 @@ So, let's start by asking the following questions about Fleet's 7 simulated Linu 2. Do these devices have a high severity vulnerable version of OpenSSL installed? -These questions can easily be answered, by running the following query: "Detect Linux hosts with high severity vulnerable versions of OpenSSL." +These questions can easily be answered, by running this simple query: "Get OpenSSL versions." -On the **Queries** page, enter the query name, "Detect Linux hosts with high severity vulnerable versions of OpenSSL," in the search box, select the query from the results table, and navigate to the **Edit or run query** page. +On the **Queries** page, enter the query name, "Get OpenSSL versions," in the search box, and select it to enter the **query console**. Then from the **query console**, hit "Run query", and from the "Select targets" page, select "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. -Fleet query search +Fleet select targets -On the **Edit or run query** page, open the "Select targets" dropdown, and press the purple "+" icon to the right of "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. - - -Fleet select targets - The query may take several seconds to complete, because Fleet has to wait for the osquery agents to respond with results. > Fleet's query response time is inherently variable because of osquery's heartbeat response time. This helps prevent performance issues on hosts. @@ -48,7 +43,7 @@ The query may take several seconds to complete, because Fleet has to wait for th When the query has finished, you should see 4 columns and several rows in the "Results" table: -Fleet query results +Fleet query results - The "hostname" column answers: which device responded for a given row of results? diff --git a/docs/1-Using-Fleet/1-Fleet-UI.md b/docs/01-Using-Fleet/01-Fleet-UI.md similarity index 97% rename from docs/1-Using-Fleet/1-Fleet-UI.md rename to docs/01-Using-Fleet/01-Fleet-UI.md index e6e0c68954..b90b2b2e89 100644 --- a/docs/1-Using-Fleet/1-Fleet-UI.md +++ b/docs/01-Using-Fleet/01-Fleet-UI.md @@ -27,7 +27,7 @@ To add queries to a pack, use the right-hand sidebar. You can take an existing s ![Schedule Query Sidebar](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/schedule-query-sidebar.png) -Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../1-Using-Fleet/5-Osquery-logs.md). +Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../01-Using-Fleet/05-Osquery-logs.md). ## Configuring agent options diff --git a/docs/1-Using-Fleet/2-fleetctl-CLI.md b/docs/01-Using-Fleet/02-fleetctl-CLI.md similarity index 96% rename from docs/1-Using-Fleet/2-fleetctl-CLI.md rename to docs/01-Using-Fleet/02-fleetctl-CLI.md index 396280f346..9e7ef885ae 100644 --- a/docs/1-Using-Fleet/2-fleetctl-CLI.md +++ b/docs/01-Using-Fleet/02-fleetctl-CLI.md @@ -24,7 +24,7 @@ This guide illustrates: ### Running Fleet -For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../3-Contributing/1-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): +For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../03-Contributing/01-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): ``` docker-compose up -d @@ -186,7 +186,7 @@ spec: Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allows the Fleet server to request files (and sets of files) from osquery agents, returning the full contents to Fleet. -File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../2-Deploying/2-Configuration.md#s3-file-carving-backend). +File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../02-Deploying/02-Configuration.md#s3-file-carving-backend). ### Configuration diff --git a/docs/1-Using-Fleet/3-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md similarity index 95% rename from docs/1-Using-Fleet/3-REST-API.md rename to docs/01-Using-Fleet/03-REST-API.md index b055201859..e2adf152c7 100644 --- a/docs/1-Using-Fleet/3-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -10,6 +10,7 @@ - [Schedule](#schedule) - [Packs](#packs) - [Policies](#policies) +- [Team Policies](#team-policies) - [Activities](#activities) - [Targets](#targets) - [Fleet configuration](#fleet-configuration) @@ -30,7 +31,7 @@ Fleet is powered by a Go API server which serves three types of endpoints: ### fleetctl -Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./2-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. +Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./02-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. ### Current API @@ -100,7 +101,7 @@ Authenticates the user with the specified credentials. Use the token returned fr ##### Request body -``` +```json { "email": "janedoe@example.com", "password": "VArCjNW7CfsxGp67" @@ -166,7 +167,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi ##### Request body -``` +```json { "email": "janedoe@example.com" } @@ -180,7 +181,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi `Status: 500` -``` +```json { "message": "Unknown Error", "errors": [ @@ -213,7 +214,7 @@ Changes the password for the authenticated user. ##### Request body -``` +```json { "old_password": "VArCjNW7CfsxGp67", "new_password": "zGq7mCLA6z4PzArC" @@ -228,7 +229,7 @@ Changes the password for the authenticated user. `Status: 422 Unprocessable entity` -``` +```json { "message": "Validation Failed", "errors": [ @@ -260,10 +261,10 @@ Resets a user's password. Which user is determined by the password reset token u ##### Request body -``` +```json { - "new_password": "abc123" - "new_password_confirmation": "abc123" + "new_password": "abc123", + "new_password_confirmation": "abc123", "password_reset_token": "UU5EK0JhcVpsRkY3NTdsaVliMEZDbHJ6TWdhK3oxQ1Q=" } ``` @@ -272,9 +273,6 @@ Resets a user's password. Which user is determined by the password reset token u `Status: 200` -``` -{} -``` --- @@ -292,7 +290,7 @@ Retrieves the user data for the authenticated user. `Status: 200` -``` +```json { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -324,7 +322,7 @@ Resets the password of the authenticated user. Requires that `force_password_res ##### Request body -``` +```json { "new_password": "sdPz8CV5YhzH47nK" } @@ -334,7 +332,7 @@ Resets the password of the authenticated user. Requires that `force_password_res `Status: 200` -``` +```json { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -368,7 +366,7 @@ Gets the current SSO configuration. `Status: 200` -``` +```json { "settings": { "idp_name": "IDP Vendor 1", @@ -396,7 +394,7 @@ Gets the current SSO configuration. ##### Request body -``` +```json { "relay_url": "/hosts/manage" } @@ -410,7 +408,7 @@ Gets the current SSO configuration. `Status: 500` -``` +```json { "message": "Unknown Error", "errors": [ @@ -440,7 +438,7 @@ This is the callback endpoint that the identity provider will use to send securi ##### Request body -``` +```json { "SAMLResponse": "" } @@ -450,9 +448,6 @@ This is the callback endpoint that the identity provider will use to send securi `Status: 200` -``` -{} -``` --- @@ -481,7 +476,7 @@ This is the callback endpoint that the identity provider will use to send securi | order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. | | status | string | query | Indicates the status of the hosts to return. Can either be `new`, `online`, `offline`, or `mia`. | | query | string | query | Search query keywords. Searchable fields include `hostname`, `machine_serial`, `uuid`, and `ipv4`. | -| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | +| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../01-Using-Fleet/02-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | | team_id | integer | query | _Available in Fleet Premium_ Filters the users to only include users in the specified team. | | policy_id | integer | query | The ID of the policy to filter hosts by. `policy_response` must also be specified with `policy_id`. | | policy_response | string | query | Valid options are `passing` or `failing`. `policy_id` must also be specified with `policy_response`. | @@ -494,7 +489,7 @@ If `additional_info_filters` is not specified, no `additional` information will ##### Request query parameters -``` +```json { "page": 0, "per_page": 100, @@ -506,7 +501,7 @@ If `additional_info_filters` is not specified, no `additional` information will `Status: 200` -``` +```json { "hosts": [ { @@ -570,7 +565,7 @@ None. `Status: 200` -``` +```json { "online_count": 2267, "offline_count": 141, @@ -583,7 +578,7 @@ None. Returns the information of the specified host. -The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../2-Deploying/2-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. +The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../02-Deploying/02-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. `GET /api/v1/fleet/hosts/{id}` @@ -601,7 +596,7 @@ The endpoint returns the host's installed `software` if the software inventory f `Status: 200` -``` +```json { "host": { "created_at": "2021-08-19T02:02:22Z", @@ -738,7 +733,7 @@ Returns the information of the host specified using the `uuid`, `osquery_host_id `Status: 200` -``` +```json { "host": { "created_at": "2020-11-05T05:09:44Z", @@ -804,9 +799,6 @@ Deletes the specified host from Fleet. Note that a deleted host will fail authen `Status: 200` -``` -{} -``` ### Refetch host @@ -828,9 +820,6 @@ Flags the host details to be refetched the next time the host checks in for live `Status: 200` -``` -{} -``` ### Transfer hosts to a team @@ -851,7 +840,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "team_id": 1, "hosts": [3, 2, 4, 6, 1, 5, 7] @@ -862,9 +851,6 @@ _Available in Fleet Premium_ `Status: 200` -``` -{} -``` ### Transfer hosts to a team by filter @@ -885,7 +871,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "team_id": 1, "filters": { @@ -898,9 +884,6 @@ _Available in Fleet Premium_ `Status: 200` -``` -{} -``` --- @@ -938,7 +921,7 @@ Creates a dynamic label. ##### Request body -``` +```json { "name": "Ubuntu hosts", "description": "Filters ubuntu hosts", @@ -951,7 +934,7 @@ Creates a dynamic label. `Status: 200` -``` +```json { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -989,7 +972,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T ##### Request body -``` +```json { "name": "macOS label", "description": "Now this label only includes macOS machines", @@ -1001,7 +984,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T `Status: 200` -``` +```json { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -1040,7 +1023,7 @@ Returns the specified label. `Status: 200` -``` +```json { "label": { "created_at": "2021-02-09T22:09:43Z", @@ -1080,7 +1063,7 @@ Returns a list of all the labels in Fleet. `Status: 200` -``` +```json { "labels": [ { @@ -1184,7 +1167,7 @@ Returns a list of the hosts that belong to the specified label. `Status: 200` -``` +```json { "hosts": [ { @@ -1251,9 +1234,6 @@ Deletes the label specified by name. `Status: 200` -``` -{} -``` ### Delete label by ID @@ -1275,9 +1255,6 @@ Deletes the label specified by ID. `Status: 200` -``` -{} -``` ### Apply labels specs @@ -1301,7 +1278,7 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als ##### Request body -``` +```json { "specs": [ { @@ -1326,9 +1303,6 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als `Status: 200` -``` -{} -``` ### Get labels specs @@ -1346,7 +1320,7 @@ None. `Status: 200` -``` +```json { "specs": [ { @@ -1421,7 +1395,7 @@ None. `Status: 200` -``` +```json { "specs": { "id": 12, @@ -1480,7 +1454,7 @@ None. `Status: 200` -``` +```json { "users": [ { @@ -1512,7 +1486,7 @@ None. `Status: 401 Authentication Failed` -``` +```json { "message": "Authentication Failed", "errors": [ @@ -1548,7 +1522,7 @@ Creates a user account after an invited user provides registration information a ##### Request query parameters -``` +```json { "email": "janedoe@example.com", "invite_token": "SjdReDNuZW5jd3dCbTJtQTQ5WjJTc2txWWlEcGpiM3c=", @@ -1572,7 +1546,7 @@ Creates a user account after an invited user provides registration information a `Status: 200` -``` +```json { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1594,7 +1568,7 @@ Creates a user account after an invited user provides registration information a `Status: 401 Authentication Failed` -``` +```json { "message": "Authentication Failed", "errors": [ @@ -1610,7 +1584,7 @@ Creates a user account after an invited user provides registration information a `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1628,7 +1602,7 @@ Creates a user account after an invited user provides registration information a The same error will be returned whenever one of the required parameters fails the validation. -``` +```json { "message": "Validation Failed", "errors": [ @@ -1664,7 +1638,7 @@ Creates a user account without requiring an invitation, the user is enabled imme ##### Request body -``` +```json { "name": "Jane Doe", "email": "janedoe@example.com", @@ -1686,7 +1660,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 200` -``` +```json { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1703,11 +1677,11 @@ Creates a user account without requiring an invitation, the user is enabled imme "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -1718,7 +1692,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1748,7 +1722,7 @@ Returns all information about a specific user. ##### Request query parameters -``` +```json { "id": 1 } @@ -1758,7 +1732,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2020-12-10T05:20:25Z", @@ -1780,7 +1754,7 @@ Returns all information about a specific user. `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1815,7 +1789,7 @@ Returns all information about a specific user. ##### Request body -``` +```json { "name": "Jane Doe", "global_role": "admin" @@ -1826,7 +1800,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1850,15 +1824,15 @@ Returns all information about a specific user. ##### Request body -``` +```json { "teams": [ { "id": 1, - "role: "observer" + "role": "observer" }, { - "id": 2 + "id": 2, "role": "maintainer" } ] @@ -1869,7 +1843,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1881,15 +1855,15 @@ Returns all information about a specific user. "force_password_reset": false, "gravatar_url": "", "sso_enabled": false, - "global_role": "admin" + "global_role": "admin", "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -1916,9 +1890,6 @@ Delete the specified user from Fleet. `Status: 200` -``` -{} -``` ### Require password reset @@ -1939,7 +1910,7 @@ The selected user is logged out of Fleet and required to reset their password du ##### Request body -``` +```json { "require": true } @@ -1949,7 +1920,7 @@ The selected user is logged out of Fleet and required to reset their password du `Status: 200` -``` +```json { "user": { "created_at": "2021-02-23T22:23:34Z", @@ -1984,7 +1955,7 @@ None. `Status: 200` -``` +```json { "sessions": [ { @@ -2026,9 +1997,6 @@ Deletes the selected user's sessions in Fleet. Also deletes the user's API token `Status: 200` -``` -{} -``` --- @@ -2057,7 +2025,7 @@ Returns the session information for the session specified by ID. `Status: 200` -``` +```json { "session_id": 1, "user_id": 1, @@ -2085,9 +2053,6 @@ Deletes the session specified by ID. When the user associated with the session n `Status: 200` -``` -{} -``` --- @@ -2130,7 +2095,7 @@ Returns the query specified by ID. `Status: 200` -``` +```json { "query": { "created_at": "2021-01-19T17:08:24Z", @@ -2179,7 +2144,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -``` +```json { "queries": [ { @@ -2250,7 +2215,7 @@ Returns a list of all queries in the Fleet instance. ##### Request body -``` +```json { "description": "This is a new query.", "name": "new_query", @@ -2262,7 +2227,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -``` +```json { "query": { "created_at": "0001-01-01T00:00:00Z", @@ -2302,7 +2267,7 @@ Returns the query specified by ID. ##### Request body -``` +```json { "name": "new_title_for_my_query" } @@ -2312,7 +2277,7 @@ Returns the query specified by ID. `Status: 200` -``` +```json { "query": { "created_at": "2021-01-22T17:23:27Z", @@ -2350,9 +2315,6 @@ Deletes the query specified by name. `Status: 200` -``` -{} -``` ### Delete query by ID @@ -2374,9 +2336,6 @@ Deletes the query specified by ID. `Status: 200` -``` -{} -``` ### Delete queries @@ -2396,7 +2355,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d ##### Request body -``` +```json { "ids": [ 2, 24, 25 @@ -2408,7 +2367,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d `Status: 200` -``` +```json { "deleted": 3 } @@ -2432,7 +2391,7 @@ None. `Status: 200` -``` +```json { "specs": [ { @@ -2469,7 +2428,7 @@ Returns the name, description, and SQL of the query specified by name. `Status: 200` -``` +```json { "specs": { "name": "query1", @@ -2497,7 +2456,7 @@ Creates and/or modifies the queries included in the specs list. To modify an exi ##### Request body -``` +```json { "specs": [ { @@ -2518,9 +2477,6 @@ Creates and/or modifies the queries included in the specs list. To modify an exi `Status: 200` -``` -{} -``` ### Live query health check @@ -2540,9 +2496,6 @@ None. `Status: 200` -``` -{} -``` ### live query result store health check @@ -2562,9 +2515,6 @@ None. `Status: 200` -``` -{} -``` ### Run live query @@ -2590,7 +2540,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info", "selected": { @@ -2603,7 +2553,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2629,7 +2579,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info;", "selected": { @@ -2642,7 +2592,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2686,7 +2636,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query_id": 1, "selected": { @@ -2701,7 +2651,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2727,7 +2677,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info", "selected": { @@ -2742,7 +2692,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2806,7 +2756,7 @@ socket.onmessage = ({ data }) => { ###### Response data -``` +```json o ``` @@ -2814,7 +2764,7 @@ o ###### Request data -``` +```json [ { "type": "auth", @@ -2823,7 +2773,7 @@ o ] ``` -``` +```json [ { "type": "select_campaign", @@ -2836,7 +2786,7 @@ o ###### Response data -``` +```json // Sends the total number of hosts targeted and segments them by status [ @@ -2853,7 +2803,7 @@ o ] ``` -``` +```json // Sends the expected results, actual results so far, and the status of the live query [ @@ -2869,7 +2819,7 @@ o ] ``` -``` +```json // Sends the result for a given host [ @@ -2889,7 +2839,7 @@ o ] ``` -``` +```json // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -2948,7 +2898,7 @@ socket.onmessage = ({ data }) => { ###### Response data -``` +```json o ``` @@ -2956,7 +2906,7 @@ o ###### Request data -``` +```json [ { "type": "auth", @@ -2965,7 +2915,7 @@ o ] ``` -``` +```json [ { "type": "select_campaign", @@ -2974,11 +2924,11 @@ o ] ``` -##### `socket.onmessage()` +##### socket.onmessage() ###### Response data -``` +```json // Sends the total number of hosts targeted and segments them by status [ @@ -2995,7 +2945,7 @@ o ] ``` -``` +```json // Sends the expected results, actual results so far, and the status of the live query [ @@ -3011,7 +2961,7 @@ o ] ``` -``` +```json // Sends the result for a given host [ @@ -3031,7 +2981,7 @@ o ] ``` -``` +```json // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -3078,7 +3028,7 @@ None. `Status: 200` -``` +```json { "global_schedule": [ { @@ -3141,7 +3091,7 @@ None. ##### Request body -``` +```json { "interval": 86400, "query_id": 2, @@ -3153,7 +3103,7 @@ None. `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3199,7 +3149,7 @@ None. ##### Request body -``` +```json { "interval": 604800, } @@ -3209,7 +3159,7 @@ None. `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3246,9 +3196,6 @@ None. `Status: 200` -``` -{} -``` --- @@ -3265,7 +3212,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Get team schedule -`GET /api/v1/fleet/team/{id}/schedule` +`GET /api/v1/fleet/teams/{id}/schedule` #### Parameters @@ -3279,13 +3226,13 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`GET /api/v1/fleet/team/2/schedule` +`GET /api/v1/fleet/teams/2/schedule` ##### Default response `Status: 200` -``` +```json { "scheduled": [ { @@ -3328,7 +3275,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Add query to team schedule -`POST /api/v1/fleet/team/{id}/schedule` +`POST /api/v1/fleet/teams/{id}/schedule` #### Parameters @@ -3345,11 +3292,11 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`POST /api/v1/fleet/team/2/schedule` +`POST /api/v1/fleet/teams/2/schedule` ##### Request body -``` +```json { "interval": 86400, "query_id": 2, @@ -3361,7 +3308,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3383,7 +3330,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Edit query in team schedule -`PATCH /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` +`PATCH /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3400,11 +3347,11 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`PATCH /api/v1/fleet/team/2/schedule/5` +`PATCH /api/v1/fleet/teams/2/schedule/5` ##### Request body -``` +```json { "interval": 604800, } @@ -3414,7 +3361,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3437,7 +3384,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Remove query from team schedule -`DELETE /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` +`DELETE /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3448,15 +3395,12 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`DELETE /api/v1/fleet/team/2/schedule/5` +`DELETE /api/v1/fleet/teams/2/schedule/5` ##### Default response `Status: 200` -``` -{} -``` --- @@ -3497,7 +3441,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -``` +```json { "description": "Collects osquery data.", "host_ids": [], @@ -3510,7 +3454,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "0001-01-01T00:00:00Z", @@ -3550,7 +3494,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -``` +```json { "description": "MacOS hosts are targeted", "host_ids": [], @@ -3562,7 +3506,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3599,7 +3543,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3637,7 +3581,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "packs": [ { @@ -3690,9 +3634,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` -{} -``` ### Delete pack by ID @@ -3712,9 +3653,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` -{} -``` ### Get scheduled queries in a pack @@ -3734,7 +3672,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": [ { @@ -3815,7 +3753,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -``` +```json { "interval": 120, "pack_id": 15, @@ -3832,7 +3770,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3871,7 +3809,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3915,7 +3853,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -``` +```json { "platform": "", } @@ -3925,7 +3863,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-01-28T19:40:04Z", @@ -3964,9 +3902,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` -{} -``` ### Get packs specs @@ -3982,7 +3917,7 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -``` +```json { "specs": [ { @@ -4102,7 +4037,7 @@ Returns the specs for all packs in the Fleet instance. ##### Request body -``` +```json { "specs": [ { @@ -4193,9 +4128,6 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -``` -{} -``` ### Get pack spec by name @@ -4217,7 +4149,7 @@ Returns the spec for the specified pack by pack name. `Status: 200` -``` +```json { "specs": { "id": 15, @@ -4316,7 +4248,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policies": [ { @@ -4355,7 +4287,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policy": { "id": 1, @@ -4383,7 +4315,7 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body -``` +```json { "query_id": 12 } @@ -4393,7 +4325,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policy": { "id": 2, @@ -4421,6 +4353,161 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body +```json +{ + "ids": [ 1 ] +} +``` + +##### Default response + +`Status: 200` + +```json +{ + "deleted": 1 +} +``` + +--- + +## Team Policies + +- [List team policies](#list-team-policies) +- [Get team policy by ID](#get-team-policy-by-id) +- [Add team policy](#add-team-policy) +- [Remove team policies](#remove-team-policies) + +_Available in Fleet Premium_ + +Team policies work the same as policies, but at the team level. + +### List team policies + +`GET /api/v1/fleet/teams/{team_id}/policies` + +#### Parameters + +| Name | Type | In | Description | +| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | + +#### Example + +`GET /api/v1/fleet/teams/1/policies` + +##### Default response + +`Status: 200` + +``` +{ + "policies": [ + { + "id": 1, + "query_id": 2, + "query_name": "Gatekeeper enabled", + "passing_host_count": 2000, + "failing_host_count": 300, + }, + { + "id": 2, + "query_id": 3, + "query_name": "Primary disk encrypted", + "passing_host_count": 2300, + "failing_host_count": 0, + } + ] +} +``` + +### Get team policy by ID + +`GET /api/v1/fleet/teams/{team_id}/policies/{id}` + +#### Parameters + +| Name | Type | In | Description | +| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| id | integer | path | **Required.** The policy's ID. | + +#### Example + +`GET /api/v1/fleet/teams/1/policies/1` + +##### Default response + +`Status: 200` + +``` +{ + "policy": { + "id": 1, + "query_id": 2, + "query_name": "Gatekeeper enabled", + "passing_host_count": 2000, + "failing_host_count": 300, + } +} +``` + +### Add team policy + +`POST /api/v1/fleet/teams/{team_id}/policies` + +#### Parameters + +| Name | Type | In | Description | +| -------- | ------- | ---- | ----------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| query_id | integer | body | **Required.** The query's ID. | + +#### Example + +`POST /api/v1/fleet/teams/1/policies` + +#### Request body + +``` +{ + "query_id": 12 +} +``` + +##### Default response + +`Status: 200` + +``` +{ + "policy": { + "id": 2, + "query_id": 2, + "query_name": "Primary disk encrypted", + "passing_host_count": 0, + "failing_host_count": 0, + }, +} +``` + +### Remove team policies + +`POST /api/v1/fleet/teams/{team_id}/policies/delete` + +#### Parameters + +| Name | Type | In | Description | +| -------- | ------- | ---- | ------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| ids | list | body | **Required.** The IDs of the policies to delete. | + +#### Example + +`POST /api/v1/fleet/teams/1/policies/delete` + +#### Request body + ``` { "ids": [ 1 ] @@ -4474,7 +4561,7 @@ Returns a list of the activities that have been performed in Fleet. The followin ##### Default response -``` +```json { "activities": [ { @@ -4637,7 +4724,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Request body -``` +```json { "query": "172", "selected": { @@ -4650,7 +4737,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Default response -``` +```json { "targets": { "hosts": [ @@ -4800,7 +4887,7 @@ None. `Status: 200` -``` +```json { "certificate_chain": } @@ -4824,7 +4911,7 @@ None. `Status: 200` -``` +```json { "org_info": { "org_name": "fleet", @@ -4915,7 +5002,7 @@ None. "enable_log_compression": false } } - } + }, "license": { "tier": "free", "organization": "fleet", @@ -4955,10 +5042,11 @@ None. "enable_log_compression": false } } - } + }, "update_interval": { - "osquery_detail": 3600000000000 - } + "osquery_detail": 3600000000000, + "osquery_policy": 3600000000000 + }, } ``` @@ -5005,13 +5093,13 @@ Modifies the Fleet's configuration with the supplied information. ##### Request body -``` +```json { "org_info": { "org_name": "Fleet Device Management", "org_logo_url": "https://fleetdm.com/logo.png" }, - "smtp_settings: { + "smtp_settings": { "enable_smtp": true, "server": "localhost", "port": "1025" @@ -5023,7 +5111,7 @@ Modifies the Fleet's configuration with the supplied information. `Status: 200` -``` +```json { "org_info": { "org_name": "Fleet Device Management", @@ -5148,7 +5236,7 @@ None. `Status: 200` -``` +```json { "spec": { "secrets": [ @@ -5181,7 +5269,7 @@ Replaces the active global enroll secrets with the secrets specified. ##### Request body -``` +```json { "spec": { "secrets": [ @@ -5199,9 +5287,6 @@ Replaces the active global enroll secrets with the secrets specified. `Status: 200` -``` -{} -``` ### Get enroll secret for a team @@ -5221,7 +5306,7 @@ None. `Status: 200` -``` +```json { "secrets": [ { @@ -5251,20 +5336,20 @@ None. ##### Request body -``` +```json { "email": "john_appleseed@example.com", - "name": John, + "name": "John", "sso_enabled": false, - "global_role": "admin" + "global_role": "admin", "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -5276,13 +5361,13 @@ None. `Status: 200` -``` +```json { "invite": { "created_at": "0001-01-01T00:00:00Z", "updated_at": "0001-01-01T00:00:00Z", "id": 3, - "invited_by": 1 + "invited_by": 1, "email": "john_appleseed@example.com", "name": "John", "sso_enabled": false, @@ -5334,7 +5419,7 @@ Returns a list of the active invitations in Fleet. `Status: 200` -``` +```json { "invites": [ { @@ -5381,9 +5466,6 @@ Delete the specified invite from Fleet. `Status: 200` -``` -{} -``` ### Verify invite @@ -5405,7 +5487,7 @@ Verify the specified invite. `Status: 200` -``` +```json { "invite": { "created_at": "2021-01-15T00:58:33Z", @@ -5424,7 +5506,7 @@ Verify the specified invite. `Status: 404` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -5454,7 +5536,7 @@ None. `Status: 200` -``` +```json { "version": "3.9.0-93-g1b67826f-dirty", "branch": "version", @@ -5477,7 +5559,7 @@ Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allo To initiate a file carve using the Fleet API, you can use the [live query](#run-live-query) or [scheduled query](#add-scheduled-query-to-a-pack) endpoints to run a query against the `carves` table. -For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../1-Using-Fleet/2-fleetctl-CLI.md#file-carving-with-fleet). +For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../01-Using-Fleet/02-fleetctl-CLI.md#file-carving-with-fleet). ### List carves @@ -5497,7 +5579,7 @@ None. `Status: 200` -``` +```json { "carves": [ { @@ -5552,7 +5634,7 @@ Retrieves the specified carve. `Status: 200` -``` +```json { "carve": { "id": 1, @@ -5592,7 +5674,7 @@ Retrieves the specified carve block. This endpoint retrieves the data that was c `Status: 200` -``` +```json { "data": "aG9zdHMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA..." } @@ -5626,9 +5708,9 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { - "teams: [ + "teams": [ { "id": 1. "created_at": "2021-07-28T15:58:21Z", @@ -5724,7 +5806,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "name": "workstations" } @@ -5734,12 +5816,12 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { - "teams: [ + "teams": [ { "name": "workstations", - "id": 1 + "id": 1, "user_ids": [], "host_ids": [], "user_count": 0, @@ -5793,7 +5875,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "user_ids": [1, 17, 22, 32], } @@ -5803,11 +5885,11 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [], "user_count": 4, @@ -5845,7 +5927,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], } @@ -5855,11 +5937,11 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -5897,7 +5979,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "agent_options": { "spec": { @@ -5929,11 +6011,11 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -5985,9 +6067,74 @@ _Available in Fleet Premium_ `Status: 200` +### Apply team spec + +_Available in Fleet Premium_ + +If the `name` specified is associated with an existing team, this API route, completely replaces this team's existing `agent_options` and `secrets` with those that are specified. + +If the `name` is not already associated with an existing team, this API route creates a new team with the specified `name`, `agent_options`, and `secrets`. + +`POST /api/v1/fleet/spec/teams` + +#### Parameters + +| Name | Type | In | Description | +| ---- | ------ | ---- | ------------------------------ | +| name | string | body | **Required.** The team's name. | +| agent_options | string | body | **Required.** The agent options spec that is applied to the hosts assigned to the specified to team. These agent agent options completely override the global agent options specified in the [`GET /api/v1/fleet/config API route`](#get-configuration)| +| secrets | list | body | **Required.** A list of plain text strings used as the enroll secrets. | + +#### Example + +`POST /api/v1/fleet/spec/teams` + +##### Request body + +```json +{ + "specs": [ + { + "name": "Client Platform Engineering", + "agent_options": { + "spec": { + "config": { + "options": { + "logger_plugin": "tls", + "pack_delimiter": "/", + "logger_tls_period": 10, + "distributed_plugin": "tls", + "disable_distributed": false, + "logger_tls_endpoint": "/api/v1/osquery/log", + "distributed_interval": 10, + "distributed_tls_max_attempts": 3 + }, + "decorators": { + "load": [ + "SELECT uuid AS host_uuid FROM system_info;", + "SELECT hostname AS hostname FROM system_info;" + ] + } + }, + "overrides": {} + } + }, + "secrets": [ + { + "secret": "fTp52/twaxBU6gIi0J6PHp8o5Sm1k1kn", + }, + { + "secret": "bhD5kiX2J+KBgZSk118qO61ZIdX/v8On", + } + ] + } + ] +} ``` -{} -``` + +#### Default response + +`Status: 200` --- @@ -6009,7 +6156,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "list": [ { @@ -6044,7 +6191,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "list": [ { @@ -6104,7 +6251,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { “software”: [ { diff --git a/docs/1-Using-Fleet/4-Adding-hosts.md b/docs/01-Using-Fleet/04-Adding-hosts.md similarity index 99% rename from docs/1-Using-Fleet/4-Adding-hosts.md rename to docs/01-Using-Fleet/04-Adding-hosts.md index 4872b7a204..9ea6e02e39 100644 --- a/docs/1-Using-Fleet/4-Adding-hosts.md +++ b/docs/01-Using-Fleet/04-Adding-hosts.md @@ -135,4 +135,4 @@ Multiple enroll secrets can be set to allow different groups of hosts to authenticate with Fleet. When a host enrolls, the corresponding enroll secret is recorded and can be used to segment hosts. -To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../1-Using-Fleet/2-fleetctl-CLI.md#enroll-secrets) +To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../01-Using-Fleet/02-fleetctl-CLI.md#enroll-secrets) diff --git a/docs/1-Using-Fleet/5-Osquery-logs.md b/docs/01-Using-Fleet/05-Osquery-logs.md similarity index 90% rename from docs/1-Using-Fleet/5-Osquery-logs.md rename to docs/01-Using-Fleet/05-Osquery-logs.md index 96109f95c2..48223c68c6 100644 --- a/docs/1-Using-Fleet/5-Osquery-logs.md +++ b/docs/01-Using-Fleet/05-Osquery-logs.md @@ -22,21 +22,21 @@ Fleet supports the following logging plugins for osquery logs: - [PubSub](#pubsub) - Logs are written to Google Cloud PubSub topics. - [Stdout](#stdout) - Logs are written to stdout. -To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../2-Deploying/2-Configuration.md#options)). +To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../02-Deploying/02-Configuration.md#options)). ### Filesystem The default logging plugin. - Plugin name: `filesystem` -- Flag namespace: [filesystem](../2-Deploying/2-Configuration.md#filesystem) +- Flag namespace: [filesystem](../02-Deploying/02-Configuration.md#filesystem) With the filesystem plugin, osquery result and/or status logs are written to the local filesystem on the Fleet server. This is typically used with a log forwarding agent on the Fleet server that will push the logs into a logging pipeline. Note that if multiple load-balanced Fleet servers are used, the logs will be load-balanced across those servers (not duplicated). ### Firehose - Plugin name: `firehose` -- Flag namespace: [firehose](../2-Deploying/2-Configuration.md#firehose) +- Flag namespace: [firehose](../02-Deploying/02-Configuration.md#firehose) With the Firehose plugin, osquery result and/or status logs are written to [AWS Firehose](https://aws.amazon.com/kinesis/data-firehose/) streams. This is a very good method for aggregating osquery logs into AWS S3 storage. @@ -45,7 +45,7 @@ Note that Firehose logging has limits [discussed in the documentation](https://d ### Kinesis - Plugin name: `kinesis` -- Flag namespace: [kinesis](../2-Deploying/2-Configuration.md#kinesis) +- Flag namespace: [kinesis](../02-Deploying/02-Configuration.md#kinesis) With the Kinesis plugin, osquery result and/or status logs are written to [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams) streams. @@ -58,7 +58,7 @@ output in the Fleet logs and those logs _will not_ be sent to Kinesis. ### Lambda - Plugin name: `lambda` -- Flag namespace: [lambda](../2-Deploying/2-Configuration.md#lambda) +- Flag namespace: [lambda](../02-Deploying/02-Configuration.md#lambda) With the Lambda plugin, osquery result and/or status logs are written to [AWS Lambda](https://aws.amazon.com/lambda/) functions. @@ -79,7 +79,7 @@ Keep this in mind when using Lambda, as you're charged based on the number of re ### PubSub - Plugin name: `pubsub` -- Flag namespace: [pubsub](../2-Deploying/2-Configuration.md#pubsub) +- Flag namespace: [pubsub](../02-Deploying/02-Configuration.md#pubsub) With the PubSub plugin, osquery result and/or status logs are written to [PubSub](https://cloud.google.com/pubsub/) topics. @@ -88,7 +88,7 @@ Note that messages over 10MB will be dropped, with a notification sent to the fl ### Stdout - Plugin name: `stdout` -- Flag namespace: [stdout](../2-Deploying/2-Configuration.md#stdout) +- Flag namespace: [stdout](../02-Deploying/02-Configuration.md#stdout) With the stdout plugin, osquery result and/or status logs are written to stdout on the Fleet server. This is typically used for debugging or with a log diff --git a/docs/1-Using-Fleet/6-Monitoring-Fleet.md b/docs/01-Using-Fleet/06-Monitoring-Fleet.md similarity index 85% rename from docs/1-Using-Fleet/6-Monitoring-Fleet.md rename to docs/01-Using-Fleet/06-Monitoring-Fleet.md index 99d1ac1ef2..f7534f6a01 100644 --- a/docs/1-Using-Fleet/6-Monitoring-Fleet.md +++ b/docs/01-Using-Fleet/06-Monitoring-Fleet.md @@ -54,7 +54,7 @@ Scaling Fleet horizontally is as simple as running more Fleet server processes c The Fleet/osquery system is resilient to loss of availability. Osquery agents will continue executing the existing configuration and buffering result logs during downtime due to lack of network connectivity, server maintenance, or any other reason. Buffering in osquery can be configured with the `--buffered_log_max` flag. -Note that short downtimes are expected during [Fleet server upgrades](./8-Updating-Fleet.md)-fleet.md) that require database migrations. +Note that short downtimes are expected during [Fleet server upgrades](./08-Updating-Fleet.md)-fleet.md) that require database migrations. ### Debugging performance issues @@ -68,7 +68,7 @@ For performance issues in the Fleet server process, please [file an issue](https ##### Generate debug archive (Fleet 3.4.0+) -Use the `fleetctl archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./2-fleetctl-CLI.md)) for details on configuring `fleetctl`. +Use the `fleetctl debug archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./02-fleetctl-CLI.md)) for details on configuring `fleetctl`. The generated `.tar.gz` archive will be available in the current directory. @@ -86,4 +86,4 @@ fleetctl debug archive --context server-a ###### Confidential information -The `fleetctl archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. +The `fleetctl debug archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. diff --git a/docs/1-Using-Fleet/7-Security-best-practices.md b/docs/01-Using-Fleet/07-Security-best-practices.md similarity index 97% rename from docs/1-Using-Fleet/7-Security-best-practices.md rename to docs/01-Using-Fleet/07-Security-best-practices.md index 4311907ce2..caa13d02b2 100644 --- a/docs/1-Using-Fleet/7-Security-best-practices.md +++ b/docs/01-Using-Fleet/07-Security-best-practices.md @@ -33,7 +33,7 @@ Passwords are never stored in plaintext in the database. We store a `bcrypt`ed h ### Authentication tokens -The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../2-Deploying/2-Configuration.md#session_duration). +The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../02-Deploying/02-Configuration.md#session_duration). It is possible to revoke all session tokens for a user by forcing a password reset. diff --git a/docs/1-Using-Fleet/8-Updating-Fleet.md b/docs/01-Using-Fleet/08-Updating-Fleet.md similarity index 92% rename from docs/1-Using-Fleet/8-Updating-Fleet.md rename to docs/01-Using-Fleet/08-Updating-Fleet.md index af41724025..65d9c6e49c 100644 --- a/docs/1-Using-Fleet/8-Updating-Fleet.md +++ b/docs/01-Using-Fleet/08-Updating-Fleet.md @@ -7,7 +7,7 @@ ## Overview -This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../2-Deploying/1-Installation.md). +This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../02-Deploying/01-Installation.md). There are two steps to perform a typical Fleet update. If any other steps are required, they will be noted in the release notes. @@ -18,7 +18,7 @@ As with any enterprise software update, it's a good idea to back up your MySQL d ## Updating the Fleet binary -To update to a new version of Fleet, follow the [same binary install instructions](../2-Deploying/1-Installation.md) from the original installation method you used to install Fleet. +To update to a new version of Fleet, follow the [same binary install instructions](../02-Deploying/01-Installation.md) from the original installation method you used to install Fleet. ### Raw binaries diff --git a/docs/1-Using-Fleet/9-Permissions.md b/docs/01-Using-Fleet/09-Permissions.md similarity index 86% rename from docs/1-Using-Fleet/9-Permissions.md rename to docs/01-Using-Fleet/09-Permissions.md index af3bc7c678..17e2acba77 100644 --- a/docs/1-Using-Fleet/9-Permissions.md +++ b/docs/01-Using-Fleet/09-Permissions.md @@ -16,6 +16,8 @@ The following table depicts various permissions levels for each role. | ---------------------------------------------------- | -------- | ---------- | ----- | | Browse all hosts | ✅ | ✅ | ✅ | | Filter hosts using labels | ✅ | ✅ | ✅ | +| Browse all policies | ✅ | ✅ | ✅ | +| Filter hosts using policies | ✅ | ✅ | ✅ | | Target hosts using labels | ✅ | ✅ | ✅ | | Run saved queries as live queries against all hosts | ✅ | ✅ | ✅ | | Run custom queries as live queries against all hosts | | ✅ | ✅ | @@ -33,6 +35,8 @@ The following table depicts various permissions levels for each role. | Create labels | | ✅ | ✅ | | Edit labels | | ✅ | ✅ | | Delete labels | | ✅ | ✅ | +| Create new global policies | | ✅ | ✅ | +| Delete global policies | | ✅ | ✅ | | Create users | | | ✅ | | Edit users | | | ✅ | | Delete users | | | ✅ | @@ -66,9 +70,13 @@ The following table depicts various permissions levels in a team. | Action | Observer | Maintainer | | ------------------------------------------------------------ | -------- | ---------- | | Browse hosts assigned to team | ✅ | ✅ | +| Browse policies for hosts assigned to team | ✅ | ✅ | +| Filter hosts assigned to team using policies | ✅ | ✅ | | Filter hosts assigned to team using labels | ✅ | ✅ | | Target hosts assigned to team using labels | ✅ | ✅ | | Run saved queries as live queries on hosts assigned to team | ✅ | ✅ | +| Create new team policies | | ✅ | +| Delete team policies | | ✅ | | Run custom queries as live queries on hosts assigned to team | | ✅ | | Enroll hosts to member team | | ✅ | | Delete hosts belonging to member team | | ✅ | diff --git a/docs/1-Using-Fleet/10-Teams.md b/docs/01-Using-Fleet/10-Teams.md similarity index 94% rename from docs/1-Using-Fleet/10-Teams.md rename to docs/01-Using-Fleet/10-Teams.md index 86591f1055..4904e900db 100644 --- a/docs/1-Using-Fleet/10-Teams.md +++ b/docs/01-Using-Fleet/10-Teams.md @@ -94,7 +94,7 @@ To add users to a team: 4. Select one or more users by searching for their full name and confirm the action. -Users will be given the [Observer role](./9-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. +Users will be given the [Observer role](./09-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. ## Edit a member's role diff --git a/docs/1-Using-Fleet/11-Usage-statistics.md b/docs/01-Using-Fleet/11-Usage-statistics.md similarity index 100% rename from docs/1-Using-Fleet/11-Usage-statistics.md rename to docs/01-Using-Fleet/11-Usage-statistics.md diff --git a/docs/1-Using-Fleet/12-Supported-browsers.md b/docs/01-Using-Fleet/12-Supported-browsers.md similarity index 68% rename from docs/1-Using-Fleet/12-Supported-browsers.md rename to docs/01-Using-Fleet/12-Supported-browsers.md index 644545005c..5fae7fb5ea 100644 --- a/docs/1-Using-Fleet/12-Supported-browsers.md +++ b/docs/01-Using-Fleet/12-Supported-browsers.md @@ -19,4 +19,7 @@ We test each browser on Windows whenever possible, because our engineering team - Mobile Safari on iOS 10 - Mobile Chrome on Android 6 -**Note:** Mobile web is not yet supported in the Fleet Product. +### Note +> - Mobile web is not yet supported in the Fleet product. +> - The Fleet user interface [may not be fully supported](https://github.com/fleetdm/fleet/issues/969) in Google Chrome when the browser is running on Chrome OS + diff --git a/docs/1-Using-Fleet/13-Vulnerability-Processing.md b/docs/01-Using-Fleet/13-Vulnerability-Processing.md similarity index 96% rename from docs/1-Using-Fleet/13-Vulnerability-Processing.md rename to docs/01-Using-Fleet/13-Vulnerability-Processing.md index 70092798c5..acb02e5a87 100644 --- a/docs/1-Using-Fleet/13-Vulnerability-Processing.md +++ b/docs/01-Using-Fleet/13-Vulnerability-Processing.md @@ -69,6 +69,6 @@ FLEET_VULNERABILITIES_DATABASES_PATH=/some/path The path specified needs to exist and Fleet needs to be able to read and write to and from it. This is the only mandatory configuration needed for vulnerability processing to work. Additional options, like vulnerability check frequency, can be -found in the [configuration documentation](../2-Deploying/2-Configuration.md#vulnerabilities). +found in the [configuration documentation](../02-Deploying/02-Configuration.md#vulnerabilities). You'll need to restart the Fleet instances after changing these settings. \ No newline at end of file diff --git a/docs/1-Using-Fleet/FAQ.md b/docs/01-Using-Fleet/FAQ.md similarity index 79% rename from docs/1-Using-Fleet/FAQ.md rename to docs/01-Using-Fleet/FAQ.md index 99e0b69318..bc334c2733 100644 --- a/docs/1-Using-Fleet/FAQ.md +++ b/docs/01-Using-Fleet/FAQ.md @@ -13,6 +13,7 @@ - [Why aren’t my live queries being logged?](#why-arent-my-live-queries-being-logged) - [Can I use the Fleet API to fetch results from a scheduled query pack?](#can-i-use-the-fleet-api-to-fetch-results-from-a-scheduled-query-pack) - [How do I automatically add hosts to packs when the hosts enroll to Fleet?](#how-do-i-automatically-add-hosts-to-packs-when-the-hosts-enroll-to-Fleet) +- [How do I automatically assign a host to a team when it enrolls with Fleet?](#how-do-i-automatically-assign-a-host-to-a-team-when-it-enrolls-with-fleet) - [How do I resolve an "unknown column" error when upgrading Fleet?](#how-do-i-resolve-an-unknown-column-error-when-upgrading-fleet) ## What do I need to do to switch from Kolide Fleet to FleetDM Fleet? @@ -21,7 +22,7 @@ The upgrade from kolide/fleet to fleetdm/fleet works the same as any minor versi Minor version upgrades in Kolide Fleet often included database migrations and the recommendation to back up the database before migrating. The same goes for FleetDM Fleet versions. -To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./8-Updating-Fleet.md) of the documentation. +To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./08-Updating-Fleet.md) of the documentation. ## Has anyone stress tested Fleet? How many clients can the Fleet server handle? @@ -33,13 +34,13 @@ It’s standard deployment practice to have multiple Fleet servers behind a load No, currently, there’s no way to retrieve the name of the enroll secret with a query. This means that there's no way to create a label using your hosts' enroll secrets and then use this label as a target for queries or query packs. -Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./2-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. +Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./02-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. There is, however, a way to accomplish this even though the answer to the question remains "no": Teams. As of Fleet v4.0.0, you can group hosts in Teams either by enrolling them with a team specific secret, or by transferring hosts to a team. One the hosts you want to target are part of a team, you can create a query and target the team in question. ## How often do labels refresh? Is the refresh frequency configurable? -The update frequency for labels is configurable with the [—osquery_label_update_interval](../2-Deploying/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). +The update frequency for labels is configurable with the [—osquery_label_update_interval](../02-Deploying/02-Configuration.md#osquery_label_update_interval) flag (default 1 hour). ## How do I revoke the authorization tokens for a user? @@ -51,7 +52,7 @@ Fleet can live query the `osquery_schedule` table. Performing this live query al ## How do I monitor a Fleet server? -Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./6-Monitoring-Fleet.md) documentation for details. +Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./06-Monitoring-Fleet.md) documentation for details. ## Why is the “Add User” button disabled? @@ -76,7 +77,7 @@ Live query results (executed in the web UI or `fleetctl query`) are pushed direc ### Scheduled Queries -Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../2-Deploying/2-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. +Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../02-Deploying/02-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. It is possible to configure osqueryd to log query results outside of Fleet. For results to go to Fleet, the `--logger_plugin` flag must be set to `tls`. @@ -84,7 +85,7 @@ It is possible to configure osqueryd to log query results outside of Fleet. For Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. -The [logger configuration options](../2-Deploying/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. +The [logger configuration options](../02-Deploying/02-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. ### Troubleshooting @@ -94,7 +95,7 @@ Expecting results, but not seeing anything in the logs? - Check whether the query is scheduled in differential mode. If so, new results will only be logged when the result set changes. - Ensure that the query is scheduled to run on the intended platforms, and that the tables queried are supported by those platforms. - Use live query to `SELECT * FROM osquery_schedule` to check whether the query has been scheduled on the host. -- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../2-Deploying/2-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. +- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../02-Deploying/02-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. ## Why aren’t my live queries being logged? @@ -104,17 +105,17 @@ Live query results are never logged to the filesystem of the Fleet server. See [ You cannot. Scheduled query results are logged to whatever logging plugin you have configured and are not stored in the Fleet DB. -However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./3-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./3-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). +However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./03-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./03-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). As an example, let's say you want to retrieve a host's OS version, installed software, and kernel version: -Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./3-REST-API.md#list-hosts). +Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./03-REST-API.md#list-hosts). -The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../2-Deploying/2-Configuration.md#feature-flags). +The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../02-Deploying/02-Configuration.md#feature-flags). -Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./3-REST-API.md#get-host). +Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./03-REST-API.md#get-host). -It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./2-fleetctl-CLI.md#fleet-configuration-options). +It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./02-fleetctl-CLI.md#fleet-configuration-options). ## How do I automatically add hosts to packs when the hosts enroll to Fleet? @@ -122,10 +123,14 @@ You can accomplish this by adding specific labels as targets of your pack. First When your hosts enroll to Fleet, they will become a member of the label and, because the label is a target of your pack, these hosts will automatically become targets of the pack. -You can also do this by setting the `targets` field in the [YAML configuration file](./2-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. +You can also do this by setting the `targets` field in the [YAML configuration file](./02-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. + +## How do I automatically assign a host to a team when it enrolls with Fleet? + +[Team Enroll Secrets](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md#enroll-hosts-to-a-team) allow you to automatically assign a host to a team. ## How do I resolve an "unknown column" error when upgrading Fleet? The `unknown column` error typically occurs when the database migrations haven't been run during the upgrade process. -Check out the [documentation on running database migrations](./8-Updating-Fleet.md#running-database-migrations) to resolve this issue. +Check out the [documentation on running database migrations](./08-Updating-Fleet.md#running-database-migrations) to resolve this issue. diff --git a/docs/1-Using-Fleet/README.md b/docs/01-Using-Fleet/README.md similarity index 68% rename from docs/1-Using-Fleet/README.md rename to docs/01-Using-Fleet/README.md index 9dbd331a0b..1edfbc95e9 100644 --- a/docs/1-Using-Fleet/README.md +++ b/docs/01-Using-Fleet/README.md @@ -1,27 +1,27 @@ # Using Fleet -### [Fleet UI](./1-Fleet-UI.md) +### [Fleet UI](./01-Fleet-UI.md) Provides documentation about running and scheduling queries from within the Fleet UI -### [fleetctl CLI](./2-fleetctl-CLI.md) +### [fleetctl CLI](./02-fleetctl-CLI.md) Includes resources for setting up and configuring Fleet via the fleetctl CLI -### [REST API](./3-REST-API.md) +### [REST API](./03-REST-API.md) Provides resources for working with Fleet's API and includes example code for endpoints -### [Adding hosts](./4-Adding-hosts.md) +### [Adding hosts](./04-Adding-hosts.md) Provides resources for enrolling your hosts to Fleet -### [Osquery logs](./5-Osquery-logs.md) +### [Osquery logs](./05-Osquery-logs.md) Includes documentation on the plugin options for working with osquery logs -### [Monitoring Fleet](./6-Monitoring-Fleet.md) +### [Monitoring Fleet](./06-Monitoring-Fleet.md) Provides documentation for load balancer health checks and working with Fleet server metrics and performance -### [Security best practices](./7-Security-best-practices.md) +### [Security best practices](./07-Security-best-practices.md) Includes resources for ways to mitigate against the OWASP top 10 issues -### [Updating Fleet](./8-Updating-Fleet.md) +### [Updating Fleet](./08-Updating-Fleet.md) Includes a guide for how to update and run new versions of Fleet ### [FAQ](./FAQ.md) diff --git a/docs/1-Using-Fleet/configuration-files/README.md b/docs/01-Using-Fleet/configuration-files/README.md similarity index 100% rename from docs/1-Using-Fleet/configuration-files/README.md rename to docs/01-Using-Fleet/configuration-files/README.md diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml similarity index 94% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml index a26f40a92c..098270f021 100644 --- a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +++ b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml @@ -1,19 +1,19 @@ -apiVersion: v1 -kind: Service -metadata: - name: fleet-loadbalancer - labels: - app: fleet-loadbalancer -spec: - type: LoadBalancer - ports: - - name: proxy-tls - port: 443 - targetPort: 443 - protocol: TCP - - name: proxy-http - port: 80 - targetPort: 80 - protocol: TCP - selector: - app: fleet-webserver +apiVersion: v1 +kind: Service +metadata: + name: fleet-loadbalancer + labels: + app: fleet-loadbalancer +spec: + type: LoadBalancer + ports: + - name: proxy-tls + port: 443 + targetPort: 443 + protocol: TCP + - name: proxy-http + port: 80 + targetPort: 80 + protocol: TCP + selector: + app: fleet-webserver diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml new file mode 100644 index 0000000000..a85702f19e --- /dev/null +++ b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: team +spec: + team: + name: Client Platform Engineerin + agent_options: + config: + decorators: + load: + - SELECT uuid AS host_uuid FROM system_info; + - SELECT hostname AS hostname FROM system_info; + options: + disable_distributed: false + distributed_interval: 10 + distributed_plugin: tls + distributed_tls_max_attempts: 3 + logger_plugin: tls + logger_tls_endpoint: /api/v1/osquery/log + logger_tls_period: 10 + pack_delimiter: / + overrides: {} + secrets: + - secret: RzTlxPvugG4o4O5IKS/HqEDJUmI1hwBoffff diff --git a/docs/1-Using-Fleet/configuration-files/single-file-configuration.yml b/docs/01-Using-Fleet/configuration-files/single-file-configuration.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/single-file-configuration.yml rename to docs/01-Using-Fleet/configuration-files/single-file-configuration.yml diff --git a/docs/1-Using-Fleet/standard-query-library/README.md b/docs/01-Using-Fleet/standard-query-library/README.md similarity index 100% rename from docs/1-Using-Fleet/standard-query-library/README.md rename to docs/01-Using-Fleet/standard-query-library/README.md diff --git a/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml b/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml similarity index 98% rename from docs/1-Using-Fleet/standard-query-library/standard-query-library.yml rename to docs/01-Using-Fleet/standard-query-library/standard-query-library.yml index 41d3e8eee0..a9f9017dce 100644 --- a/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml +++ b/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml @@ -446,13 +446,13 @@ spec: query: SELECT name, path, pid FROM processes WHERE on_disk = 0; purpose: Incident response contributors: alphabrevity - --- +--- apiVersion: v1 kind: query spec: name: Get user files matching a specific hash platforms: macOS, Linux - Description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) + description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) query: SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE ""/Users/%/Documents/%%"") AND sha256 = ""16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75""; purpose: Informational contributors: alphabrevity @@ -466,13 +466,13 @@ spec: query: SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid; purpose: Informational contributors: alphabrevity -—-- +--- apiVersion: v1 kind: query spec: name: Get all listening ports, by process platforms: Linux, macOS, Windows - Description: List ports that are listening on all interfaces, along with the process to which they are attached. + description: List ports that are listening on all interfaces, along with the process to which they are attached. query: SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = "0.0.0.0"; purpose: Informational contributors: alphabrevity @@ -482,7 +482,7 @@ kind: query spec: name: Get whether TeamViewer is installed/running platforms: Windows - description: Description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. + description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. query: SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE "%teamviewer%"; purpose: Informational contributors: alphabrevity diff --git a/docs/2-Deploying/1-Installation.md b/docs/02-Deploying/01-Installation.md similarity index 91% rename from docs/2-Deploying/1-Installation.md rename to docs/02-Deploying/01-Installation.md index 7a1f0f89c7..d59018f353 100644 --- a/docs/2-Deploying/1-Installation.md +++ b/docs/02-Deploying/01-Installation.md @@ -15,7 +15,7 @@ The Fleet application is distributed as a single static binary. This binary serv - The Fleet application API endpoints - The osquery TLS server API endpoints -All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./2-Configuration.md) documentation for information on how to use and configure the Fleet application. +All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./02-Configuration.md) documentation for information on how to use and configure the Fleet application. ## Installing the Fleet binary @@ -29,7 +29,7 @@ Pull the latest Fleet docker image: docker pull fleetdm/fleet ``` -For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. ### Raw binaries @@ -47,7 +47,7 @@ unzip fleet.zip 'linux/*' -d fleet ./fleet/linux/fleet_linux_amd64 --help ``` -For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. ## TLS configuration @@ -65,7 +65,7 @@ Fleet currently has two infrastructure dependencies in addition to the `fleet` w ### MySQL -Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./2-Configuration.md) document. +Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./02-Configuration.md) document. Fleet requires at least MySQL version 5.7. @@ -73,4 +73,4 @@ For host expiry configuration, the [event scheduler](https://dev.mysql.com/doc/r ### Redis -Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./2-Configuration.md) document. +Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./02-Configuration.md) document. diff --git a/docs/2-Deploying/2-Configuration.md b/docs/02-Deploying/02-Configuration.md similarity index 84% rename from docs/2-Deploying/2-Configuration.md rename to docs/02-Deploying/02-Configuration.md index 1a41211d35..91ab7e63d1 100644 --- a/docs/2-Deploying/2-Configuration.md +++ b/docs/02-Deploying/02-Configuration.md @@ -131,7 +131,9 @@ All duration-based settings accept valid time units of `s`, `m`, `h`. ##### MySQL -This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica. +This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica, and it is recommended in that case to set a non-zero value for `mysql_read_replica_conn_max_lifetime` as in some environments, the replica's address may dynamically change to point +from the primary to an actual distinct replica based on auto-scaling options, so existing idle connections need to be recycled +periodically. ###### mysql_address @@ -289,7 +291,7 @@ Maximum idle connections to database. This value should be equal to or less than max_idle_conns: 50 ``` -###### conn_max_lifetime +###### mysql_conn_max_lifetime Maximum amount of time, in seconds, a connection may be reused. @@ -358,7 +360,7 @@ Whether or not to duplicate Live Query results to another Redis channel named `L ###### redis_connect_timeout -Timeout for redis connection. +Timeout for redis connection. - Default value: 5s - Environment variable: `FLEET_REDIS_CONNECT_TIMEOUT` @@ -382,6 +384,38 @@ Interval between keep alive probes. keep_alive: 30s ``` +###### redis_connect_retry_attempts + +Maximum number of attempts to retry a failed connection to a redis node. Only +certain type of errors are retried, such as connection timeouts. + +- Default value: 0 (no retry) +- Environment variable: `FLEET_REDIS_CONNECT_RETRY_ATTEMPTS` +- Config file format: + + ``` + redis: + connect_retry_attempts: 2 + ``` + +###### redis_cluster_follow_redirections + +Whether or not to automatically follow redirection errors received from the +Redis server. Applies only to Redis Cluster setups, ignored in standalone +Redis. In Redis Cluster, keys can be moved around to different nodes when the +cluster is unstable and reorganizing the data. With this configuration option +set to true, those (typically short and transient) redirection errors can be +handled transparently instead of ending in an error. + +- Default value: false +- Environment variable: `FLEET_REDIS_CLUSTER_FOLLOW_REDIRECTIONS` +- Config file format: + + ``` + redis: + cluster_follow_redirections: true + ``` + ##### Server ###### server_address @@ -401,7 +435,7 @@ The address to serve the Fleet webserver. The TLS cert to use when terminating TLS. -See [TLS certificate considerations](./1-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. +See [TLS certificate considerations](./01-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. - Default value: `./tools/osquery/fleet.crt` - Environment variable: `FLEET_SERVER_CERT` @@ -647,6 +681,23 @@ Valid time units are `s`, `m`, `h`. osquery: label_update_interval: 30m ``` + +###### osquery_policy_update_interval + +The interval at which Fleet will ask osquery agents to update their results for policy queries. + +Setting this to a higher value can reduce baseline load on the Fleet server in larger deployments. + +Valid time units are `s`, `m`, `h`. + +- Default value: `1h` +- Environment variable: `FLEET_OSQUERY_POLICY_UPDATE_INTERVAL` +- Config file format: + + ``` + osquery: + policy_update_interval: 30m + ``` ###### osquery_detail_update_interval @@ -695,6 +746,24 @@ Options are `filesystem`, `firehose`, `kinesis`, `lambda`, `pubsub`, and `stdout result_log_plugin: firehose ``` +###### osquery_max_jitter_percent + +Given an update interval (label, or details), this will add up to the defined percentage in randomness to the interval. + +The goal of this is to prevent all hosts from checking in with data at the same time. + +So for example, if the label_update_interval is 1h, and this is set to 10. It'll add up a random number between 0 and 6 minutes +to the amount of time it takes for fleet to give the host the label queries. + +- Default value: `10` +- Environment variable: `FLEET_OSQUERY_MAX_JITTER_PERCENT` +- Config file format: + + ``` + osquery: + max_jitter_percent: 10 + ``` + ##### Logging (Fleet server logging) ###### logging_debug @@ -1296,13 +1365,13 @@ When `current_instance_checks` is set to `auto` (the default), Fleet instances w How often vulnerabilities are checked. -- Default value: `1hr` +- Default value: `1h` - Environment variable: `FLEET_VULNERABILITIES_PERIODICITY` - Config file format: ``` vulnerabilities: - periodicity: 1hr + periodicity: 1h ``` ###### cpe_database_url @@ -1412,7 +1481,7 @@ sudo systemctl daemon-reload sudo systemctl restart fleet.service ``` -## Configuring Single Sign On +## Configuring Single Sign On (SSO) Fleet supports SAML single sign on capability. @@ -1422,33 +1491,22 @@ Fleet supports the SAML Web Browser SSO Profile using the HTTP Redirect Binding. ### Identity Provider (IDP) Configuration -Setting up the connected application (Fleet) with an identity provider generally requires the following information: +Setting up the service provider (Fleet) with an identity provider generally requires the following information: - _Assertion Consumer Service_ - This is the call back URL that the identity provider - will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. The value that you supply will be a fully qualified URL - consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, - if your Fleet web address is https://fleet.acme.org, then the value you would - use in the identity provider configuration would be: + will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. On Google it is "ACS URL". The value that you supply will be a fully qualified URL consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, if your Fleet web address is https://fleet.example.com, then the value you would use in the identity provider configuration would be: ``` - https://fleet.acme.org/api/v1/fleet/sso/callback + https://fleet.example.com/api/v1/fleet/sso/callback ``` -- _Entity ID_ - This value is a URI that you define. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. +- _Entity ID_ - This value is an identifier that you choose. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. - _Name ID Format_ - The value should be `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`. This may be shortened in the IDP setup to something like `email` or `EmailAddress`. - _Subject Type (Application username in Okta)_ - `email`. -After supplying the above information, the IDP will generate an issuer URI and a metadata URL that will be used to configure Fleet as a service provider. - - #### Example Okta IDP Configuration - - ![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) - -> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. - -> Individual users must also be setup on the IDP before they can sign in to Fleet. +After supplying the above information, the IDP will generate an issuer URI and a metadata that will be used to configure Fleet as a service provider. ### Fleet SSO Configuration @@ -1458,12 +1516,10 @@ If your IDP supports dynamic configuration, like Okta, you only need to provide Otherwise, the following values are required: -- _Identity Provider Name_ - A human friendly name of the IDP. +- _Identity Provider Name_ - A human readable name of the IDP. This is rendered on the login page. - _Entity ID_ - A URI that identifies your Fleet instance as the issuer of authorization - requests. Assuming your company name is Acme, an example might be `fleet.acme.org` although - the value could be anything as long as it is unique to Fleet as a service provider - and matches the entity provider value used in the IDP configuration. + requests (eg. `fleet.example.com`). This much match the _Entity ID_ configured with the IDP. - _Issuer URI_ - This value is obtained from the IDP. @@ -1487,6 +1543,69 @@ It is strongly recommended that at least one admin user is set up to use the tra based log in so that there is a fallback method for logging into Fleet in the event of SSO configuration problems. +#### Okta IDP Configuration + +![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) + +> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. + +> Individual users must also be setup on the IDP before they can sign in to Fleet. + +### Google Workspace IDP Configuration + +Follow these steps to configure Fleet SSO with Google Workspace. This will require administrator permissions in Google Workspace. + +1. Navigate to the [Web and Mobile Apps](https://admin.google.com/ac/apps/unified) section of the Google Workspace dashboard. Click _Add App -> Add custom SAML app_. + +Screen Shot 2021-09-15 at 2 47 36 PM + +2. Enter `Fleet` for the _App name_ and click _Continue_. + +Screen Shot 2021-09-15 at 2 50 06 PM + +3. Click _Download Metadata_, saving the metadata to your computer. Copy the _SSO URL_. Click _Continue_. + +Screen Shot 2021-09-15 at 2 52 56 PM + +4. In Fleet, navigate to the _Organization Settings_ page. Configure the _SAML Single Sign On Options_ section. + +- Check the _Enable Single Sign On_ checkbox. +- For _Identity Provider Name_ use `Google`. +- For _Entity ID_, use a unique identifier such as `fleet.example.com`. Note that Google seems to error when the provided ID includes `https://`. +- For _Issuer URI_, paste the _SSO URL_ copied from step 3. +- For _Metadata_, paste the contents of the downloaded metadata XML from step 3. +- All other fields can be left blank. + +Click _Update settings_ at the bottom of the page. + +Screen Shot 2021-09-15 at 5 32 25 PM + +5. In Google Workspace, configure the _Service provider details_. + +- For _ACS URL_, use `https:///api/v1/fleet/sso/callback` (eg. `https://fleet.example.com/api/v1/fleet/sso/callback`). +- For Entity ID, use **the same unique identifier from step 4** (eg. `fleet.example.com`). +- For _Name ID format_ choose `EMAIL`. +- For _Name ID_ choose `Basic Information > Primary email`. +- All other fields can be left blank. + +Click _Continue_ at the bottom of the page. + +Screen Shot 2021-09-15 at 5 36 56 PM + +6. Click _Finish_. + +Screen Shot 2021-09-15 at 2 57 20 PM + +7. Click the down arrow on the _User access_ section of the app details page. + +Screen Shot 2021-09-15 at 2 57 51 PM + +8. Check _ON for everyone_. Click _Save_. + +Screen Shot 2021-09-15 at 2 58 09 PM + +9. Enable SSO for a test user and try logging in. Note that Google sometimes takes a long time to propagate the SSO configuration, and it can help to try logging in to Fleet with an Incognito/Private window in the browser. + ## Feature flags Fleet features are sometimes gated behind feature flags. This will usually be due to not-yet-stable APIs, or not-fully-tested performance characteristics. diff --git a/docs/2-Deploying/3-Example-deployment-scenarios.md b/docs/02-Deploying/03-Example-deployment-scenarios.md similarity index 95% rename from docs/2-Deploying/3-Example-deployment-scenarios.md rename to docs/02-Deploying/03-Example-deployment-scenarios.md index b1b004859e..7dde2f46d7 100644 --- a/docs/2-Deploying/3-Example-deployment-scenarios.md +++ b/docs/02-Deploying/03-Example-deployment-scenarios.md @@ -48,7 +48,7 @@ vagrant ssh ### Installing Fleet -To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/1-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. +To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/01-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. For example, after downloading: ```sh @@ -190,11 +190,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on CentOS, you can run the following: @@ -357,11 +357,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on Ubuntu, you can run the following: @@ -453,14 +453,14 @@ We will use this address when we configure the Kubernetes deployment and databas The last step is to run the Fleet database migrations on your new MySQL server. To do this, run the following: ``` -kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` In Kubernetes, you can only run a job once. If you'd like to run it again (i.e.: you'd like to run the migrations again using the same file), you must delete the job before re-creating it. To delete the job and re-run it, you can run the following commands: ``` -kubectl delete -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml -kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl delete -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` #### Redis @@ -536,7 +536,7 @@ ts=2017-11-16T02:48:38.441148166Z transport=https address=0.0.0.0:443 msg=listen Now that the Fleet server is running on our cluster, we have to expose the Fleet webservers to the internet via a load balancer. To create a Kubernetes `Service` of type `LoadBalancer`, run the following: ``` -kubectl apply -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +kubectl apply -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml ``` #### Configure DNS diff --git a/docs/2-Deploying/4-fleetctl-agent-updates.md b/docs/02-Deploying/04-fleetctl-agent-updates.md similarity index 100% rename from docs/2-Deploying/4-fleetctl-agent-updates.md rename to docs/02-Deploying/04-fleetctl-agent-updates.md diff --git a/docs/2-Deploying/FAQ.md b/docs/02-Deploying/FAQ.md similarity index 84% rename from docs/2-Deploying/FAQ.md rename to docs/02-Deploying/FAQ.md index 4514a88d9b..5ed8391279 100644 --- a/docs/2-Deploying/FAQ.md +++ b/docs/02-Deploying/FAQ.md @@ -12,6 +12,8 @@ - [Why am I receiving a database connection error when attempting to "prepare" the database?](#why-am-i-receiving-a-database-connection-error-when-attempting-to-prepare-the-database) - [Is Fleet available as a SaaS product?](#is-fleet-available-as-a-saas-product) - [Is Fleet compatible with X flavor of MySQL?](#is-fleet-compatible-with-x-flavor-of-mysql) +- [What are the MySQL user access requirements?](#what-are-the-mysql-user-requirements) +- [What is duplicate enrollment and how do I fix it?](#what-is-duplicate-enrollment-and-how-do-i-fix-it) ## How do I get support for working with Fleet? @@ -25,7 +27,7 @@ Yes. Fleet scales horizontally out of the box as long as all of the Fleet server Note that osquery logs will be distributed across the Fleet servers. -Read the [performance documentation](../1-Using-Fleet/6-Monitoring-Fleet.md#fleet-server-performance) for more. +Read the [performance documentation](../01-Using-Fleet/06-Monitoring-Fleet.md#fleet-server-performance) for more. ## Why aren't my osquery agents connecting to Fleet? @@ -71,15 +73,15 @@ These configurations cannot be managed centrally from Fleet. ## What do I do about "too many open files" errors? -This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./2-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. +This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./02-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. -Some deployments may benefit by setting the [`--server_keepalive`](./2-Configuration.md#server_keepalive) flag to false. +Some deployments may benefit by setting the [`--server_keepalive`](./02-Configuration.md#server_keepalive) flag to false. This was also seen as a symptom of a different issue: if you're deploying on AWS on T type instances, there are different scenarios where the activity can increase and the instances will burst. If they run out of credits, then they'll stop processing leaving the file descriptors open. ## I upgraded my database, but Fleet is still running slowly. What could be going on? -This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./2-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./2-Configuration.md#mysql_max_idle_conns) flags appropriately. +This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./02-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./02-Configuration.md#mysql_max_idle_conns) flags appropriately. ## Why am I receiving a database connection error when attempting to "prepare" the database? @@ -104,3 +106,13 @@ No. Currently, Fleet is only available for self-hosting on premises or in the cl ## Is Fleet compatible with X flavor of MySQL? Fleet is built to run on MySQL 5.7 or above. However, particularly with AWS Aurora, we recommend 2.10.0 and above, as we've seen issues with anything below that. + +## What are the MySQL user requirements? + +The user `fleet prepare db` (via environment variable `FLEET_MYSQL_USERNAME` or command line flag `--mysql_username=`) uses to interact with the database needs to be able to create, alter, and drop tables as well as the ability to create temporary tables. + +## What is duplicate enrollment and how do I fix it? + +Duplicate host enrollment is when more than one host enrolls in Fleet using the same identifier (hardware UUID or osquery generated UUID). This can be caused by cloning a VM Image with an already enrolled +osquery client. To resolve the issues, it's advised to configure `--osquery_host_identifier` to `uuid`, and then delete the single host record for that whole set of hosts in the Fleet UI. You can find more information about +[host identifiers here](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#osquery_host_identifier). \ No newline at end of file diff --git a/docs/2-Deploying/README.md b/docs/02-Deploying/README.md similarity index 67% rename from docs/2-Deploying/README.md rename to docs/02-Deploying/README.md index 19ad6389e1..5b62992f4b 100644 --- a/docs/2-Deploying/README.md +++ b/docs/02-Deploying/README.md @@ -1,15 +1,15 @@ # Deployment -### [Installation](./1-Installation.md) +### [Installation](./01-Installation.md) Provides documentation on installing the Fleet binary and Fleet’s infrastructure dependencies -### [Configuration](./2-Configuration.md) +### [Configuration](./02-Configuration.md) Includes resources for configuring the Fleet binary, managing osquery configurations, and running with systemd -### [Example deployment scenarios](./3-Example-deployment-scenarios.md) +### [Example deployment scenarios](./03-Example-deployment-scenarios.md) Includes deployment walkthroughs for Fleet on CentOS, Ubuntu, and Kubernetes. -### [Self-managed agent updates](./4-fleetctl-agent-updates.md) +### [Self-managed agent updates](./04-fleetctl-agent-updates.md) Information about running an update server with fleetctl. ### [FAQ](./FAQ.md) diff --git a/docs/3-Contributing/1-Building-Fleet.md b/docs/03-Contributing/01-Building-Fleet.md similarity index 100% rename from docs/3-Contributing/1-Building-Fleet.md rename to docs/03-Contributing/01-Building-Fleet.md diff --git a/docs/3-Contributing/2-Testing.md b/docs/03-Contributing/02-Testing.md similarity index 78% rename from docs/3-Contributing/2-Testing.md rename to docs/03-Contributing/02-Testing.md index 73ef0a7d42..1aae17579d 100644 --- a/docs/3-Contributing/2-Testing.md +++ b/docs/03-Contributing/02-Testing.md @@ -7,7 +7,7 @@ - [Test hosts](#test-hosts) - [Email](#email) - [Database backup/restore](#database-backuprestore) -- [Teams seed data](#teams-seed-data) +- [Seeding Data](./06-Seeding-Data.md) - [MySQL shell](#mysql-shell) - [Testing SSO](#testing-sso) @@ -40,7 +40,7 @@ go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.0 Make sure it is available in your PATH. To execute the basic unit and integration tests, run the following from the root of the repository: ``` -MYSQL_TEST=1 make test +REDIS_TEST=1 MYSQL_TEST=1 make test ``` It is a good idea to run `make test` before submitting a Pull Request. @@ -50,7 +50,7 @@ It is a good idea to run `make test` before submitting a Pull Request. To run all Go unit tests, run the following: ``` -make test-go +REDIS_TEST=1 MYSQL_TEST=1 make test-go ``` #### Go linters @@ -105,6 +105,14 @@ To run email related integration tests using MailHog set environment as follows: MAIL_TEST=1 make test-go ``` +#### Network tests + +A few tests require network access as they make requests to external hosts. Given that the network is unreliable, may not be available, and those hosts may also not be unavailable, those tests are skipped by default and are opt-in via the `NETWORK_TEST` environment variable. To run them: + +``` +NETWORK_TEST=1 make test-go +``` + ### Viewing test coverage When you run `make test` or `make test-go` from the root of the repository, test coverage reports are generated in every subpackage. For example, the `server` subpackage will have a coverage report generated in `./server/server.cover` @@ -129,7 +137,7 @@ E2E tests are run using Docker and Cypress. #### Preparation -Make sure dependencies are up to date and the [Fleet binaries are built locally](./1-Building-Fleet.md). +Make sure dependencies are up to date and the [Fleet binaries are built locally](./01-Building-Fleet.md). For Fleet Free tests: @@ -223,51 +231,6 @@ Restore: Note that a "restore" will replace the state of the development database with the state from the backup. -## Teams seed data - -When developing Fleet, it may be useful to create seed data that includes users and teams. - -Check out this Loom demo video that walks through creating teams seed data: -https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 - -For a text-based walkthrough, check out the following steps: - -First, create a `env` file with the following contents: - -``` -export SERVER_URL=https://localhost:8080 # your fleet server url and port -export CURL_FLAGS='-k -s' # set insecure flag -export TOKEN=eyJhbGciOi... # your login token -``` - -Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. - -``` -export FLEET_ENV_PATH=/Users/victor/fleet_env -``` - -Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. - -The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. - -``` -./tools/api/fleet/teams/create_free -``` - -The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. - -``` -./tools/api/fleet/teams/create_premium -``` - -The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. - -``` -./tools/api/fleet/teams/create_figma -``` - -Each user generated by the script has their password set to `user123#`. - ## MySQL shell Connect to the MySQL shell to view and interact directly with the contents of the development database. diff --git a/docs/3-Contributing/3-Migrations.md b/docs/03-Contributing/03-Migrations.md similarity index 100% rename from docs/3-Contributing/3-Migrations.md rename to docs/03-Contributing/03-Migrations.md diff --git a/docs/3-Contributing/4-Committing-Changes.md b/docs/03-Contributing/04-Committing-Changes.md similarity index 96% rename from docs/3-Contributing/4-Committing-Changes.md rename to docs/03-Contributing/04-Committing-Changes.md index ac99857ae9..73ed37ff02 100644 --- a/docs/3-Contributing/4-Committing-Changes.md +++ b/docs/03-Contributing/04-Committing-Changes.md @@ -12,11 +12,14 @@ For significant changes, it is a good idea to discuss the proposal with the Flee Please keep in mind that any code merged to the Fleet repository becomes the responsibility of the Fleet team to maintain. Because of this, we are careful to ensure any contributions fit Fleet's vision, are well-tested, and high quality. We will work with contributors to ensure the appropriate standards are met. +## Fleet Device Management team members +Fleet Device Management team members may not copy queries from external sources except when that content has an explicit license allowing such use, or permission has been granted by the creator. + ## Pull Requests Each developer (internal or external) creates a fork of the Fleet repository, committing changes to a branch within their fork. Changes are submitted by PR to be merged into Fleet. -GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [2-Testing.md](./2-Testing.md). +GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [02-Testing.md](./02-Testing.md). For features that are still in-progress, the Pull Request can be marked as a "Draft". This helps make it clear which PRs are ready for review and merge. diff --git a/docs/3-Contributing/5-Releasing-Fleet.md b/docs/03-Contributing/05-Releasing-Fleet.md similarity index 100% rename from docs/3-Contributing/5-Releasing-Fleet.md rename to docs/03-Contributing/05-Releasing-Fleet.md diff --git a/docs/03-Contributing/06-Seeding-Data.md b/docs/03-Contributing/06-Seeding-Data.md new file mode 100644 index 0000000000..4c18effe4a --- /dev/null +++ b/docs/03-Contributing/06-Seeding-Data.md @@ -0,0 +1,44 @@ +# Seeding Data + +When developing Fleet, it may be useful to create seed data that includes users and teams. + +Check out this Loom demo video that walks through creating teams seed data: +https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 + +For a text-based walkthrough, check out the following steps: + +First, create a `env` file with the following contents: + +``` +export SERVER_URL=https://localhost:8080 # your fleet server url and port +export CURL_FLAGS='-k -s' # set insecure flag +export TOKEN=eyJhbGciOi... # your login token +``` + +Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. + +``` +export FLEET_ENV_PATH=/Users/victor/fleet_env +``` + +Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. + +The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. + +``` +./tools/api/fleet/teams/create_free +``` + +The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. + +``` +./tools/api/fleet/teams/create_premium +``` + +The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. + +``` +./tools/api/fleet/teams/create_figma +``` + +Each user generated by the script has their password set to `user123#`. \ No newline at end of file diff --git a/docs/3-Contributing/FAQ.md b/docs/03-Contributing/FAQ.md similarity index 93% rename from docs/3-Contributing/FAQ.md rename to docs/03-Contributing/FAQ.md index 4020cd0854..68c8d53b1b 100644 --- a/docs/3-Contributing/FAQ.md +++ b/docs/03-Contributing/FAQ.md @@ -25,7 +25,7 @@ server/fleet/emails.go:90:23: undefined: Asset make: *** [fleet] Error 2 ``` -If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./1-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. +If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./01-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. ## Adding hosts for testing diff --git a/docs/3-Contributing/README.md b/docs/03-Contributing/README.md similarity index 69% rename from docs/3-Contributing/README.md rename to docs/03-Contributing/README.md index 159d3045b4..98a9b38856 100644 --- a/docs/3-Contributing/README.md +++ b/docs/03-Contributing/README.md @@ -1,18 +1,18 @@ # Contribution -### [Building Fleet](./1-Building-Fleet.md) +### [Building Fleet](./01-Building-Fleet.md) Provides documentation about building the code, development infrastructure, and database migrations -### [Testing](./2-Testing.md) +### [Testing](./02-Testing.md) Includes documentation about Fleet's full test suite and integration tests -### [Migrations](./3-Migrations.md) +### [Migrations](./03-Migrations.md) Information about creating and updating database migrations -### [Committing Changes](./4-Committing-Changes.md) +### [Committing Changes](./04-Committing-Changes.md) Contains information about how to merge changes into the codebase -### [Releasing Fleet](./5-Releasing-Fleet.md) +### [Releasing Fleet](./05-Releasing-Fleet.md) Provides a guide for Fleet's release process ### [FAQ](./FAQ.md) diff --git a/docs/README.md b/docs/README.md index 3da7649ef8..5e29c0c967 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,13 +2,13 @@ Welcome to the documentation for the Fleet osquery fleet manager. -### [Using Fleet](./1-Using-Fleet/README.md) +### [Using Fleet](./01-Using-Fleet/README.md) Resources for using the Fleet UI, fleetctl CLI, and Fleet REST API. -### [Deploying](./2-Deploying/README.md) +### [Deploying](./02-Deploying/README.md) Resources for installing Fleet's infrastructure dependencies, configuring Fleet, deploying osquery to hosts, and viewing example deployment scenarios. -### [Contributing](./3-Contributing/README.md) +### [Contributing](./03-Contributing/README.md) If you're interested in interacting with the Fleet source code, you'll find information on modifying and building the code here. --- diff --git a/frontend/README.md b/frontend/README.md index 6331e7ee9f..147b24000f 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,34 +1,40 @@ # Fleet Front-End -The Fleet front-end is a Single Page Application using React and Redux. +The Fleet front-end is a Single Page Application using React with Typescript and Hooks. + +## Table of Contents +- [Running the Fleet web app](#running-the-fleet-web-app) +- [Directory Structure](#directory-structure) +- [Deprecated](#deprecated) +- [Patterns](#patterns) + - [Typing](#typing) + - [React Hooks (Functional Components)](#react-hooks-functional-components) + - [React Context](#react-context) + - [Fleet API Calls](#fleet-api-calls) + - [Page Routing](#page-routing) + - [Other](#other) ## Running the Fleet web app For details instruction on building and serving the Fleet web application -consult the [Contributing documentation](../docs/3-Contributing/README.md). +consult the [Contributing documentation](../docs/03-Contributing/README.md). ## Directory Structure Component directories in the Fleet front-end application encapsulate the entire -component, including files for the component, helper functions, styles, and tests. The +component, including files for the component and its styles. The typical directory structure for a component is as follows: ``` -|-- ComponentName -| |-- _styles.scss -| |-- ComponentName.jsx -| |-- ComponentName.tests.jsx -| |-- helpers.js -| |-- helpers.tests.js -| |-- index.js +└── ComponentName + ├── _styles.scss + ├── ComponentName.tsx + ├── index.ts ``` - `_styles.scss`: The component css styles -- `ComponentName.jsx`: The React component -- `ComponentName.tests.jsx`: The React component tests -- `helpers.js`: Helper functions used by the component -- `helpers.tests.js`: Tests for the component's helper functions -- `index.js`: Exports the React component +- `ComponentName.tsx`: The React component +- `index.ts`: Exports the React component - This file is helpful as it allows other components to import the component by it's directory name. Without this file the component name would have to be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). @@ -46,40 +52,34 @@ The component directory contains the React components rendered by pages. They are typically not connected to the redux state but receive props from their parent components to render data and handle user interactions. +### [context](./context) + +The context directory contains the React Context API pattern for various entities. +Only entities that are needed across the app has a global context. For example, +the [logged in user](./context/app.tsx) (`currentUser`) has multiple pages and components +where its information is pulled. + ### [interfaces](./interfaces) -Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet +Files in the interfaces directory are used to specify the Typescript interface for a reusable Fleet entity. This is designed to DRY up the code and increase re-usability. These -interfaces are imported into component files and implemented when defining the -component's PropTypes. +interfaces are imported in to component files and implemented when defining the +component's props. -### [fleet](./fleet) - -The default export of the `fleet` directory is the API client. More info can be -found at the [API client documentation page](./fleet/README.md). +**Additionally, local interfaces are used for props of local components.** ### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). -The Layout is rendered from the [router](./router/index.jsx) and are used to set up the general app UI (header, sidebar) and render child components. +The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general +app UI (header, sidebar) and render child components. The child components rendered by the layout are typically page components. ### [pages](./pages) Page components are React components typically rendered from the [router](./router). -These components are connected to redux state and are used to gather data from -redux and pass that data to child components (located in the [components -directory](./components). As -connected components, Pages are also used to dispatch actions. Actions -dispatched from Pages are intended to update redux state and oftentimes include -making a call to the Fleet API. - -### [redux](./redux) - -The redux directory holds all of the application's redux middleware, actions, -and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. -More information about the redux configuration can be found at the [Redux -Documentation page](./redux/README.md) +React Router passed props to these pages in case they are needed. Examples include +the `router`, `location`, and `params` objects. ### [router](./router) @@ -90,6 +90,10 @@ file which holds the application paths as string constants for reference throughout the app. These paths are typically referenced from the [App Constants](./app_constants) object. +### [services](./services) + +CRUD functions for all Fleet entities (e.g. `query`) that link directly to the Fleet API. + ### [styles](./styles) The styles directory contains the general app style setup and variables. It @@ -100,22 +104,221 @@ includes variables for the app color hex codes, fonts (families, weights and siz The templates directory contains the HTML file that renders the React application via including the `bundle.js` and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. -### [test](./test) - -The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. -More on test helpers, stubs, and request mocks [here](./test/README.md). - ### [utilities](./utilities) -The utilities directory contains re-usable functions for use throughout the +The utilities directory contains re-usable functions and constants for use throughout the application. The functions include helpers to convert an array of objects to CSV, debounce functions to prevent multiple form submissions, format API errors, etc. -## Forms +## Deprecated -For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). +These directories and files are still used (as of 9/14/21) but are being replaced by newer code: -## API Client +- [fleet](./fleet), now using [services](./services) +- [redux](./redux), now using [services](./services), local states, and various entities directly (e.g. React Router) +- [Form.jsx Higher Order Component](./components/forms/README.md), now creating forms with local states with React Hooks (i.e. `useState`) -For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). +To view the deprecated documentation, [click here](./README_deprecated.md). + +## Patterns + +### Typing +All Javascript and React files use Typescript, meaning the extensions are `.ts` and `.tsx`. Here are the guidelines on how we type at Fleet: + +- Use *[global entity interfaces](#interfaces)* when interfaces are used multiple times across the app +- Use *local interfaces* when typing entities limited to the specific page or component +- Local interfaces for page and component props + + ```typescript + // page + interface IPageProps { + prop1: string; + prop2: number; + ... + } + + // Note: Destructure props in page/component signature + const PageOrComponent = ({ + prop1, + prop2, + }: IPageProps) => { + + return ( + // ... + ); + }; + ``` + +- Local states +```typescript +const [item, setItem] = useState(""); +``` + +- Fetch function signatures (i.e. `react-query`) +```typescript +useQuery(params) +``` + +- Custom functions, including callbacks +```typescript +const functionWithTableName = (tableName: string): boolean => { + // do something +}; +``` + +### React Hooks (Functional Components) + +[Hooks](https://reactjs.org/docs/hooks-intro.html) are used to track state and use other features +of React. Hooks are only allowed in functional components, which are created like so: + +```typescript +import React, { useState, useEffect } from "React"; + +const PageOrComponent = (props) => { + const [item, setItem] = useState(""); + + // runs only on first mount (replaces componentDidMount) + useEffect(() => { + // do something + }, []); + + // runs only when `item` changes (replaces componentDidUpdate) + useEffect(() => { + // do something + }, [item]); + + return ( + // ... + ); +}; +``` + +**Note: Other hooks are available per [React's documentation](https://reactjs.org/docs/hooks-intro.html).** + +### React Context + +[React Context](https://reactjs.org/docs/context.html) is a store similar to Redux. It stores +data that is desired and allows for retrieval of that data in whatever component is in need. +View currently working contexts in the [context directory](./context). + +### Fleet API Calls + +**Deprecated:** + +Redux was used to make API calls, along with the [fleet](./fleet) directory. + +**Current:** + +The [services](./services) directory stores all API calls and is to be used in two ways: +- A direct `async/await` assignment +- Using `react-query` if requirements call for loading data right away or based on dependencies. + +Examples below: + +*Direct assignment* +```typescript +// page +import ... +import queryAPI from "services/entities/queries"; + +const PageOrComponent = (props) => { + const doSomething = async () => { + const response = await queryAPI.load(param); + // do something + }; + + return ( + // ... + ); +}; +``` + +*React Query* + +`react-query` ([docs here](https://react-query.tanstack.com/overview)) is a data-fetching library that +gives us the ability to fetch, cache, sync and update data with a myriad of options and properties. + +```typescript +import ... +import { useQuery, useMutation } from "react-query"; +import queryAPI from "services/entities/queries"; + +const PageOrComponent = (props) => { + // retrieve the query based on page/component load + // and dependencies for when to refetch + const { + isLoading, + data, + error, + ...otherProps, + } = useQuery( + "query", + () => queryAPI.load(param), + { + ...options + } + ); + + // `props` is a bucket of properties that can be used when + // updating data. for example, if you need to know whether + // a mutation is loading, there is a prop for that. + const { ...props } = useMutation((formData: IForm) => + queryAPI.create(formData) + ); + + return ( + // ... + ); +}; +``` + +### Page Routing + +**Deprecated:** + +Redux was used to manage redirecting to different pages of the app. + +**Current:** + +We use React Router directly to navigate between pages. For page components, +React Router (v3) supplies a `router` prop that can be easily accessed. +When needed, the `router` object contains a `push` function that redirects +a user to whatever page desired. For example: + +```typescript +// page +import PATHS from "router/paths"; + +interface IPageProps { + router: any; // no typing in react-router v3 +} + +const PageOrComponent = ({ + router, +}: IPageProps) => { + const doSomething = async () => { + router.push(PATHS.HOME); + }; + + return ( + // ... + ); +}; +``` + +### Other + +**Local states** + +Our first line of defense for state management is local states (i.e. `useState`). We +use local states to keep pages/components separate from one another and easy to +maintain. If states need to be passed to direct children, then prop-drilling should +suffice as long as we do not go more than two levels deep. Otherwise, if states need +to be used across multiple unrelated components or 3+ levels from a parent, +then the [app's context](#react-context) should be used. + +**File size** + +The recommend line limit per page/component is 500 lines. This is only a recommendation. +Larger files are to be split into multiple files if possible. \ No newline at end of file diff --git a/frontend/README_deprecated.md b/frontend/README_deprecated.md new file mode 100644 index 0000000000..79ac72db76 --- /dev/null +++ b/frontend/README_deprecated.md @@ -0,0 +1,128 @@ +**This documentation has many deprecated patterns. Please follow the current [README](./README.md).** + +# Fleet Front-End + +**Note: Redux is deprecated.** + +The Fleet front-end is a Single Page Application using React and Redux. + +## Running the Fleet web app + +For details instruction on building and serving the Fleet web application +consult the [Contributing documentation](../docs/03-Contributing/README.md). + +## Directory Structure + +Component directories in the Fleet front-end application encapsulate the entire +component, including files for the component, helper functions, styles, and tests. The +typical directory structure for a component is as follows: + +**Note: `.jsx` and `.js` is deprecated.** +``` +|-- ComponentName +| |-- _styles.scss +| |-- ComponentName.jsx +| |-- ComponentName.tests.jsx // deprecated +| |-- helpers.js +| |-- helpers.tests.js +| |-- index.js +``` + +- `_styles.scss`: The component css styles +- `ComponentName.jsx`: The React component +- `ComponentName.tests.jsx`: The React component tests `Deprecated` +- `helpers.js`: Helper functions used by the component +- `helpers.tests.js`: Tests for the component's helper functions +- `index.js`: Exports the React component + - This file is helpful as it allows other components to import the component + by it's directory name. Without this file the component name would have to + be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). + +### [app_constants](./app_constants) + +The app_constants directory exports the constants used in the app. Examples +include the app's URL paths, settings, and http statuses. When building features +that require constants, the constants should be added here for accessibility +throughout the application. + +### [components](./components) + +The component directory contains the React components rendered by pages. They +are typically not connected to the redux state but receive props from their +parent components to render data and handle user interactions. + +### [interfaces](./interfaces) + +**Note: PropTypes is deprecated.** + +Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet +entity. This is designed to DRY up the code and increase re-usability. These +interfaces are imported into component files and implemented when defining the +component's PropTypes. + +### [fleet](./fleet) `Deprecated` + +The default export of the `fleet` directory is the API client. More info can be +found at the [API client documentation page](./fleet/README.md). + +### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) + +The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). +The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general app UI (header, sidebar) and render child components. +The child components rendered by the layout are typically page components. + +### [pages](./pages) + +Page components are React components typically rendered from the [router](./router). +These components are connected to redux state and are used to gather data from +redux and pass that data to child components (located in the [components +directory](./components). As +connected components, Pages are also used to dispatch actions. Actions +dispatched from Pages are intended to update redux state and oftentimes include +making a call to the Fleet API. + +### [redux](./redux) `Deprecated` + +The redux directory holds all of the application's redux middleware, actions, +and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. +More information about the redux configuration can be found at the [Redux +Documentation page](./redux/README.md) + +### [router](./router) + +The router directory is where the react router lives. The router decides which +component will render at a given URL. Components rendered from the router are +typically located in the [pages directory](./pages). The router directory also holds a `paths` +file which holds the application paths as string constants for reference +throughout the app. These paths are typically referenced from the [App +Constants](./app_constants) object. + +### [styles](./styles) + +The styles directory contains the general app style setup and variables. It +includes variables for the app color hex codes, fonts (families, weights and sizes), and padding. + +### [templates](./templates) + +The templates directory contains the HTML file that renders the React application via including the `bundle.js` +and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. + +### [test](./test) `Deprecated` + +The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. +More on test helpers, stubs, and request mocks [here](./test/README.md). + +### [utilities](./utilities) + +The utilities directory contains re-usable functions for use throughout the +application. The functions include helpers to convert an array of objects to +CSV, debounce functions to prevent multiple form submissions, format API errors, +etc. + +## Forms + +For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). + +## API Client + +For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). diff --git a/frontend/components/App/App.tsx b/frontend/components/App/App.tsx index 6ba08768d8..0f5bfd600c 100644 --- a/frontend/components/App/App.tsx +++ b/frontend/components/App/App.tsx @@ -26,9 +26,15 @@ interface IRootState { const App = ({ children }: IAppProps) => { const dispatch = useDispatch(); - const { setCurrentUser, setConfig } = useContext(AppContext); const user = useSelector((state: IRootState) => state.auth.user); const queryClient = new QueryClient(); + const { + setCurrentUser, + setConfig, + currentUser, + isGlobalObserver, + isOnlyObserver, + } = useContext(AppContext); useDeepEffect(() => { // on page refresh @@ -41,10 +47,22 @@ const App = ({ children }: IAppProps) => { dispatch(getConfig()) .then((config: IConfig) => setConfig(config)) .catch(() => false); - dispatch(getEnrollSecret()).catch(() => false); } }, [user]); + useDeepEffect(() => { + const canGetEnrollSecret = + currentUser && + typeof isGlobalObserver !== "undefined" && + !isGlobalObserver && + typeof isOnlyObserver !== "undefined" && + !isOnlyObserver; + + if (canGetEnrollSecret) { + dispatch(getEnrollSecret()).catch(() => false); + } + }, [currentUser, isGlobalObserver, isOnlyObserver]); + const wrapperStyles = classnames("wrapper"); return ( diff --git a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx index e037aca8f3..caca7075c4 100644 --- a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx +++ b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx @@ -11,7 +11,6 @@ import enrollSecretInterface from "interfaces/enroll_secret"; import EnrollSecretTable from "components/config/EnrollSecretTable"; import InputField from "components/forms/fields/InputField"; import OrgLogoIcon from "components/icons/OrgLogoIcon"; -import Slider from "components/forms/fields/Slider"; import validate from "components/forms/admin/AppConfigForm/validate"; import IconToolTip from "components/IconToolTip"; import InfoBanner from "components/InfoBanner/InfoBanner"; diff --git a/frontend/components/forms/queries/QueryForm/QueryForm.tsx b/frontend/components/forms/queries/QueryForm/QueryForm.tsx index 03c8959dde..7bf58d4528 100644 --- a/frontend/components/forms/queries/QueryForm/QueryForm.tsx +++ b/frontend/components/forms/queries/QueryForm/QueryForm.tsx @@ -40,9 +40,9 @@ interface IQueryFormProps { } interface IRenderProps { - nameText: string; - descText: string; queryValue: string; + nameText?: string; + descText?: string; queryError?: any; queryOnChange?: any; name?: IFormField; @@ -251,6 +251,42 @@ const QueryForm = ({ ); + const renderCreateForTeamMaintainer = ({ + queryValue, + queryOnChange, + queryError, + }: IRenderProps) => ( + <> +
+

New query

+ {baseError &&
{baseError}
} + + {renderLiveQueryWarning()} +
+ +
+ + + ); + const renderForGlobalAdminOrMaintainer = ({ nameText, descText, @@ -268,14 +304,14 @@ const QueryForm = ({ name?.onChange(evt.target.value) } /> ) : ( -

New query

+

New query

)} {isEditMode && ( void; setConfig: (config: IConfig) => void; }; @@ -26,14 +26,14 @@ type InitialStateType = { const initialState = { currentUser: null, config: null, - isFreeTier: false, - isPremiumTier: false, - isGlobalAdmin: false, - isGlobalMaintainer: false, - isGlobalObserver: false, - isOnGlobalTeam: false, - isAnyTeamMaintainer: false, - isOnlyObserver: false, + isFreeTier: undefined, + isPremiumTier: undefined, + isGlobalAdmin: undefined, + isGlobalMaintainer: undefined, + isGlobalObserver: undefined, + isOnGlobalTeam: undefined, + isAnyTeamMaintainer: undefined, + isOnlyObserver: undefined, setCurrentUser: () => null, setConfig: () => null, }; diff --git a/frontend/fleet/endpoints.ts b/frontend/fleet/endpoints.ts index 13267fbd1c..6b58583227 100644 --- a/frontend/fleet/endpoints.ts +++ b/frontend/fleet/endpoints.ts @@ -41,7 +41,7 @@ export default { STATUS_LABEL_COUNTS: "/v1/fleet/host_summary", TARGETS: "/v1/fleet/targets", TEAM_SCHEDULE: (id: number): string => { - return `/v1/fleet/team/${id}/schedule`; + return `/v1/fleet/teams/${id}/schedule`; }, USERS: "/v1/fleet/users", USERS_ADMIN: "/v1/fleet/users/admin", diff --git a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx index 25c15dc3b8..499d103055 100644 --- a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx +++ b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx @@ -41,7 +41,7 @@ const CreateTeamModal = (props: ICreateTeamModalProps): JSX.Element => { return ( -
+ {

- -); +}: ITargetPillSelectorProps): JSX.Element => { + const displayText = () => { + switch (entity.display_text) { + case "All Hosts": + return "All hosts"; + case "All Linux": + return "Linux"; + default: + return entity.display_text; + } + }; + + return ( + + ); +}; const SelectTargets = ({ baseClass, @@ -228,7 +241,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsLoading) { return (
-

Select Targets

+

Select targets

@@ -239,7 +252,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsError) { return (
-

Select Targets

+

Select targets

@@ -264,7 +277,7 @@ const SelectTargets = ({ return (
-

Select Targets

+

Select targets

{allHostsLabels && allHostsLabels.length > 0 && diff --git a/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx b/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx index ff881bc279..84dc6cbb20 100644 --- a/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx +++ b/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx @@ -25,6 +25,7 @@ import paths from "router/paths"; import Button from "components/buttons/Button"; // @ts-ignore import Dropdown from "components/forms/fields/Dropdown"; +import IconToolTip from "components/IconToolTip"; import TableDataError from "components/TableDataError"; import ScheduleListWrapper from "./components/ScheduleListWrapper"; import ScheduleEditorModal from "./components/ScheduleEditorModal"; @@ -33,8 +34,10 @@ import RemoveScheduledQueryModal from "./components/RemoveScheduledQueryModal"; const baseClass = "manage-schedule-page"; const renderTable = ( - onRemoveScheduledQueryClick: React.MouseEventHandler, - onEditScheduledQueryClick: React.MouseEventHandler, + onRemoveScheduledQueryClick: (selectIds: number[]) => void, + onEditScheduledQueryClick: ( + selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery + ) => void, allScheduledQueriesList: IGlobalScheduledQuery[] | ITeamScheduledQuery[], allScheduledQueriesError: { name: string; reason: string }[], toggleScheduleEditorModal: () => void, @@ -55,6 +58,26 @@ const renderTable = ( ); }; +const renderAllTeamsTable = ( + teamId: number, + allTeamsScheduledQueriesList: IGlobalScheduledQuery[], + allTeamsScheduledQueriesError: { name: string; reason: string }[] +): JSX.Element => { + if (Object.keys(allTeamsScheduledQueriesError).length > 0) { + return ; + } + + return ( +
+ +
+ ); +}; + interface ITeamSchedulesPageProps { params: { team_id: string; @@ -139,6 +162,18 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { const allScheduledQueriesList = Object.values(allScheduledQueries.data); const allScheduledQueriesError = allScheduledQueries.errors; + const allTeamsScheduledQueries = useSelector((state: IRootState) => { + return state.entities.global_scheduled_queries; + }); + + const allTeamsScheduledQueriesList = Object.values( + allTeamsScheduledQueries.data + ); + const allTeamsScheduledQueriesError = allTeamsScheduledQueries.errors; + + const inheritedQueryOrQueries = + allTeamsScheduledQueriesList.length === 1 ? "query" : "queries"; + const allTeams = useSelector((state: IRootState) => state.entities.teams); const allTeamsList = Object.values(allTeams.data); @@ -148,7 +183,7 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { const teamOptions: ITeamOptions[] = [ { disabled: false, - label: "Global", + label: "All teams", value: "global", }, ]; @@ -163,6 +198,9 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { return teamOptions; }; + const [showInheritedQueries, setShowInheritedQueries] = useState( + false + ); const [showScheduleEditorModal, setShowScheduleEditorModal] = useState(false); const [ showRemoveScheduledQueryModal, @@ -175,6 +213,10 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { IGlobalScheduledQuery | ITeamScheduledQuery >(); + const toggleInheritedQueries = () => { + setShowInheritedQueries(!showInheritedQueries); + }; + const toggleScheduleEditorModal = useCallback(() => { setSelectedScheduledQuery(undefined); // create modal renders setShowScheduleEditorModal(!showScheduleEditorModal); @@ -184,12 +226,16 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { setShowRemoveScheduledQueryModal(!showRemoveScheduledQueryModal); }, [showRemoveScheduledQueryModal, setShowRemoveScheduledQueryModal]); - const onRemoveScheduledQueryClick = (selectedTableQueryIds: any): void => { + const onRemoveScheduledQueryClick = ( + selectedTableQueryIds: number[] + ): void => { toggleRemoveScheduledQueryModal(); setSelectedQueryIds(selectedTableQueryIds); }; - const onEditScheduledQueryClick = (selectedQuery: any): void => { + const onEditScheduledQueryClick = ( + selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery + ): void => { toggleScheduleEditorModal(); setSelectedScheduledQuery(selectedQuery); // edit modal renders }; @@ -379,6 +425,39 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { teamId )}
+ {/* must use ternary for NaN */} + {teamId && allTeamsScheduledQueriesList.length > 0 ? ( + <> + + + +
+

Queries from the “All teams”
schedule run on this team’s hosts.

\ + " + } + /> +
+ + ) : null} + {showInheritedQueries && + renderAllTeamsTable( + teamId, + allTeamsScheduledQueriesList, + allTeamsScheduledQueriesError + )} {showScheduleEditorModal && ( void; + onEditScheduledQueryClick?: ( + selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery + ) => void; allScheduledQueriesList: IGlobalScheduledQuery[] | ITeamScheduledQuery[]; - toggleScheduleEditorModal: () => void; + toggleScheduleEditorModal?: () => void; teamId: number; + inheritedQueries?: boolean; } interface IRootState { entities: { @@ -47,6 +54,7 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { toggleScheduleEditorModal, onEditScheduledQueryClick, teamId, + inheritedQueries, } = props; const dispatch = useDispatch(); const { MANAGE_PACKS } = paths; @@ -92,10 +100,14 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { ): void => { switch (action) { case "edit": - onEditScheduledQueryClick(global_scheduled_query); + if (onEditScheduledQueryClick) { + onEditScheduledQueryClick(global_scheduled_query); + } break; default: - onRemoveScheduledQueryClick([global_scheduled_query.id]); + if (onRemoveScheduledQueryClick) { + onRemoveScheduledQueryClick([global_scheduled_query.id]); + } break; } }; @@ -123,6 +135,33 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { [dispatch] ); + const loadingInheritedQueriesTableData = useSelector((state: IRootState) => { + return state.entities.global_scheduled_queries.isLoading; + }); + + if (inheritedQueries) { + const inheritedQueriesTableHeaders = generateInheritedQueriesTableHeaders(); + + return ( +
+ +
+ ); + } + return (
{ + return [ + { + title: "Query", + Header: "Query", + disableSortBy: true, + accessor: "query_name", + Cell: (cellProps: ICellProps): JSX.Element => ( + + ), + }, + { + title: "Frequency", + Header: "Frequency", + disableSortBy: true, + accessor: "interval", + Cell: (cellProps: ICellProps): JSX.Element => ( + + ), + }, + ]; +}; + const generateActionDropdownOptions = (): IDropdownOption[] => { const dropdownOptions = [ { @@ -162,4 +185,8 @@ const generateDataSet = ( return [...enhanceAllScheduledQueryData(all_scheduled_queries, teamId)]; }; -export { generateTableHeaders, generateDataSet }; +export { + generateInheritedQueriesTableHeaders, + generateTableHeaders, + generateDataSet, +}; diff --git a/frontend/styles/global/_global.scss b/frontend/styles/global/_global.scss index 586d4bb247..1e335648f7 100644 --- a/frontend/styles/global/_global.scss +++ b/frontend/styles/global/_global.scss @@ -55,11 +55,12 @@ a { border-radius: 3px; background-color: $core-white; border: solid 1px $core-white; - min-width: 910px; + min-width: 798px; } .has-sidebar { display: flex; + flex-grow: 1; & > *:first-child { flex-grow: 1; @@ -121,4 +122,4 @@ hr { margin-bottom: $pad-xlarge; border: none; border-bottom: 1px solid $ui-fleet-blue-15; -} \ No newline at end of file +} diff --git a/frontend/test/mocks/team_scheduled_query_mocks.js b/frontend/test/mocks/team_scheduled_query_mocks.js index 30a5533a34..bfbe3e7d1a 100644 --- a/frontend/test/mocks/team_scheduled_query_mocks.js +++ b/frontend/test/mocks/team_scheduled_query_mocks.js @@ -16,7 +16,7 @@ export default { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${params.team_id}/schedule`, + endpoint: `/api/v1/fleet/teams/${params.team_id}/schedule`, method: "post", params, response: { scheduled: teamScheduledQueryStub }, @@ -28,7 +28,7 @@ export default { valid: (bearerToken, teamScheduledQuery) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "delete", response: {}, }); @@ -38,7 +38,7 @@ export default { valid: (bearerToken, teamID) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamID}/schedule`, + endpoint: `/api/v1/fleet/teams/${teamID}/schedule`, method: "get", response: { scheduled: [teamScheduledQueryStub] }, }); @@ -48,7 +48,7 @@ export default { valid: (bearerToken, teamScheduledQuery, params) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "patch", params, response: { scheduled: { ...teamScheduledQuery, ...params } }, diff --git a/go.mod b/go.mod index b7b7271608..6b0d2d7ea9 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/gomodule/redigo v1.8.5 github.com/google/go-cmp v0.5.6 github.com/google/go-github/v37 v37.0.0 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/goreleaser/nfpm/v2 v2.2.2 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index dd625a9fa6..c045b59475 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,8 @@ github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/handbook/README.md b/handbook/README.md index 3ea51a385e..d33d6ed6f6 100644 --- a/handbook/README.md +++ b/handbook/README.md @@ -98,7 +98,7 @@ If the action fails, please complete the following steps: ##### Browser compatibility checking -A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/1-Using-Fleet/12-Supported-browsers.md). +A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/01-Using-Fleet/12-Supported-browsers.md). - We use [BrowserStack](https://www.browserstack.com/users/sign_in) (logins can be found in [1Password](https://start.1password.com/open/i?a=N3F7LHAKQ5G3JPFPX234EC4ZDQ&v=3ycqkai6naxhqsylmsos6vairu&i=nwnxrrbpcwkuzaazh3rywzoh6e&h=fleetdevicemanagement.1password.com)) for our cross-browser checks. - Check for issues against the latest version of Google Chrome (macOS). We use this as our baseline for quality assurance. diff --git a/handbook/growth.md b/handbook/growth.md new file mode 100644 index 0000000000..694a6b5e5e --- /dev/null +++ b/handbook/growth.md @@ -0,0 +1,8 @@ +# Growth handbook + +Welcome to the Fleet growth handbook. + +TODO + + + diff --git a/handbook/manual-qa.md b/handbook/manual-qa.md index 6af722b196..3f84eb2412 100644 --- a/handbook/manual-qa.md +++ b/handbook/manual-qa.md @@ -2,63 +2,31 @@ This living document outlines the manual quality assurance process conducted to ensure each release of Fleet meets organization standards. -All steps should be conducted during each QA pass. +All steps should be conducted during each QA pass. All steps are possible with `fleetctrl preview`. As new features are added to Fleet, new steps and flows will be added. ## Collecting bugs -The goal of manual QA is to catch unexpected behavior prior to release. +The goal of manual QA is to catch unexpected behavior prior to release. All Manual QA steps should be possible using `fleetctl preview`. Please refer to [docs/03-Contributing/02-Testing.md](https://github.com/fleetdm/fleet/blob/main/docs/03-Contributing/02-Testing.md) for flows that cannot be completed using `fleetctl preview`. Please start the manual QA process by creating a blank GitHub issue. As you complete each of the flows, record a list of the bugs you encounter in this new issue. Each item in this list should contain one sentence describing the bug and a screenshot if the item is a frontend bug. ## Fleet UI -### Clear your local MySQL database - -Before you fire up your local Fleet server, wipe your local MySQL database by running the following command: - -``` -docker volume rm fleet_mysql-persistent-volume -``` - -If you receive an error that says "No such volume," double check that the MySQL volume doesn't have a different name by running this command: - -``` -docker volume ls -``` - -### Start your development server - -Next, fire up your local Fleet server. Check out [this Loom video](https://www.loom.com/share/e7439f058eb44c45af872abe8f8de4a1) for instructions on starting up your local development environment. +For all following flows, please refer to the [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) to ensure that actions are limited to the appropriate user type. Any users with access beyond what this document lists as availale should be considered a bug and reported for either documentation updates or investigation. ### Set up flow -Successfully set up Fleet. +Successfully set up `fleetctl preview` using the preview steps outlined [here](https://fleetdm.com/get-started) ### Login and logout flow Successfully logout and then login to your local Fleet. -### Enroll host flow - -Enroll your local machine to Fleet. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/orbit#orbit-osquery) for instructions on generating and installing an Orbit package. - -### Host page - -To populate the Fleet UI with more than just one host you'll need to use the [fleetdm/osquery-perf tool](https://github.com/fleetdm/osquery-perf/tree/629a7efb6097f9108f706ccd45828793ff73cf9c). - -First, clone the fleetdm/osquery perf repo and then run the following commands from the top level of the cloned directory: - -``` -go run agent.go --host_count 200 --enroll_secret -``` - -After about 10 seconds, the Fleet UI should be populated with 200 simulated hosts. - ### Host details page -Select a host from the "Hosts" table as a global user with the Maintainer role. +Select a host from the "Hosts" table as a global user with the Maintainer role. You may create a user with a fake email for this purpose. You should be able to see and select the "Delete" button on this host's **Host details** page. @@ -70,7 +38,9 @@ You should be able to see and select the "Query" button on this host's **Host de Create a new label by selecting "Add a new label" on the Hosts page. Make sure it correctly filters the host on the hosts page. -Edit this label and then delete this label. +Edit this label. Confirm users can only edit the "Name" and "Description" fields for a label. Users cannot edit the "Query" field because label queries are immutable. + +Delete this label. ### Query flow @@ -86,88 +56,12 @@ Edit this query and then delete this query. `Flow is covered by e2e testing` -Create a new pack. +Create a new pack (under Schedule/advanced). Add a query as a saved query to the pack. Remove this query. Delete the pack. -### Organization settings flow - -As an admin user, select the "Settings" tab in the top navigation and then select "Organization settings". - -Follow [the instructions outlined in the Testing documentation](../docs/3-Contributing/2-Testing.md#email) to set up a local SMTP server. - -Successfully edit your organization's name in Fleet. - -### Manage users flow - -Invite a new user. To be able to invite users, you must have your local SMTP server configured. Instructions for setting up a local SMTP server are outlined in [the Testing documentation](../docs/3-Contributing/2-Testing.md#email) - -Logout of your current admin user and accept the invitation for the newly invited user. With your local SMTP server configured, head to https://localhost:8025 to view and select the invitation link. - -### Agent options flow - -Head to the global agent options page and set the `distributed_iterval` field to `5`. - -Refresh the page to confirm that the agent options have been updated. ### My account flow -Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. +Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. Please do this with an extra user created for this purpose to maintain accessibility of `fleetctl preview` admin user. -## `fleetctl` CLI - -### Set up flow - -Successfully set up Fleet by running the `fleetctl setup` command. - -You may have to wipe your local MySQL database in order to successfully set up Fleet. Check out the [Clear your local MySQL database](#clear-your-local-mysql-database) section of this document for instructions. - -### Login and logout flow - -Successfully login by running the `fleetctl login` command. - -Successfully logout by running the `fleetctl logout` command. Then, log in again. - -### Hosts - -Run the `fleetctl get hosts` command. - -You should see your local machine returned. If your host isn't showing up, you may have to reenroll your local machine. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/fleet/blob/main/orbit/README.md) for instructions on generating and installing an Orbit package. - -### Query flow - -Apply the standard query library by running the following command: - -`fleetctl apply -f docs/1-Using-Fleet/standard-query-library/standard-query-library.yml` - -Make sure all queries were successfully added by running the following command: - -`fleetctl get queries` - -Run the "Get the version of the resident operating system" query against your local machine by running the following command: - -`fleetctl query --hosts --query-name "Get the version of the resident operating system"` - -### Pack flow - -Apply a pack by running the following commands: - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml` - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml` - -Make sure the pack was successfully added by running the following command: - -`fleetctl get packs` - -### Organization settings flow - -Apply organization settings by running the following command: - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml` - -### Manage users flow - -Create a new user by running the `fleetctl user create` command. - -Logout of your current user and log in with the newly created user. diff --git a/handbook/release-process.md b/handbook/release-process.md index 52b5c6aaec..9e7d01f262 100644 --- a/handbook/release-process.md +++ b/handbook/release-process.md @@ -28,7 +28,7 @@ Check out the [Fleet 4.1.0 blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa **More improvements** - Includes each additional feature's name, availability (Free v. Premium), and 1-2 sentences that answer the 'why should the user care?' questions. -**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/8-Updating-Fleet.md +**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/08-Updating-Fleet.md ### Manual QA @@ -38,4 +38,4 @@ Documentation on conducting the manual QA pass can be found [here](./manual-qa.m ## Release day -Documentation on completing the release process can be found [here](../docs/3-Contributing/5-Releasing-Fleet.md). +Documentation on completing the release process can be found [here](../docs/03-Contributing/05-Releasing-Fleet.md). diff --git a/handbook/support-process.md b/handbook/support-process.md index cbacf8587c..58d09e964e 100644 --- a/handbook/support-process.md +++ b/handbook/support-process.md @@ -90,6 +90,6 @@ There are four sources that the individual on-call should monitor for activity: There are several locations in Fleet's public and internal documentation that can be helpful when answering questions raised by the community: -1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/1-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/2-Deploying/FAQ.md), and [Contributing FAQ](../docs/3-Contributing/FAQ.md). +1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/01-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/02-Deploying/FAQ.md), and [Contributing FAQ](../docs/03-Contributing/FAQ.md). 2. The [Internal FAQ](https://docs.google.com/document/d/1I6pJ3vz0EE-qE13VmpE2G3gd5zA1m3bb_u8Q2G3Gmp0/edit#heading=h.ltavvjy511qv) document. diff --git a/handbook/why-fleet.md b/handbook/why-fleet.md deleted file mode 100644 index 54ad90368f..0000000000 --- a/handbook/why-fleet.md +++ /dev/null @@ -1,43 +0,0 @@ -## Why Fleet? - -We believe device management should be safe, easy, and transparent to end users. - -### The premier osquery fleet manager. -Fleet makes it easier to query and track your servers, containers, and laptops. It extends osquery to answer questions about multiple devices at the same time and provides log streams that enable automated threat detection. You can run it on your own hardware or deploy it in any cloud. - -#### Realtime visibility -- Share with any team (ops, security, helpdesk, etc) -- Unmetered access to user seats -- Intuitive user interface -- Verify remediations were actually successful -- Make compliance goals highly visible -- Track progress towards compliance goals together -- Auto-ticket any compliance regressions -- Reliable, efficient source of truth - -#### Get flexible -- Receive data from any device -- Send data anywhere it needs to go -- Express and track custom compliance goals -- Integrate existing business processes -- Play nice with internal tools and requirements -- Self-managed (cloud first, but can still be within your VPC) -- Complete control (no need to rely on 3rd party vendors, reduced risk of getting hacked) -- Deploy anywhere - -#### Developer productivity -- Programmable (REST API, ±gRPC) -- Command line tool (`fleetctl`) -- "Infrastructure as code"-ready (YAML) -- All UI functionality accessible via APIs -- Focus on value, not boilerplate -- Skip reinventing the wheel -- Ad hoc audits (live queries answer in seconds) -- 100% source code available - -#### Unlock efficiencies -- Designed for enterprise (from day 1) -- Built by osquery creators (Zach Wasserman + Mike Arpaia) -- Battle-tested in production (3+ years) -- Learn from peers' mistakes & best practices -- Widespread adoption in F1000 diff --git a/orbit/README.md b/orbit/README.md index 18cd740e52..3b3d1da3be 100644 --- a/orbit/README.md +++ b/orbit/README.md @@ -192,7 +192,7 @@ Yes! Orbit is licensed under an MIT license and all uses are encouraged. ### How does orbit update osquery? And how do the stable and edge channels get triggered to update osquery on a self hosted Fleet instance? -Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/4-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). +Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/04-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). ## Community diff --git a/package.json b/package.json index cfa86f317f..340b3a1b8b 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,7 @@ "babel-jest": "^23.4.2", "babel-loader": "^8.2.1", "css-loader": "0.28.11", - "cypress": "^6.9.1", + "cypress": "8.4.0", "cypress-wait-until": "^1.7.1", "enzyme": "^3.11.0", "enzyme-adapter-react-16": "^1.15.6", diff --git a/server/authz/authz.go b/server/authz/authz.go index 91e8895e4b..35d02b96ac 100644 --- a/server/authz/authz.go +++ b/server/authz/authz.go @@ -119,6 +119,41 @@ func (a *Authorizer) Authorize(ctx context.Context, object, action interface{}) return nil } +func (a *Authorizer) TeamAuthorize(ctx context.Context, teamID uint, action string) error { + subject := UserFromContext(ctx) + if subject == nil { + return ForbiddenWithInternal("nil subject always forbidden", subject, nil, action) + } + + // global admins and maintainers are authorized to work with teams + if subject.GlobalRole != nil { + switch *subject.GlobalRole { + case fleet.RoleAdmin, fleet.RoleMaintainer: + return nil + case fleet.RoleObserver: + if action == fleet.ActionRead { + return nil + } + } + } + + for _, team := range subject.Teams { + if teamID == team.ID { + switch action { + case fleet.ActionWrite: + if team.Role == fleet.RoleMaintainer { + return nil + } + return ForbiddenWithInternal("team observer cannot write", subject, nil, action) + default: + return nil + } + } + } + + return ForbiddenWithInternal("not a member of the team", subject, nil, action) +} + // AuthzTyper is the interface that may be implemented to get a `type` // property added during marshaling for authorization. Any struct that will be // used as a subject or object in authorization should implement this interface. diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 9ff87d8780..ff9a5ecf8a 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -15,6 +15,7 @@ list := "list" write := "write" write_role := "write_role" run := "run" +run_new := "run_new" # Roles admin := "admin" @@ -265,6 +266,16 @@ allow { subject.global_role == maintainer action = run } +allow { + object.type == "query" + subject.global_role == admin + action = run_new +} +allow { + object.type == "query" + subject.global_role == maintainer + action = run_new +} # Team maintainer running a non-observers_can_run query must have the targets # filtered to only teams that they maintain allow { @@ -274,6 +285,22 @@ allow { action == run } +# Team maintainer can run a new query +allow { + object.type == "query" + # If role is maintainer on any team + team_role(subject, subject.teams[_].id) == maintainer + action == run_new +} + +# Team admin can run a new query +allow { + object.type == "query" + # If role is maintainer on any team + team_role(subject, subject.teams[_].id) == admin + action == run_new +} + # (Team) observers can run only if observers_can_run allow { object.type == "query" @@ -342,11 +369,41 @@ allow { } allow { + is_null(object.team_id) object.type == "policy" subject.global_role == maintainer action == [read, write][_] } +allow { + object.type == "policy" + subject.global_role == maintainer + action == [read][_] +} + +# Global Observer users can read policies +allow { + object.type == "policy" + subject.global_role == observer + action == [read][_] +} + +# Team Maintainers can read and write policies +allow { + not is_null(object.team_id) + object.type == "policy" + team_role(subject, subject.teams[_].id) == maintainer + action == [read, write][_] +} + +# Team Observer can read policies +allow { + not is_null(object.team_id) + object.type == "policy" + team_role(subject, subject.teams[_].id) == observer + action == [read][_] +} + ## # Software ## diff --git a/server/config/config.go b/server/config/config.go index e687581014..d81baa58ba 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -37,13 +37,15 @@ type MysqlConfig struct { // RedisConfig defines configs related to Redis type RedisConfig struct { - Address string - Password string - Database int - UseTLS bool `yaml:"use_tls"` - DuplicateResults bool `yaml:"duplicate_results"` - ConnectTimeout time.Duration `yaml:"connect_timeout"` - KeepAlive time.Duration `yaml:"keep_alive"` + Address string + Password string + Database int + UseTLS bool `yaml:"use_tls"` + DuplicateResults bool `yaml:"duplicate_results"` + ConnectTimeout time.Duration `yaml:"connect_timeout"` + KeepAlive time.Duration `yaml:"keep_alive"` + ConnectRetryAttempts int `yaml:"connect_retry_attempts"` + ClusterFollowRedirections bool `yaml:"cluster_follow_redirections"` } const ( @@ -89,10 +91,12 @@ type OsqueryConfig struct { StatusLogPlugin string `yaml:"status_log_plugin"` ResultLogPlugin string `yaml:"result_log_plugin"` LabelUpdateInterval time.Duration `yaml:"label_update_interval"` + PolicyUpdateInterval time.Duration `yaml:"policy_update_interval"` DetailUpdateInterval time.Duration `yaml:"detail_update_interval"` StatusLogFile string `yaml:"status_log_file"` ResultLogFile string `yaml:"result_log_file"` EnableLogRotation bool `yaml:"enable_log_rotation"` + MaxJitterPercent int `yaml:"max_jitter_percent"` } // LoggingConfig defines configs related to logging @@ -243,6 +247,8 @@ func (man Manager) addConfigs() { man.addConfigBool("redis.duplicate_results", false, "Duplicate Live Query results to another Redis channel") man.addConfigDuration("redis.connect_timeout", 5*time.Second, "Timeout at connection time") man.addConfigDuration("redis.keep_alive", 10*time.Second, "Interval between keep alive probes") + man.addConfigInt("redis.connect_retry_attempts", 0, "Number of attempts to retry a failed connection") + man.addConfigBool("redis.cluster_follow_redirections", false, "Automatically follow Redis Cluster redirections") // Server man.addConfigString("server.address", "0.0.0.0:8080", @@ -294,6 +300,8 @@ func (man Manager) addConfigs() { "Log plugin to use for result logs") man.addConfigDuration("osquery.label_update_interval", 1*time.Hour, "Interval to update host label membership (i.e. 1h)") + man.addConfigDuration("osquery.policy_update_interval", 1*time.Hour, + "Interval to update host policy membership (i.e. 1h)") man.addConfigDuration("osquery.detail_update_interval", 1*time.Hour, "Interval to update host details (i.e. 1h)") man.addConfigString("osquery.status_log_file", "", @@ -302,6 +310,8 @@ func (man Manager) addConfigs() { "(DEPRECATED: Use filesystem.result_log_file) Path for osqueryd result logs") man.addConfigBool("osquery.enable_log_rotation", false, "(DEPRECATED: Use filesystem.enable_log_rotation) Enable automatic rotation for osquery log files") + man.addConfigInt("osquery.max_jitter_percent", 10, + "Maximum percentage of the interval to add as jitter") // Logging man.addConfigBool("logging.debug", false, @@ -417,13 +427,15 @@ func (man Manager) LoadConfig() FleetConfig { Mysql: loadMysqlConfig("mysql"), MysqlReadReplica: loadMysqlConfig("mysql_read_replica"), Redis: RedisConfig{ - Address: man.getConfigString("redis.address"), - Password: man.getConfigString("redis.password"), - Database: man.getConfigInt("redis.database"), - UseTLS: man.getConfigBool("redis.use_tls"), - DuplicateResults: man.getConfigBool("redis.duplicate_results"), - ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), - KeepAlive: man.getConfigDuration("redis.keep_alive"), + Address: man.getConfigString("redis.address"), + Password: man.getConfigString("redis.password"), + Database: man.getConfigInt("redis.database"), + UseTLS: man.getConfigBool("redis.use_tls"), + DuplicateResults: man.getConfigBool("redis.duplicate_results"), + ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), + KeepAlive: man.getConfigDuration("redis.keep_alive"), + ConnectRetryAttempts: man.getConfigInt("redis.connect_retry_attempts"), + ClusterFollowRedirections: man.getConfigBool("redis.cluster_follow_redirections"), }, Server: ServerConfig{ Address: man.getConfigString("server.address"), @@ -455,8 +467,10 @@ func (man Manager) LoadConfig() FleetConfig { StatusLogFile: man.getConfigString("osquery.status_log_file"), ResultLogFile: man.getConfigString("osquery.result_log_file"), LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), + PolicyUpdateInterval: man.getConfigDuration("osquery.policy_update_interval"), DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), EnableLogRotation: man.getConfigBool("osquery.enable_log_rotation"), + MaxJitterPercent: man.getConfigInt("osquery.max_jitter_percent"), }, Logging: LoggingConfig{ Debug: man.getConfigBool("logging.debug"), @@ -742,7 +756,9 @@ func TestConfig() FleetConfig { StatusLogPlugin: "filesystem", ResultLogPlugin: "filesystem", LabelUpdateInterval: 1 * time.Hour, + PolicyUpdateInterval: 1 * time.Hour, DetailUpdateInterval: 1 * time.Hour, + MaxJitterPercent: 0, }, Logging: LoggingConfig{ Debug: true, diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index 5dbdc7b6fc..af97ca730f 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -10,10 +10,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestActivityUsernameChange(t *testing.T) { +func TestActivity(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"UsernameChange", testActivityUsernameChange}, + {"New", testActivityNew}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testActivityUsernameChange(t *testing.T, ds *Datastore) { u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -52,10 +67,7 @@ func TestActivityUsernameChange(t *testing.T) { assert.Nil(t, activities[0].ActorGravatar) } -func TestNewActivity(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testActivityNew(t *testing.T, ds *Datastore) { u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -86,4 +98,12 @@ func TestNewActivity(t *testing.T) { assert.Len(t, activities, 1) assert.Equal(t, "fullname", activities[0].ActorFullName) assert.Equal(t, "test2", activities[0].Type) + + opt = fleet.ListOptions{ + Page: 0, + PerPage: 10, + } + activities, err = ds.ListActivities(context.Background(), opt) + require.NoError(t, err) + assert.Len(t, activities, 2) } diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index 58b0e3906d..8959912dad 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -14,10 +14,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestOrgInfo(t *testing.T) { +func TestAppConfig(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"OrgInfo", testAppConfigOrgInfo}, + {"AdditionalQueries", testAppConfigAdditionalQueries}, + {"EnrollSecrets", testAppConfigEnrollSecrets}, + {"EnrollSecretsCaseSensitive", testAppConfigEnrollSecretsCaseSensitive}, + {"EnrollSecretRoundtrip", testAppConfigEnrollSecretRoundtrip}, + {"EnrollSecretUniqueness", testAppConfigEnrollSecretUniqueness}, + {"Defaults", testAppConfigDefaults}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.fn(t, ds) + }) + } +} + +func testAppConfigOrgInfo(t *testing.T, ds *Datastore) { info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ OrgName: "Test", @@ -94,10 +113,7 @@ func TestOrgInfo(t *testing.T) { assert.False(t, verify.SSOEnabled) } -func TestAdditionalQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testAppConfigAdditionalQueries(t *testing.T, ds *Datastore) { additional := ptr.RawMessage(json.RawMessage("not valid json")) info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ @@ -123,9 +139,8 @@ func TestAdditionalQueries(t *testing.T) { assert.JSONEq(t, `{"foo":"bar"}`, string(rawJson)) } -func TestEnrollSecrets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecrets(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -179,9 +194,8 @@ func TestEnrollSecrets(t *testing.T) { assert.Equal(t, (*uint)(nil), secret.TeamID) } -func TestEnrollSecretsCaseSensitive(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretsCaseSensitive(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) err := ds.ApplyEnrollSecrets( context.Background(), @@ -198,9 +212,8 @@ func TestEnrollSecretsCaseSensitive(t *testing.T) { assert.Error(t, err, "enroll secret with different case should not verify") } -func TestEnrollSecretRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretRoundtrip(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -240,9 +253,8 @@ func TestEnrollSecretRoundtrip(t *testing.T) { } -func TestEnrollSecretUniqueness(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretUniqueness(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -258,10 +270,7 @@ func TestEnrollSecretUniqueness(t *testing.T) { require.Error(t, err) } -func TestAppConfigDefaults(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testAppConfigDefaults(t *testing.T, ds *Datastore) { insertAppConfigQuery := `INSERT INTO app_config_json(json_value) VALUES(?) ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)` _, err := ds.writer.Exec(insertAppConfigQuery, `{}`) require.NoError(t, err) diff --git a/server/datastore/mysql/campaigns_test.go b/server/datastore/mysql/campaigns_test.go index 245795981f..87d9316b6a 100644 --- a/server/datastore/mysql/campaigns_test.go +++ b/server/datastore/mysql/campaigns_test.go @@ -12,24 +12,30 @@ import ( "github.com/stretchr/testify/require" ) -func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { - targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) - require.Nil(t, err) - assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) - assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) - assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) +func TestCampaigns(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"DistributedQuery", testCampaignsDistributedQuery}, + {"CleanupDistributedQuery", testCampaignsCleanupDistributedQuery}, + {"SaveDistributedQuery", testCampaignsSaveDistributedQuery}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + + c.fn(t, ds) + }) + } } -func TestDistributedQueryCampaign(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) - mockClock := clock.NewMockClock() - query := test.NewQuery(t, ds, "test", "select * from time", user.ID, false) - campaign := test.NewCampaign(t, ds, query.ID, fleet.QueryRunning, mockClock.Now()) { @@ -71,13 +77,9 @@ func TestDistributedQueryCampaign(t *testing.T) { test.AddHostToCampaign(t, ds, campaign.ID, h3.ID) checkTargets(t, ds, campaign.ID, fleet.HostTargets{HostIDs: []uint{h1.ID, h2.ID, h3.ID}, LabelIDs: []uint{l1.ID, l2.ID}}) - } -func TestCleanupDistributedQueryCampaigns(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsCleanupDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -149,13 +151,9 @@ func TestCleanupDistributedQueryCampaigns(t *testing.T) { assert.Equal(t, c2.QueryID, retrieved.QueryID) assert.Equal(t, fleet.QueryComplete, retrieved.Status) } - } -func TestSaveDistributedQueryCampaign(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsSaveDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, t.Name(), t.Name()+"zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -174,3 +172,11 @@ func TestSaveDistributedQueryCampaign(t *testing.T) { require.NoError(t, err) require.Equal(t, fleet.QueryComplete, gotC.Status) } + +func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { + targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) + require.Nil(t, err) + assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) + assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) + assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) +} diff --git a/server/datastore/mysql/carves_test.go b/server/datastore/mysql/carves_test.go index 22f84f772d..2b50ee4298 100644 --- a/server/datastore/mysql/carves_test.go +++ b/server/datastore/mysql/carves_test.go @@ -14,10 +14,28 @@ import ( var mockCreatedAt = time.Now().UTC().Truncate(time.Second) -func TestCarveMetadata(t *testing.T) { +func TestCarves(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Metadata", testCarvesMetadata}, + {"Blocks", testCarvesBlocks}, + {"Cleanup", testCarvesCleanup}, + {"List", testCarvesList}, + {"Update", testCarvesUpdate}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testCarvesMetadata(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -75,10 +93,7 @@ func TestCarveMetadata(t *testing.T) { assert.Equal(t, expectedCarve, carve) } -func TestCarveBlocks(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesBlocks(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -116,13 +131,9 @@ func TestCarveBlocks(t *testing.T) { require.NoError(t, err, "get block %d %v", i, expectedBlocks[i]) assert.Equal(t, expectedBlocks[i], data) } - } -func TestCarveCleanupCarves(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesCleanup(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -174,10 +185,7 @@ func TestCarveCleanupCarves(t *testing.T) { assert.True(t, carve.Expired) } -func TestCarveListCarves(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesList(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -235,10 +243,7 @@ func TestCarveListCarves(t *testing.T) { assert.Len(t, carves, 2) } -func TestCarveUpdateCarve(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesUpdate(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) actualCount := int64(10) diff --git a/server/datastore/mysql/delete.go b/server/datastore/mysql/delete.go index 384327b8b0..f310769469 100644 --- a/server/datastore/mysql/delete.go +++ b/server/datastore/mysql/delete.go @@ -10,41 +10,41 @@ import ( // deleteEntity deletes an entity with the given id from the given DB table, // returning a notFound error if appropriate. -func (d *Datastore) deleteEntity(ctx context.Context, dbTable string, id uint) error { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable) +func (d *Datastore) deleteEntity(ctx context.Context, dbTable entity, id uint) error { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable.name) result, err := d.writer.ExecContext(ctx, deleteStmt, id) if err != nil { return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable).WithID(id) + return notFound(dbTable.name).WithID(id) } return nil } // deleteEntityByName deletes an entity with the given name from the given DB // table, returning a notFound error if appropriate. -func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable string, name string) error { - deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable) +func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable entity, name string) error { + deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable.name) result, err := d.writer.ExecContext(ctx, deleteStmt, name) if err != nil { if isMySQLForeignKey(err) { - return foreignKey(dbTable, name) + return foreignKey(dbTable.name, name) } return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable).WithName(name) + return notFound(dbTable.name).WithName(name) } return nil } // deleteEntities deletes the existing entity objects with the provided IDs. // The number of deleted entities is returned along with any error. -func (d *Datastore) deleteEntities(ctx context.Context, dbTable string, ids []uint) (uint, error) { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable) +func (d *Datastore) deleteEntities(ctx context.Context, dbTable entity, ids []uint) (uint, error) { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable.name) query, args, err := sqlx.In(deleteStmt, ids) if err != nil { diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index abe0cc26fd..ea039461d2 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -11,13 +11,31 @@ import ( "github.com/stretchr/testify/require" ) -func TestDeleteEntity(t *testing.T) { +func TestDelete(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Entity", testDeleteEntity}, + {"EntityByName", testDeleteEntityByName}, + {"Entities", testDeleteEntities}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.fn(t, ds) + }) + } +} + +func testDeleteEntity(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1", UUID: t.Name() + "1", @@ -29,35 +47,33 @@ func TestDeleteEntity(t *testing.T) { require.NoError(t, err) require.NotNil(t, host) - require.NoError(t, ds.deleteEntity(context.Background(), "hosts", host.ID)) + require.NoError(t, ds.deleteEntity(context.Background(), hostsTable, host.ID)) host, err = ds.Host(context.Background(), host.ID) require.Error(t, err) assert.Nil(t, host) } -func TestDeleteEntityByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testDeleteEntityByName(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) query1 := test.NewQuery(t, ds, t.Name()+"time", "select * from time", 0, true) - require.NoError(t, ds.deleteEntityByName(context.Background(), "queries", query1.Name)) + require.NoError(t, ds.deleteEntityByName(context.Background(), queriesTable, query1.Name)) gotQ, err := ds.Query(context.Background(), query1.ID) require.Error(t, err) assert.Nil(t, gotQ) } -func TestDeleteEntities(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testDeleteEntities(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) query1 := test.NewQuery(t, ds, t.Name()+"time1", "select * from time", 0, true) query2 := test.NewQuery(t, ds, t.Name()+"time2", "select * from time", 0, true) query3 := test.NewQuery(t, ds, t.Name()+"time3", "select * from time", 0, true) - count, err := ds.deleteEntities(context.Background(), "queries", []uint{query1.ID, query2.ID}) + count, err := ds.deleteEntities(context.Background(), queriesTable, []uint{query1.ID, query2.ID}) require.NoError(t, err) assert.Equal(t, uint(2), count) diff --git a/server/datastore/mysql/email_changes_test.go b/server/datastore/mysql/email_changes_test.go index e60a1d2a3f..1d8a84d5e1 100644 --- a/server/datastore/mysql/email_changes_test.go +++ b/server/datastore/mysql/email_changes_test.go @@ -11,10 +11,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestChangeEmail(t *testing.T) { +func TestEmailChanges(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Confirm", testEmailChangesConfirm}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testEmailChangesConfirm(t *testing.T, ds *Datastore) { user := &fleet.User{ Password: []byte("foobar"), Email: "bob@bob.com", @@ -45,5 +59,4 @@ func TestChangeEmail(t *testing.T) { require.Nil(t, err) _, err = ds.ConfirmPendingEmailChange(context.Background(), otheruser.ID, "uniquetoken") assert.NotNil(t, err) - } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index f54d922da6..6e558b8091 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -21,6 +21,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, osquery_host_id, detail_updated_at, label_updated_at, + policy_updated_at, node_key, hostname, uuid, @@ -32,7 +33,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, seen_time, team_id ) - VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,? ) + VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,?,? ) ` result, err := d.writer.ExecContext( ctx, @@ -40,6 +41,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, host.OsqueryHostID, host.DetailUpdatedAt, host.LabelUpdatedAt, + host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -65,6 +67,7 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { UPDATE hosts SET detail_updated_at = ?, label_updated_at = ?, + policy_updated_at = ?, node_key = ?, hostname = ?, uuid = ?, @@ -101,6 +104,7 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { _, err := tx.ExecContext(ctx, sqlStatement, host.DetailUpdatedAt, host.LabelUpdatedAt, + host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -146,7 +150,12 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { } } - if host.HostSoftware.Modified { + ac, err := d.AppConfig(ctx) + if err != nil { + return errors.Wrap(err, "failed to get app config to see if we need to update host users and inventory") + } + + if host.HostSoftware.Modified && ac.HostSettings.EnableSoftwareInventory { if err := saveHostSoftwareDB(ctx, tx, host); err != nil { return errors.Wrap(err, "failed to save host software") } @@ -157,8 +166,10 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { return errors.Wrap(err, "failed to save host additional") } - if err := saveHostUsersDB(ctx, tx, host); err != nil { - return errors.Wrap(err, "failed to save host users") + if ac.HostSettings.EnableHostUsers { + if err := saveHostUsersDB(ctx, tx, host); err != nil { + return errors.Wrap(err, "failed to save host users") + } } } @@ -285,7 +296,7 @@ func loadHostUsersDB(ctx context.Context, db sqlx.QueryerContext, host *fleet.Ho } func (d *Datastore) DeleteHost(ctx context.Context, hid uint) error { - err := d.deleteEntity(ctx, "hosts", hid) + err := d.deleteEntity(ctx, hostsTable, hid) if err != nil { return errors.Wrapf(err, "deleting host with id %d", hid) } @@ -485,13 +496,14 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin INSERT INTO hosts ( detail_updated_at, label_updated_at, + policy_updated_at, osquery_host_id, seen_time, node_key, team_id - ) VALUES (?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) + result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) if err != nil { return errors.Wrap(err, "insert host") @@ -554,6 +566,7 @@ func (d *Datastore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee updated_at, detail_updated_at, label_updated_at, + policy_updated_at, node_key, hostname, uuid, diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 323cec3d7b..37cc857e5f 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -45,13 +46,57 @@ var enrollTests = []struct { }, } -func TestSaveHosts(t *testing.T) { +func TestHosts(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Save", testHostsSave}, + {"DeleteWithSoftware", testHostsDeleteWithSoftware}, + {"SavePackStats", testHostsSavePackStats}, + {"SavePackStatsOverwrites", testHostsSavePackStatsOverwrites}, + {"IgnoresTeamPackStats", testHostsIgnoresTeamPackStats}, + {"Delete", testHostsDelete}, + {"ListFilterAdditional", testHostsListFilterAdditional}, + {"ListStatus", testHostsListStatus}, + {"ListQuery", testHostsListQuery}, + {"Enroll", testHostsEnroll}, + {"Authenticate", testHostsAuthenticate}, + {"AuthenticateCaseSensitive", testHostsAuthenticateCaseSensitive}, + {"Search", testHostsSearch}, + {"SearchLimit", testHostsSearchLimit}, + {"GenerateStatusStatistics", testHostsGenerateStatusStatistics}, + {"MarkSeen", testHostsMarkSeen}, + {"MarkSeenMany", testHostsMarkSeenMany}, + {"CleanupIncoming", testHostsCleanupIncoming}, + {"IDsByName", testHostsIDsByName}, + {"Additional", testHostsAdditional}, + {"ByIdentifier", testHostsByIdentifier}, + {"AddToTeam", testHostsAddToTeam}, + {"SaveUsers", testHostsSaveUsers}, + {"SaveUsersWithoutUid", testHostsSaveUsersWithoutUid}, + {"TotalAndUnseenSince", testHostsTotalAndUnseenSince}, + {"ListByPolicy", testHostsListByPolicy}, + {"SaveTonsOfUsers", testHostsSaveTonsOfUsers}, + {"SavePackStatsConcurrent", testHostsSavePackStatsConcurrent}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + + c.fn(t, ds) + }) + } +} + +func testHostsSave(t *testing.T, ds *Datastore) { + policyUpdatedAt := time.Now().UTC().Truncate(time.Second) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: policyUpdatedAt, SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -71,6 +116,7 @@ func TestSaveHosts(t *testing.T) { assert.Equal(t, "bar.local", host.Hostname) assert.Equal(t, "192.168.1.1", host.PrimaryIP) assert.Equal(t, "30-65-EC-6F-C4-58", host.PrimaryMac) + assert.Equal(t, policyUpdatedAt.UTC(), host.PolicyUpdatedAt) additionalJSON := json.RawMessage(`{"foobar": "bim"}`) host.Additional = &additionalJSON @@ -99,13 +145,11 @@ func TestSaveHosts(t *testing.T) { assert.Nil(t, host) } -func TestDeleteHostWithSoftware(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsDeleteWithSoftware(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -135,13 +179,11 @@ func TestDeleteHostWithSoftware(t *testing.T) { assert.Nil(t, host) } -func TestSaveHostPackStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -250,13 +292,11 @@ func TestSaveHostPackStats(t *testing.T) { require.Len(t, host.PackStats, 2) } -func TestSaveHostPackStatsOverwrites(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStatsOverwrites(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -396,13 +436,11 @@ func TestSaveHostPackStatsOverwrites(t *testing.T) { assert.Equal(t, execTime2, gotHost.PackStats[0].QueryStats[0].LastExecuted) } -func TestIgnoresTeamPackStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsIgnoresTeamPackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -478,13 +516,11 @@ func TestIgnoresTeamPackStats(t *testing.T) { assert.ElementsMatch(t, host.PackStats[0].QueryStats, stats1) } -func TestDeleteHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsDelete(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -500,13 +536,11 @@ func TestDeleteHost(t *testing.T) { assert.NotNil(t, err) } -func TestListHostsFilterAdditional(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListFilterAdditional(t *testing.T, ds *Datastore) { h, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -540,14 +574,12 @@ func TestListHostsFilterAdditional(t *testing.T) { assert.Equal(t, &additional, hosts[0].Additional) } -func TestListHostsStatus(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListStatus(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -579,15 +611,13 @@ func TestListHostsStatus(t *testing.T) { assert.Equal(t, 10, len(hosts)) } -func TestListHostsQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListQuery(t *testing.T, ds *Datastore) { hosts := []*fleet.Host{} for i := 0; i < 10; i++ { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -669,10 +699,7 @@ func TestListHostsQuery(t *testing.T) { assert.Equal(t, 1, len(gotHosts)) } -func TestEnrollHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsEnroll(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) @@ -709,10 +736,7 @@ func TestEnrollHost(t *testing.T) { } } -func TestAuthenticateHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAuthenticate(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -730,10 +754,7 @@ func TestAuthenticateHost(t *testing.T) { assert.Error(t, err) } -func TestAuthenticateHostCaseSensitive(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAuthenticateCaseSensitive(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -744,14 +765,12 @@ func TestAuthenticateHostCaseSensitive(t *testing.T) { } } -func TestSearchHosts(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSearch(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -763,6 +782,7 @@ func TestSearchHosts(t *testing.T) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -774,6 +794,7 @@ func TestSearchHosts(t *testing.T) { OsqueryHostID: "99999", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "3", UUID: "abc-def-ghi", @@ -836,16 +857,14 @@ func TestSearchHosts(t *testing.T) { assert.Equal(t, 1, len(hits)) } -func TestSearchHostsLimit(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSearchLimit(t *testing.T, ds *Datastore) { filter := fleet.TeamFilter{User: test.UserAdmin} for i := 0; i < 15; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -860,10 +879,7 @@ func TestSearchHostsLimit(t *testing.T) { assert.Len(t, hosts, 10) } -func TestGenerateHostStatusStatistics(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { filter := fleet.TeamFilter{User: test.UserAdmin} mockClock := clock.NewMockClock() @@ -881,6 +897,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { NodeKey: "1", DetailUpdatedAt: mockClock.Now().Add(-30 * time.Second), LabelUpdatedAt: mockClock.Now().Add(-30 * time.Second), + PolicyUpdatedAt: mockClock.Now().Add(-30 * time.Second), SeenTime: mockClock.Now().Add(-30 * time.Second), }) require.Nil(t, err) @@ -895,6 +912,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { NodeKey: "2", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Minute), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Minute), + PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Minute), SeenTime: mockClock.Now().Add(-1 * time.Minute), }) require.Nil(t, err) @@ -909,6 +927,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { NodeKey: "3", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Hour), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Hour), + PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Hour), SeenTime: mockClock.Now().Add(-1 * time.Hour), }) require.Nil(t, err) @@ -923,6 +942,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { NodeKey: "4", DetailUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), LabelUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), + PolicyUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)), }) require.Nil(t, err) @@ -942,10 +962,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { assert.Equal(t, uint(4), new) } -func TestMarkHostSeen(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsMarkSeen(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() anHourAgo := mockClock.Now().Add(-1 * time.Hour).UTC() @@ -958,6 +975,7 @@ func TestMarkHostSeen(t *testing.T) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) assert.Nil(t, err) @@ -980,10 +998,7 @@ func TestMarkHostSeen(t *testing.T) { } } -func TestMarkHostsSeen(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() aSecondAgo := mockClock.Now().Add(-1 * time.Second).UTC() @@ -997,6 +1012,7 @@ func TestMarkHostsSeen(t *testing.T) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1008,6 +1024,7 @@ func TestMarkHostsSeen(t *testing.T) { NodeKey: "2", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1041,13 +1058,9 @@ func TestMarkHostsSeen(t *testing.T) { require.NotNil(t, h2Verify) assert.WithinDuration(t, aSecondAgo, h2Verify.SeenTime, time.Second) } - } -func TestCleanupIncomingHosts(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() h1, err := ds.NewHost(context.Background(), &fleet.Host{ @@ -1057,6 +1070,7 @@ func TestCleanupIncomingHosts(t *testing.T) { NodeKey: "1", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1070,6 +1084,7 @@ func TestCleanupIncomingHosts(t *testing.T) { OsqueryVersion: "3.2.3", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1093,14 +1108,12 @@ func TestCleanupIncomingHosts(t *testing.T) { assert.Nil(t, err) } -func TestHostIDsByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsIDsByName(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1117,13 +1130,11 @@ func TestHostIDsByName(t *testing.T) { assert.Equal(t, hosts, []uint{2, 3, 6}) } -func TestHostAdditional(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAdditional(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -1191,14 +1202,12 @@ func TestHostAdditional(t *testing.T) { assert.Equal(t, &additional, h.Additional) } -func TestHostByIdentifier(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsByIdentifier(t *testing.T, ds *Datastore) { for i := 1; i <= 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("osquery_host_id_%d", i), NodeKey: fmt.Sprintf("node_key_%d", i), @@ -1232,10 +1241,7 @@ func TestHostByIdentifier(t *testing.T) { require.Error(t, err) } -func TestAddHostsToTeam(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAddToTeam(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -1282,13 +1288,11 @@ func TestAddHostsToTeam(t *testing.T) { } } -func TestSaveHostUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveUsers(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1354,13 +1358,11 @@ func TestSaveHostUsers(t *testing.T) { test.ElementsMatchSkipID(t, host.Users, []fleet.HostUser{u1, u2}) } -func TestSaveUsersWithoutUid(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveUsersWithoutUid(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1416,6 +1418,7 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-1 * time.Duration(days) * 24 * time.Hour), OsqueryHostID: fmt.Sprintf("%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1428,10 +1431,7 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { require.NotNil(t, host) } -func TestTotalAndUnseenHostsSince(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { addHostSeenLast(t, ds, 1, 0) total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) @@ -1448,14 +1448,12 @@ func TestTotalAndUnseenHostsSince(t *testing.T) { assert.Equal(t, 2, unseen) } -func TestListHostsByPolicy(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListByPolicy(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -1506,13 +1504,11 @@ func TestListHostsByPolicy(t *testing.T) { require.Len(t, hosts, 8) } -func TestSaveTonsOfUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1527,6 +1523,7 @@ func TestSaveTonsOfUsers(t *testing.T) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -1545,7 +1542,12 @@ func TestSaveTonsOfUsers(t *testing.T) { var count1 int32 var count2 int32 + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for { host1, err := ds.Host(context.Background(), host1.ID) if err != nil { @@ -1584,7 +1586,9 @@ func TestSaveTonsOfUsers(t *testing.T) { errCh <- err return } - atomic.AddInt32(&count1, 1) + if atomic.AddInt32(&count1, 1) >= 100 { + return + } select { case <-ctx.Done(): @@ -1595,6 +1599,8 @@ func TestSaveTonsOfUsers(t *testing.T) { }() go func() { + defer wg.Done() + for { host2, err := ds.Host(context.Background(), host2.ID) if err != nil { @@ -1633,7 +1639,9 @@ func TestSaveTonsOfUsers(t *testing.T) { errCh <- err return } - atomic.AddInt32(&count2, 1) + if atomic.AddInt32(&count2, 1) >= 100 { + return + } select { case <-ctx.Done(): @@ -1644,24 +1652,28 @@ func TestSaveTonsOfUsers(t *testing.T) { }() ticker := time.NewTicker(10 * time.Second) + go func() { + wg.Wait() + cancelFunc() + }() select { case err := <-errCh: - require.NoError(t, err) cancelFunc() + require.NoError(t, err) + case <-ctx.Done(): case <-ticker.C: + require.Fail(t, "timed out") } - fmt.Println("Count1", atomic.LoadInt32(&count1)) - fmt.Println("Count2", atomic.LoadInt32(&count2)) + t.Log("Count1", atomic.LoadInt32(&count1)) + t.Log("Count2", atomic.LoadInt32(&count2)) } -func TestSaveHostPackStatsConcurrent(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1676,6 +1688,7 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -1743,14 +1756,18 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { }, }, } - return ds.SaveHost(context.Background(), host1) + return ds.SaveHost(context.Background(), host) } errCh := make(chan error) var counter int32 - total := int32(1000) + const total = int32(100) + + var wg sync.WaitGroup loopAndSaveHost := func(host *fleet.Host) { + defer wg.Done() + for { err := saveHostRandomStats(host) if err != nil { @@ -1770,10 +1787,13 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { } } + wg.Add(3) go loopAndSaveHost(host1) go loopAndSaveHost(host2) go func() { + defer wg.Done() + for { specs := []*fleet.PackSpec{ { @@ -1817,13 +1837,14 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { } }() - ticker := time.NewTicker(1 * time.Minute) + ticker := time.NewTicker(10 * time.Second) select { case err := <-errCh: + cancelFunc() require.NoError(t, err) case <-ctx.Done(): - return + wg.Wait() case <-ticker.C: - t.Fail() + require.Fail(t, "timed out") } } diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go index bd92bb8aaa..c2d9371c75 100644 --- a/server/datastore/mysql/invites.go +++ b/server/datastore/mysql/invites.go @@ -138,7 +138,7 @@ func (d *Datastore) InviteByToken(ctx context.Context, token string) (*fleet.Inv } func (d *Datastore) DeleteInvite(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "invites", id) + return d.deleteEntity(ctx, invitesTable, id) } func (d *Datastore) loadTeamsForInvites(ctx context.Context, invites []*fleet.Invite) error { diff --git a/server/datastore/mysql/invites_test.go b/server/datastore/mysql/invites_test.go index a1b28edbc5..b1f6a0e587 100644 --- a/server/datastore/mysql/invites_test.go +++ b/server/datastore/mysql/invites_test.go @@ -12,10 +12,29 @@ import ( "gopkg.in/guregu/null.v3" ) -func TestCreateInvite(t *testing.T) { +func TestInvites(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Create", testInvitesCreate}, + {"List", testInvitesList}, + {"Delete", testInvitesDelete}, + {"ByToken", testInvitesByToken}, + {"ByEmail", testInvitesByEmail}, + {"Invite", testInvitesInvite}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testInvitesCreate(t *testing.T, ds *Datastore) { for i := 0; i < 3; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -64,13 +83,9 @@ func setupTestInvites(t *testing.T, ds fleet.Datastore) { _, err := ds.NewInvite(context.Background(), &i) require.NoError(t, err, "Failure creating user", user) } - } -func TestListInvites(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesList(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) opt := fleet.ListOptions{ @@ -94,13 +109,9 @@ func TestListInvites(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 4, len(result)) // allow for admin we created assert.Equal(t, "User00", result[3].Name) - } -func TestDeleteInvite(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesDelete(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) invite, err := ds.InviteByEmail(context.Background(), "user0@foo.com") @@ -114,13 +125,9 @@ func TestDeleteInvite(t *testing.T) { invite, err = ds.InviteByEmail(context.Background(), "user0@foo.com") assert.NotNil(t, err) assert.Nil(t, invite) - } -func TestInviteByToken(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesByToken(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) var inviteTests = []struct { @@ -137,7 +144,7 @@ func TestInviteByToken(t *testing.T) { } for _, tt := range inviteTests { - t.Run("", func(t *testing.T) { + t.Run(tt.token, func(t *testing.T) { invite, err := ds.InviteByToken(context.Background(), tt.token) if tt.wantErr != nil { require.NotNil(t, err) @@ -146,15 +153,11 @@ func TestInviteByToken(t *testing.T) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) - }) } } -func TestInviteByEmail(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesByEmail(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) var inviteTests = []struct { @@ -171,7 +174,7 @@ func TestInviteByEmail(t *testing.T) { } for _, tt := range inviteTests { - t.Run("", func(t *testing.T) { + t.Run(tt.email, func(t *testing.T) { invite, err := ds.InviteByEmail(context.Background(), tt.email) if tt.wantErr != nil { require.NotNil(t, err) @@ -180,15 +183,11 @@ func TestInviteByEmail(t *testing.T) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) - }) } } -func TestInvite(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesInvite(t *testing.T, ds *Datastore) { admin := &fleet.Invite{ Email: "admin@foo.com", Name: "Xadmin", diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 5da41d18da..9b46ac0714 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -217,33 +217,35 @@ func (d *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...fl } func (d *Datastore) SaveLabel(ctx context.Context, label *fleet.Label) (*fleet.Label, error) { - query := ` - UPDATE labels SET - name = ?, - description = ? - WHERE id = ? - ` + query := `UPDATE labels SET name = ?, description = ? WHERE id = ?` _, err := d.writer.ExecContext(ctx, query, label.Name, label.Description, label.ID) if err != nil { return nil, errors.Wrap(err, "saving label") } - return label, nil + return labelDB(ctx, label.ID, d.writer) } // DeleteLabel deletes a fleet.Label func (d *Datastore) DeleteLabel(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "labels", name) + return d.deleteEntityByName(ctx, labelsTable, name) } // Label returns a fleet.Label identified by lid if one exists. func (d *Datastore) Label(ctx context.Context, lid uint) (*fleet.Label, error) { + return labelDB(ctx, lid, d.reader) +} + +func labelDB(ctx context.Context, lid uint, q sqlx.QueryerContext) (*fleet.Label, error) { sql := ` - SELECT * FROM labels - WHERE id = ? + SELECT + l.*, + (SELECT COUNT(1) FROM label_membership lm JOIN hosts h ON (lm.host_id = h.id) WHERE label_id = l.id) AS host_count + FROM labels l + WHERE id = ? ` label := &fleet.Label{} - if err := sqlx.GetContext(ctx, d.reader, label, sql, lid); err != nil { + if err := sqlx.GetContext(ctx, q, label, sql, lid); err != nil { return nil, errors.Wrap(err, "selecting label") } @@ -283,39 +285,12 @@ func platformForHost(host *fleet.Host) string { return host.Platform } -func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { +func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { var rows *sql.Rows var err error platform := platformForHost(host) - if host.LabelUpdatedAt.Before(cutoff) { - // Retrieve all labels (with matching platform) for this host - sql := ` - SELECT id, query - FROM labels - WHERE platform = ? OR platform = '' - AND label_membership_type = ? -` - rows, err = d.reader.QueryContext(ctx, sql, platform, fleet.LabelMembershipTypeDynamic) - } else { - // Retrieve all labels (with matching platform) iff there is a label - // that has been created since this host last reported label query - // executions - sql := ` - SELECT id, query - FROM labels - WHERE ((SELECT max(created_at) FROM labels WHERE platform = ? OR platform = '') > ?) - AND (platform = ? OR platform = '') - AND label_membership_type = ? -` - rows, err = d.reader.QueryContext( - ctx, - sql, - platform, - host.LabelUpdatedAt, - platform, - fleet.LabelMembershipTypeDynamic, - ) - } + query := `SELECT id, query FROM labels WHERE platform = ? OR platform = '' AND label_membership_type = ?` + rows, err = d.reader.QueryContext(ctx, query, platform, fleet.LabelMembershipTypeDynamic) if err != nil && err != sql.ErrNoRows { return nil, errors.Wrap(err, "selecting label queries for host") @@ -370,14 +345,8 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Complete inserts if necessary if len(vals) > 0 { - sql := ` - INSERT IGNORE INTO label_membership (updated_at, label_id, host_id) VALUES - ` - sql += strings.Join(bindvars, ",") + - ` - ON DUPLICATE KEY UPDATE - updated_at = VALUES(updated_at) - ` + sql := `INSERT INTO label_membership (updated_at, label_id, host_id) VALUES ` + sql += strings.Join(bindvars, ",") + ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` _, err := tx.ExecContext(ctx, sql, vals...) if err != nil { @@ -387,9 +356,7 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. // Complete deletions if necessary if len(removes) > 0 { - sql := ` - DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?) - ` + sql := `DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?)` query, args, err := sqlx.In(sql, host.ID, removes) if err != nil { return errors.Wrap(err, "IN for DELETE FROM label_membership") @@ -666,5 +633,12 @@ func (d *Datastore) LabelIDsByName(ctx context.Context, labels []string) ([]uint } return labelIDs, nil - +} + +func (d *Datastore) CleanupOrphanLabelMembership(ctx context.Context) error { + _, err := d.writer.ExecContext(ctx, `DELETE FROM label_membership where not exists (select 1 from labels where id=label_id) or not exists (select 1 from hosts where id=host_id)`) + if err != nil { + return errors.Wrap(err, "cleaning orphan label_membership by label") + } + return nil } diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index 30fef5d73e..c610ce7859 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -37,9 +37,37 @@ func TestBatchHostnamesLarge(t *testing.T) { } func TestLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() + ds := CreateMySQLDS(t) + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"AddAllHosts", testLabelsAddAllHosts}, + {"Search", testLabelsSearch}, + {"ListHostsInLabel", testLabelsListHostsInLabel}, + {"ListHostsInLabelAndStatus", testLabelsListHostsInLabelAndStatus}, + {"ListHostsInLabelAndTeamFilter", testLabelsListHostsInLabelAndTeamFilter}, + {"BuiltIn", testLabelsBuiltIn}, + {"ListUniqueHostsInLabels", testLabelsListUniqueHostsInLabels}, + {"ChangeDetails", testLabelsChangeDetails}, + {"GetSpec", testLabelsGetSpec}, + {"ApplySpecsRoundtrip", testLabelsApplySpecsRoundtrip}, + {"IDsByName", testLabelsIDsByName}, + {"Save", testLabelsSave}, + {"QueriesForCentOSHost", testLabelsQueriesForCentOSHost}, + {"RecordNonExistentQueryLabelExecution", testLabelsRecordNonexistentQueryLabelExecution}, + {"LabelMembershipCleanup", testLabelMembershipCleanup}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testLabelsAddAllHosts(t *testing.T, db *Datastore) { test.AddAllHostsLabel(t, db) hosts := []fleet.Host{} var host *fleet.Host @@ -52,10 +80,8 @@ func TestLabels(t *testing.T) { host.Platform = "darwin" require.NoError(t, db.SaveHost(context.Background(), host)) - baseTime := time.Now() - // No labels to check - queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err := db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Len(t, queries, 0) @@ -99,7 +125,7 @@ func TestLabels(t *testing.T) { host.Platform = "darwin" // Now queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Equal(t, expectQueries, queries) @@ -108,6 +134,8 @@ func TestLabels(t *testing.T) { assert.Nil(t, err) assert.Len(t, labels, 1) + baseTime := time.Now() + // Record a query execution err = db.RecordLabelQueryExecutions( context.Background(), @@ -120,14 +148,6 @@ func TestLabels(t *testing.T) { require.NoError(t, err) host.LabelUpdatedAt = baseTime - // Now no queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) - assert.Nil(t, err) - assert.Len(t, queries, 0) - - // Ensure enough gap in created_at - time.Sleep(2 * time.Second) - // A new label targeting another platform should not effect the labels for // this host err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -138,9 +158,9 @@ func TestLabels(t *testing.T) { }, }) require.NoError(t, err) - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) - assert.Len(t, queries, 0) + assert.Len(t, queries, 4) // If a new label is added, all labels should be returned err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -152,29 +172,7 @@ func TestLabels(t *testing.T) { }) require.NoError(t, err) expectQueries["7"] = "query6" - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) - assert.Nil(t, err) - assert.Len(t, queries, 5) - - // After expiration, all queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add((2 * time.Minute))) - assert.Nil(t, err) - assert.Equal(t, expectQueries, queries) - - // Now the two matching labels should be returned - labels, err = db.ListLabelsForHost(context.Background(), host.ID) - assert.Nil(t, err) - if assert.Len(t, labels, 2) { - labelNames := []string{labels[0].Name, labels[1].Name} - sort.Strings(labelNames) - assert.Equal(t, "All Hosts", labelNames[0]) - assert.Equal(t, "label1", labelNames[1]) - } - - // A host that hasn't executed any label queries should still be asked - // to execute those queries - hosts[0].Platform = "darwin" - queries, err = db.LabelQueriesForHost(context.Background(), &hosts[0], time.Now()) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Len(t, queries, 5) @@ -185,10 +183,7 @@ func TestLabels(t *testing.T) { assert.Len(t, labels, 1) } -func TestSearchLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsSearch(t *testing.T, db *Datastore) { specs := []*fleet.LabelSpec{ {ID: 1, Name: "foo"}, {ID: 2, Name: "bar"}, @@ -241,13 +236,11 @@ func TestSearchLabels(t *testing.T) { assert.Contains(t, labels, all) } -func TestListHostsInLabel(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -259,6 +252,7 @@ func TestListHostsInLabel(t *testing.T) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "2", NodeKey: "2", @@ -270,6 +264,7 @@ func TestListHostsInLabel(t *testing.T) { h3, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "3", NodeKey: "3", @@ -306,13 +301,11 @@ func TestListHostsInLabel(t *testing.T) { } } -func TestListHostsInLabelAndStatus(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -325,6 +318,7 @@ func TestListHostsInLabelAndStatus(t *testing.T) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, + PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -362,13 +356,11 @@ func TestListHostsInLabelAndStatus(t *testing.T) { } } -func TestListHostsInLabelAndTeamFilter(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -381,6 +373,7 @@ func TestListHostsInLabelAndTeamFilter(t *testing.T) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, + PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -439,10 +432,7 @@ func TestListHostsInLabelAndTeamFilter(t *testing.T) { } } -func TestBuiltInLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsBuiltIn(t *testing.T, db *Datastore) { require.Nil(t, db.MigrateData(context.Background())) user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} @@ -456,15 +446,13 @@ func TestBuiltInLabels(t *testing.T) { assert.Equal(t, fleet.LabelTypeBuiltIn, hits[1].LabelType) } -func TestListUniqueHostsInLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListUniqueHostsInLabels(t *testing.T, db *Datastore) { hosts := []*fleet.Host{} for i := 0; i < 4; i++ { h, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -508,13 +496,9 @@ func TestListUniqueHostsInLabels(t *testing.T) { labels, err := db.ListLabels(context.Background(), filter, fleet.ListOptions{}) require.Nil(t, err) require.Len(t, labels, 2) - } -func TestChangeLabelDetails(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsChangeDetails(t *testing.T, db *Datastore) { label := fleet.LabelSpec{ ID: 1, Name: "my label", @@ -539,6 +523,7 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -583,10 +568,7 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { return expectedSpecs } -func TestGetLabelSpec(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsGetSpec(t *testing.T, ds *Datastore) { expectedSpecs := setupLabelSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -596,10 +578,7 @@ func TestGetLabelSpec(t *testing.T) { } } -func TestApplyLabelSpecsRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsApplySpecsRoundtrip(t *testing.T, ds *Datastore) { expectedSpecs := setupLabelSpecsTest(t, ds) specs, err := ds.GetLabelSpecs(context.Background()) @@ -614,10 +593,7 @@ func TestApplyLabelSpecsRoundtrip(t *testing.T) { test.ElementsMatchSkipTimestampsID(t, expectedSpecs, specs) } -func TestLabelIDsByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsIDsByName(t *testing.T, ds *Datastore) { setupLabelSpecsTest(t, ds) labels, err := ds.LabelIDsByName(context.Background(), []string{"foo", "bar", "bing"}) @@ -626,9 +602,18 @@ func TestLabelIDsByName(t *testing.T) { assert.Equal(t, []uint{1, 2, 3}, labels) } -func TestSaveLabel(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() +func testLabelsSave(t *testing.T, db *Datastore) { + h1, err := db.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + OsqueryHostID: "1", + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + }) + require.NoError(t, err) label := &fleet.Label{ Name: "my label", @@ -636,22 +621,23 @@ func TestSaveLabel(t *testing.T) { Query: "select 1 from processes;", Platform: "darwin", } - label, err := db.NewLabel(context.Background(), label) - require.Nil(t, err) + label, err = db.NewLabel(context.Background(), label) + require.NoError(t, err) label.Name = "changed name" label.Description = "changed description" + + require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) + _, err = db.SaveLabel(context.Background(), label) - require.Nil(t, err) + require.NoError(t, err) saved, err := db.Label(context.Background(), label.ID) - require.Nil(t, err) + require.NoError(t, err) assert.Equal(t, label.Name, saved.Name) assert.Equal(t, label.Description, saved.Description) + assert.Equal(t, 1, saved.HostCount) } -func TestLabelQueriesForCentOSHost(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { host, err := db.EnrollHost(context.Background(), "0", "0", nil, 0) require.Nil(t, err, "enrollment should succeed") host.Platform = "rhel" @@ -672,21 +658,17 @@ func TestLabelQueriesForCentOSHost(t *testing.T) { }) require.NoError(t, err) - baseTime := time.Now().Add(-5 * time.Minute) - - queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err := db.LabelQueriesForHost(context.Background(), host) require.NoError(t, err) require.Len(t, queries, 1) assert.Equal(t, "select 1;", queries[fmt.Sprint(label.ID)]) } -func TestRecordNonexistentQueryLabelExecution(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -705,3 +687,69 @@ func TestRecordNonexistentQueryLabelExecution(t *testing.T) { require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{99999: ptr.Bool(true)}, time.Now())) } + +func testLabelMembershipCleanup(t *testing.T, ds *Datastore) { + setupTest := func() (*fleet.Host, *fleet.Label) { + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + OsqueryHostID: "1", + }) + require.NoError(t, err) + require.NotNil(t, host) + + label := &fleet.Label{ + Name: "foo", + Query: "select * from foo;", + } + label, err = ds.NewLabel(context.Background(), label) + require.NoError(t, err) + + require.NoError(t, ds.RecordLabelQueryExecutions(context.Background(), host, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) + return host, label + } + + checkCount := func(before, after int) { + var count int + require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) + assert.Equal(t, before, count) + + require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) + + require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) + assert.Equal(t, after, count) + } + + t.Run("none gone", func(t *testing.T) { + host, label := setupTest() + checkCount(1, 1) + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) + }) + t.Run("label gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + checkCount(1, 0) + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + }) + t.Run("host gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + checkCount(1, 0) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + }) + t.Run("both gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + checkCount(1, 0) + }) +} diff --git a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go index f1721cf57b..4de803112b 100644 --- a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go +++ b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go @@ -31,13 +31,13 @@ func Up_20210819131107(tx *sql.Tx) error { } // Clear any orphan software and host_software - _, err = tx.Exec(`DELETE FROM host_software WHERE NOT EXISTS (select 1 from hosts h where h.id=host_software.host_id)`) + _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_host_software AS SELECT * FROM host_software;`) if err != nil { - return errors.Wrap(err, "clearing orphan host_software") + return errors.Wrap(err, "save current host software to a temp table") } - _, err = tx.Exec(`DELETE FROM software WHERE NOT EXISTS (select 1 from host_software hs where hs.software_id=software.id)`) + _, err = tx.Exec(`DELETE FROM host_software;`) if err != nil { - return errors.Wrap(err, "clearing orphan software") + return errors.Wrap(err, "clear all host software") } if _, err := tx.Exec(` @@ -48,6 +48,16 @@ func Up_20210819131107(tx *sql.Tx) error { return errors.Wrap(err, "add fk on host_software hosts & software") } + _, err = tx.Exec(`INSERT IGNORE INTO host_software SELECT * FROM temp_host_software;`) + if err != nil { + return errors.Wrap(err, "reinserting host software") + } + + _, err = tx.Exec(`DROP TABLE temp_host_software;`) + if err != nil { + return errors.Wrap(err, "dropping temp table") + } + return nil } diff --git a/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go b/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go new file mode 100644 index 0000000000..40cec1783a --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go @@ -0,0 +1,25 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210915144307, Down_20210915144307) +} + +func Up_20210915144307(tx *sql.Tx) error { + if _, err := tx.Exec(`ALTER TABLE policies + ADD COLUMN team_id INT UNSIGNED, + ADD FOREIGN KEY fk_policies_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE + `); err != nil { + return errors.Wrap(err, "add column team_id") + } + return nil +} + +func Down_20210915144307(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go b/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go new file mode 100644 index 0000000000..6c7466e722 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go @@ -0,0 +1,24 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210920155130, Down_20210920155130) +} + +func Up_20210920155130(tx *sql.Tx) error { + _, err := tx.Exec( + `ALTER TABLE label_membership DROP FOREIGN KEY fk_lm_host_id, DROP FOREIGN KEY fk_lm_label_id;`) + if err != nil { + return errors.Wrap(err, "dropping foreign keys for label_membership") + } + return nil +} + +func Down_20210920155130(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go b/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go new file mode 100644 index 0000000000..43385178ee --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go @@ -0,0 +1,29 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210927143115, Down_20210927143115) +} + +func Up_20210927143115(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE hosts ADD COLUMN policy_updated_at timestamp NOT NULL DEFAULT '2000-01-01 00:00:00'") + if err != nil { + return errors.Wrap(err, "adding policy_updated_at column") + } + + _, err = tx.Exec("delete from policy_membership_history") + if err != nil { + return errors.Wrap(err, "clearing policy_membership_history") + } + + return err +} + +func Down_20210927143115(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations_test.go b/server/datastore/mysql/migrations_test.go index f30833f41c..57d469eec4 100644 --- a/server/datastore/mysql/migrations_test.go +++ b/server/datastore/mysql/migrations_test.go @@ -5,12 +5,9 @@ import ( "context" "os/exec" "testing" - "time" "github.com/fleetdm/fleet/v4/server/config" - "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables" "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -68,54 +65,3 @@ func createMySQLDSForMigrationTests(t *testing.T, dbName string) *Datastore { require.NoError(t, err) return ds } - -func Test20210819131107_AddCascadeToHostSoftware(t *testing.T) { - ds := createMySQLDSForMigrationTests(t, t.Name()) - defer ds.Close() - - for { - version, err := tables.MigrationClient.GetDBVersion(ds.writer.DB) - require.NoError(t, err) - - // break right before the the constraint migration - if version == 20210818182258 { - break - } - require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) - } - - host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) - host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) - - soft1 := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, - }, - } - host1.HostSoftware = soft1 - soft2 := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}, - {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, - {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, - }, - } - host2.HostSoftware = soft2 - host2.Modified = true - - require.NoError(t, ds.SaveHostSoftware(context.Background(), host1)) - require.NoError(t, ds.SaveHostSoftware(context.Background(), host2)) - - require.NoError(t, ds.DeleteHost(context.Background(), host1.ID)) - - require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) - - // Make sure we don't delete more than we need - hostCheck, err := ds.Host(context.Background(), host2.ID) - require.NoError(t, err) - require.NoError(t, ds.LoadHostSoftware(context.Background(), hostCheck)) - require.Len(t, hostCheck.Software, 3) -} diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 6d72fab858..9664df5111 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -62,6 +62,22 @@ type Datastore struct { type txFn func(sqlx.ExtContext) error +type entity struct { + name string +} + +var ( + hostsTable = entity{"hosts"} + invitesTable = entity{"invites"} + labelsTable = entity{"labels"} + packsTable = entity{"packs"} + queriesTable = entity{"queries"} + scheduledQueriesTable = entity{"scheduled_queries"} + sessionsTable = entity{"sessions"} + teamsTable = entity{"teams"} + usersTable = entity{"users"} +) + // retryableError determines whether a MySQL error can be retried. By default // errors are considered non-retryable. Only errors that we know have a // possibility of succeeding on a retry should return true in this function. diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index 68b40cbcdd..3e10b99d20 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -50,6 +50,7 @@ func TestDatastoreReplica(t *testing.T) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index 20b2e89fed..3e03afc873 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -365,7 +365,7 @@ func (d *Datastore) SavePack(ctx context.Context, pack *fleet.Pack) error { // DeletePack deletes a fleet.Pack so that it won't show up in results. func (d *Datastore) DeletePack(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "packs", name) + return d.deleteEntityByName(ctx, packsTable, name) } // Pack fetch fleet.Pack with matching ID diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 710d752a22..c571b3b36f 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -15,10 +15,39 @@ import ( "github.com/stretchr/testify/require" ) -func TestDeletePack(t *testing.T) { +func TestPacks(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Delete", testPacksDelete}, + {"Save", testPacksSave}, + {"GetByName", testPacksGetByName}, + {"List", testPacksList}, + {"ApplySpecRoundtrip", testPacksApplySpecRoundtrip}, + {"GetSpec", testPacksGetSpec}, + {"ApplySpecMissingQueries", testPacksApplySpecMissingQueries}, + {"ApplySpecMissingName", testPacksApplySpecMissingName}, + {"ListForHost", testPacksListForHost}, + {"EnsureGlobal", testPacksEnsureGlobal}, + {"EnsureTeam", testPacksEnsureTeam}, + {"TeamNameChangesTeamSchedule", testPacksTeamNameChangesTeamSchedule}, + {"TeamScheduleNamesMigrateToNewFormat", testPacksTeamScheduleNamesMigrateToNewFormat}, + {"ApplySpecFailsOnTargetIDNull", testPacksApplySpecFailsOnTargetIDNull}, + {"ApplyStatsNotLocking", testPacksApplyStatsNotLocking}, + {"ApplyStatsNotLockingTryTwo", testPacksApplyStatsNotLockingTryTwo}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPacksDelete(t *testing.T, ds *Datastore) { pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -33,10 +62,7 @@ func TestDeletePack(t *testing.T) { assert.NotNil(t, err) } -func TestSavePack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksSave(t *testing.T, ds *Datastore) { expectedPack := &fleet.Pack{ Name: "foo", HostIDs: []uint{1}, @@ -70,10 +96,7 @@ func TestSavePack(t *testing.T) { test.EqualSkipTimestampsID(t, expectedPack, pack) } -func TestGetPackByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksGetByName(t *testing.T, ds *Datastore) { pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -87,13 +110,9 @@ func TestGetPackByName(t *testing.T) { require.Nil(t, err) assert.False(t, ok) assert.Nil(t, pack) - } -func TestListPacks(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksList(t *testing.T, ds *Datastore) { p1 := &fleet.PackSpec{ ID: 1, Name: "foo_pack", @@ -223,10 +242,7 @@ func setupPackSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.PackSpec { return expectedSpecs } -func TestApplyPackSpecRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecRoundtrip(t *testing.T, ds *Datastore) { expectedSpecs := setupPackSpecsTest(t, ds) gotSpec, err := ds.GetPackSpecs(context.Background()) @@ -234,10 +250,7 @@ func TestApplyPackSpecRoundtrip(t *testing.T) { assert.Equal(t, expectedSpecs, gotSpec) } -func TestGetPackSpec(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksGetSpec(t *testing.T, ds *Datastore) { expectedSpecs := setupPackSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -247,10 +260,7 @@ func TestGetPackSpec(t *testing.T) { } } -func TestApplyPackSpecMissingQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecMissingQueries(t *testing.T, ds *Datastore) { // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -275,10 +285,7 @@ func TestApplyPackSpecMissingQueries(t *testing.T) { } } -func TestApplyPackSpecMissingName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecMissingName(t *testing.T, ds *Datastore) { setupPackSpecsTest(t, ds) specs := []*fleet.PackSpec{ @@ -304,10 +311,7 @@ func TestApplyPackSpecMissingName(t *testing.T) { assert.Equal(t, "foo", spec.Queries[0].Name) } -func TestListPacksForHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksListForHost(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() l1 := &fleet.LabelSpec{ @@ -416,10 +420,7 @@ func TestListPacksForHost(t *testing.T) { } } -func TestEnsureGlobalPack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksEnsureGlobal(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) @@ -450,10 +451,7 @@ func TestEnsureGlobalPack(t *testing.T) { assert.Equal(t, "global", *gp.Type) } -func TestEnsureTeamPack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksEnsureTeam(t *testing.T, ds *Datastore) { packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) require.Nil(t, err) assert.Len(t, packs, 0) @@ -499,10 +497,7 @@ func TestEnsureTeamPack(t *testing.T) { assert.Equal(t, []uint{team2.ID}, tp2.TeamIDs) } -func TestTeamNameChangesTeamSchedule(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksTeamNameChangesTeamSchedule(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -521,10 +516,7 @@ func TestTeamNameChangesTeamSchedule(t *testing.T) { assert.Equal(t, teamScheduleName(team1), tp.Name) } -func TestTeamScheduleNamesMigrateToNewFormat(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksTeamScheduleNamesMigrateToNewFormat(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -547,10 +539,7 @@ func TestTeamScheduleNamesMigrateToNewFormat(t *testing.T) { require.Equal(t, teamScheduleName(team1), tp.Name) } -func TestApplyPackSpecFailsOnTargetIDNull(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecFailsOnTargetIDNull(t *testing.T, ds *Datastore) { // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -602,17 +591,15 @@ func randomPackStatsForHost(hostID, packID uint, scheduledQueries []*fleet.Sched } } -func TestPackApplyStatsNotLocking(t *testing.T) { +func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { t.Skip("This can be too much for the test db if you're running all tests") - ds := CreateMySQLDS(t) - defer ds.Close() - specs := setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -650,17 +637,15 @@ func TestPackApplyStatsNotLocking(t *testing.T) { cancelFunc() } -func TestPackApplyStatsNotLockingTryTwo(t *testing.T) { +func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { t.Skip("This can be too much for the test db if you're running all tests") - ds := CreateMySQLDS(t) - defer ds.Close() - setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/password_reset_test.go b/server/datastore/mysql/password_reset_test.go index 71c0c5ca62..ce9b2fcc40 100644 --- a/server/datastore/mysql/password_reset_test.go +++ b/server/datastore/mysql/password_reset_test.go @@ -9,11 +9,25 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPasswordResetRequests(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() +func TestPasswordReset(t *testing.T) { + ds := CreateMySQLDS(t) - createTestUsers(t, db) + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Requests", testPasswordResetRequests}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPasswordResetRequests(t *testing.T, ds *Datastore) { + createTestUsers(t, ds) now := time.Now().UTC() tomorrow := now.Add(time.Hour * 24) var passwordResetTests = []struct { @@ -30,7 +44,7 @@ func TestPasswordResetRequests(t *testing.T) { ExpiresAt: tt.expires, Token: tt.token, } - req, err := db.NewPasswordResetRequest(context.Background(), r) + req, err := ds.NewPasswordResetRequest(context.Background(), r) assert.Nil(t, err) assert.Equal(t, tt.userID, req.UserID) } diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 256d11a1f5..1b0a196d7e 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -22,23 +22,30 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, queryID uint) (*fleet. return nil, errors.Wrap(err, "getting last id after inserting policy") } - return policyDB(ctx, ds.writer, uint(lastIdInt64)) + return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) } func (ds *Datastore) Policy(ctx context.Context, id uint) (*fleet.Policy, error) { - return policyDB(ctx, ds.reader, id) + return policyDB(ctx, ds.reader, id, nil) } -func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.Policy, error) { +func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint, teamID *uint) (*fleet.Policy, error) { + teamWhere := "TRUE" + args := []interface{}{id} + if teamID != nil { + teamWhere = "team_id = ?" + args = append(args, *teamID) + } + var policy fleet.Policy err := sqlx.GetContext(ctx, q, &policy, - `SELECT + fmt.Sprintf(`SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=?`, - id) + FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=? AND %s`, teamWhere), + args...) if err != nil { return nil, errors.Wrap(err, "getting policy") } @@ -78,17 +85,27 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee } func (ds *Datastore) ListGlobalPolicies(ctx context.Context) ([]*fleet.Policy, error) { + return listPoliciesDB(ctx, ds.reader, nil) +} + +func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID *uint) ([]*fleet.Policy, error) { + teamWhere := "p.team_id is NULL" + var args []interface{} + if teamID != nil { + teamWhere = "p.team_id = ?" + args = append(args, *teamID) + } var policies []*fleet.Policy err := sqlx.SelectContext( ctx, - ds.reader, + q, &policies, - `SELECT + fmt.Sprintf(`SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id)`, + FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE %s`, teamWhere), args..., ) if err != nil { return nil, errors.Wrap(err, "listing policies") @@ -97,33 +114,91 @@ func (ds *Datastore) ListGlobalPolicies(ctx context.Context) ([]*fleet.Policy, e } func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { - stmt := `DELETE FROM policies WHERE id IN (?)` + return deletePolicyDB(ctx, ds.writer, ids, nil) +} + +func deletePolicyDB(ctx context.Context, q sqlx.ExtContext, ids []uint, teamID *uint) ([]uint, error) { + stmt := `DELETE FROM policies WHERE id IN (?) AND %s` stmt, args, err := sqlx.In(stmt, ids) if err != nil { return nil, errors.Wrap(err, "IN for DELETE FROM policies") } - stmt = ds.writer.Rebind(stmt) - if _, err := ds.writer.ExecContext(ctx, stmt, args...); err != nil { + stmt = q.Rebind(stmt) + + teamWhere := "TRUE" + if teamID != nil { + teamWhere = "team_id = ?" + args = append(args, *teamID) + } + + if _, err := q.ExecContext(ctx, fmt.Sprintf(stmt, teamWhere), args...); err != nil { return nil, errors.Wrap(err, "delete policies") } return ids, nil } -func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, _ *fleet.Host) (map[string]string, error) { - var rows []struct { +func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { + var globalRows, teamRows []struct { Id string `db:"id"` Query string `db:"query"` } - err := sqlx.SelectContext(ctx, ds.reader, &rows, `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id)`) + err := sqlx.SelectContext( + ctx, + ds.reader, + &globalRows, + `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id is NULL`, + ) if err != nil { return nil, errors.Wrap(err, "selecting policies for host") } results := map[string]string{} - for _, row := range rows { + if host.TeamID != nil { + err := sqlx.SelectContext( + ctx, + ds.reader, + &teamRows, + `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id = ?`, + *host.TeamID, + ) + if err != nil { + return nil, errors.Wrap(err, "selecting policies for host in team") + } + } + + for _, row := range globalRows { + results[row.Id] = row.Query + } + + for _, row := range teamRows { results[row.Id] = row.Query } return results, nil } + +func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + res, err := ds.writer.ExecContext(ctx, `INSERT INTO policies (query_id, team_id) VALUES (?, ?)`, queryID, teamID) + if err != nil { + return nil, errors.Wrap(err, "inserting new team policy") + } + lastIdInt64, err := res.LastInsertId() + if err != nil { + return nil, errors.Wrap(err, "getting last id after inserting policy") + } + + return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) +} + +func (ds *Datastore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + return listPoliciesDB(ctx, ds.reader, &teamID) +} + +func (ds *Datastore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + return deletePolicyDB(ctx, ds.writer, ids, &teamID) +} + +func (ds *Datastore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + return policyDB(ctx, ds.reader, policyID, &teamID) +} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index fca687aa4c..8897d98995 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -12,10 +12,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewGlobalPolicy(t *testing.T) { +func TestPolicies(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"NewGlobalPolicy", testPoliciesNewGlobalPolicy}, + {"MembershipView", testPoliciesMembershipView}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPoliciesNewGlobalPolicy(t *testing.T, ds *Datastore) { q, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "query1", Description: "query1 desc", @@ -58,14 +73,12 @@ func TestNewGlobalPolicy(t *testing.T) { require.NoError(t, ds.DeleteQuery(context.Background(), q.Name)) } -func TestPolicyMembershipView(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPoliciesMembershipView(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -77,6 +90,7 @@ func TestPolicyMembershipView(t *testing.T) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -142,9 +156,138 @@ func TestPolicyMembershipView(t *testing.T) { require.NoError(t, err) assert.Equal(t, policies[0], policy) - queries, err := ds.PolicyQueriesForHost(context.Background(), nil) + queries, err := ds.PolicyQueriesForHost(context.Background(), host1) require.NoError(t, err) require.Len(t, queries, 2) assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) } + +func TestTeamPolicy(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + q, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Description: "query1 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + + team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) + require.NoError(t, err) + + q2, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Description: "query2 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + + prevPolicies, err := ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + + _, err = ds.NewTeamPolicy(context.Background(), 99999999, q.ID) + require.Error(t, err) + + p, err := ds.NewTeamPolicy(context.Background(), team1.ID, q.ID) + require.NoError(t, err) + + assert.Equal(t, "query1", p.QueryName) + + globalPolicies, err := ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + require.Len(t, globalPolicies, len(prevPolicies)) + + _, err = ds.NewTeamPolicy(context.Background(), team2.ID, q2.ID) + require.NoError(t, err) + + teamPolicies, err := ds.ListTeamPolicies(context.Background(), team1.ID) + require.NoError(t, err) + require.Len(t, teamPolicies, 1) + assert.Equal(t, q.ID, teamPolicies[0].QueryID) + + team2Policies, err := ds.ListTeamPolicies(context.Background(), team2.ID) + require.NoError(t, err) + require.Len(t, team2Policies, 1) + assert.Equal(t, q2.ID, team2Policies[0].QueryID) + + _, err = ds.DeleteTeamPolicies(context.Background(), team1.ID, []uint{teamPolicies[0].ID}) + require.NoError(t, err) + + teamPolicies, err = ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + require.Len(t, teamPolicies, 0) +} + +func TestPolicyQueriesForHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + host1, err := ds.NewHost(context.Background(), &fleet.Host{ + OsqueryHostID: "1234", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + }) + require.NoError(t, err) + + require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID})) + host1, err = ds.Host(context.Background(), host1.ID) + require.NoError(t, err) + + host2, err := ds.NewHost(context.Background(), &fleet.Host{ + OsqueryHostID: "5679", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "2", + UUID: "2", + Hostname: "bar.local", + }) + require.NoError(t, err) + + q, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Description: "query1 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewGlobalPolicy(context.Background(), q.ID) + require.NoError(t, err) + + q2, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Description: "query2 desc", + Query: "select 42;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewTeamPolicy(context.Background(), team1.ID, q2.ID) + require.NoError(t, err) + + queries, err := ds.PolicyQueriesForHost(context.Background(), host1) + require.NoError(t, err) + require.Len(t, queries, 2) + assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) + assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) + + queries, err = ds.PolicyQueriesForHost(context.Background(), host2) + require.NoError(t, err) + require.Len(t, queries, 1) + assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) +} diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index 56c98ad104..8a6eabeafd 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -138,13 +138,13 @@ func (d *Datastore) SaveQuery(ctx context.Context, q *fleet.Query) error { // DeleteQuery deletes Query identified by Query.ID. func (d *Datastore) DeleteQuery(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "queries", name) + return d.deleteEntityByName(ctx, queriesTable, name) } // DeleteQueries deletes the existing query objects with the provided IDs. The // number of deleted queries is returned along with any error. func (d *Datastore) DeleteQueries(ctx context.Context, ids []uint) (uint, error) { - return d.deleteEntities(ctx, "queries", ids) + return d.deleteEntities(ctx, queriesTable, ids) } // Query returns a single Query identified by id, if such exists. @@ -170,7 +170,7 @@ func (d *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { // ListQueries returns a list of queries with sort order and results limit // determined by passed in fleet.ListOptions -func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { +func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { sql := ` SELECT q.*, COALESCE(u.name, '') AS author_name FROM queries q @@ -178,7 +178,11 @@ func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]* ON q.author_id = u.id WHERE saved = true ` - sql = appendListOptionsToSQL(sql, opt) + if opt.OnlyObserverCanRun { + sql += " AND q.observer_can_run=true" + } + sql = appendListOptionsToSQL(sql, opt.ListOptions) + results := []*fleet.Query{} if err := sqlx.SelectContext(ctx, d.reader, &results, sql); err != nil { diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index 593193c8b7..5217522ff5 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -12,10 +12,31 @@ import ( "github.com/stretchr/testify/require" ) -func TestApplyQueries(t *testing.T) { +func TestQueries(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Apply", testQueriesApply}, + {"Delete", testQueriesDelete}, + {"GetByName", testQueriesGetByName}, + {"DeleteMany", testQueriesDeleteMany}, + {"Save", testQueriesSave}, + {"List", testQueriesList}, + {"LoadPacksForQueries", testQueriesLoadPacksForQueries}, + {"DuplicateNew", testQueriesDuplicateNew}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testQueriesApply(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) @@ -29,7 +50,7 @@ func TestApplyQueries(t *testing.T) { err := ds.ApplyQueries(context.Background(), zwass.ID, expectedQueries) require.Nil(t, err) - queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -47,7 +68,7 @@ func TestApplyQueries(t *testing.T) { err = ds.ApplyQueries(context.Background(), groob.ID, expectedQueries) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -65,7 +86,7 @@ func TestApplyQueries(t *testing.T) { err = ds.ApplyQueries(context.Background(), zwass.ID, []*fleet.Query{expectedQueries[2]}) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -79,10 +100,7 @@ func TestApplyQueries(t *testing.T) { assert.Equal(t, &zwass.ID, queries[2].AuthorID) } -func TestDeleteQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDelete(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -103,10 +121,7 @@ func TestDeleteQuery(t *testing.T) { assert.NotNil(t, err) } -func TestGetQueryByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesGetByName(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) actual, err := ds.QueryByName(context.Background(), "q1") @@ -119,10 +134,7 @@ func TestGetQueryByName(t *testing.T) { assert.True(t, fleet.IsNotFound(err)) } -func TestDeleteQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDeleteMany(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) q1 := test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) @@ -130,7 +142,7 @@ func TestDeleteQueries(t *testing.T) { q3 := test.NewQuery(t, ds, "q3", "select 1", user.ID, true) q4 := test.NewQuery(t, ds, "q4", "select * from osquery_info", user.ID, true) - queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 4) @@ -138,7 +150,7 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(2), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 2) @@ -146,7 +158,7 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 1) @@ -154,16 +166,12 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 0) - } -func TestSaveQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesSave(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -190,10 +198,7 @@ func TestSaveQuery(t *testing.T) { assert.True(t, queryVerify.ObserverCanRun) } -func TestListQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesList(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) for i := 0; i < 10; i++ { @@ -215,16 +220,13 @@ func TestListQuery(t *testing.T) { }) require.Nil(t, err) - opts := fleet.ListOptions{} + opts := fleet.ListQueryOptions{} results, err := ds.ListQueries(context.Background(), opts) assert.Nil(t, err) assert.Equal(t, 10, len(results)) } -func TestLoadPacksForQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "q1", Query: "select * from time"}, @@ -234,9 +236,9 @@ func TestLoadPacksForQueries(t *testing.T) { require.Nil(t, err) specs := []*fleet.PackSpec{ - &fleet.PackSpec{Name: "p1"}, - &fleet.PackSpec{Name: "p2"}, - &fleet.PackSpec{Name: "p3"}, + {Name: "p1"}, + {Name: "p2"}, + {Name: "p3"}, } err = ds.ApplyPackSpecs(context.Background(), specs) require.Nil(t, err) @@ -250,10 +252,10 @@ func TestLoadPacksForQueries(t *testing.T) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p2", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { Name: "q0", QueryName: queries[0].Name, Interval: 60, @@ -275,19 +277,19 @@ func TestLoadPacksForQueries(t *testing.T) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p1", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { QueryName: queries[1].Name, Interval: 60, }, }, }, - &fleet.PackSpec{ + { Name: "p3", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { QueryName: queries[1].Name, Interval: 60, }, @@ -312,15 +314,15 @@ func TestLoadPacksForQueries(t *testing.T) { } specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p3", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { Name: "q0", QueryName: queries[0].Name, Interval: 60, }, - fleet.PackSpecQuery{ + { Name: "q1", QueryName: queries[1].Name, Interval: 60, @@ -348,10 +350,7 @@ func TestLoadPacksForQueries(t *testing.T) { } } -func TestDuplicateNewQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDuplicateNew(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Mike Arpaia", "mike@fleet.co", true) q1, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "foo", @@ -370,3 +369,40 @@ func TestDuplicateNewQuery(t *testing.T) { // is private to the individual datastore implementations assert.Contains(t, err.Error(), "already exists") } + +func TestListQueryFiltersObserver(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + _, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + query3, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query3", + Query: "select 1;", + Saved: true, + ObserverCanRun: true, + }) + require.NoError(t, err) + + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + require.NoError(t, err) + require.Len(t, queries, 3) + + queries, err = ds.ListQueries( + context.Background(), + fleet.ListQueryOptions{OnlyObserverCanRun: true, ListOptions: fleet.ListOptions{PerPage: 1}}, + ) + require.NoError(t, err) + require.Len(t, queries, 1) + assert.Equal(t, query3.ID, queries[0].ID) +} diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 8605f60b35..560ce5bec9 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -123,7 +123,7 @@ func saveScheduledQueryDB(ctx context.Context, exec sqlx.ExecerContext, sq *flee } func (d *Datastore) DeleteScheduledQuery(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "scheduled_queries", id) + return d.deleteEntity(ctx, scheduledQueriesTable, id) } func (d *Datastore) ScheduledQuery(ctx context.Context, id uint) (*fleet.ScheduledQuery, error) { diff --git a/server/datastore/mysql/scheduled_queries_test.go b/server/datastore/mysql/scheduled_queries_test.go index 8345c08118..a6268c665b 100644 --- a/server/datastore/mysql/scheduled_queries_test.go +++ b/server/datastore/mysql/scheduled_queries_test.go @@ -12,10 +12,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestListScheduledQueriesInPack(t *testing.T) { +func TestScheduledQueries(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"ListInPack", testScheduledQueriesListInPack}, + {"New", testScheduledQueriesNew}, + {"Get", testScheduledQueriesGet}, + {"Delete", testScheduledQueriesDelete}, + {"CascadingDelete", testScheduledQueriesCascadingDelete}, + {"CleanupOrphanStats", testScheduledQueriesCleanupOrphanStats}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testScheduledQueriesListInPack(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -81,10 +100,7 @@ func TestListScheduledQueriesInPack(t *testing.T) { require.Len(t, gotQueries, 3) } -func TestNewScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesNew(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -100,10 +116,7 @@ func TestNewScheduledQuery(t *testing.T) { assert.Equal(t, "select * from time;", query.Query) } -func TestScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesGet(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -127,10 +140,7 @@ func TestScheduledQuery(t *testing.T) { assert.False(t, *query.Denylist) } -func TestDeleteScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesDelete(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -147,10 +157,7 @@ func TestDeleteScheduledQuery(t *testing.T) { require.NotNil(t, err) } -func TestCascadingDeletionOfQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesCascadingDelete(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -199,10 +206,7 @@ func TestCascadingDeletionOfQueries(t *testing.T) { require.Len(t, gotQueries, 1) } -func TestCleanupOrphanScheduledQueryStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesCleanupOrphanStats(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 8db6f3112e..00f3bea508 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -221,6 +221,7 @@ CREATE TABLE `hosts` ( `team_id` int(10) unsigned DEFAULT NULL, `gigs_disk_space_available` float NOT NULL DEFAULT '0', `percent_disk_space_available` float NOT NULL DEFAULT '0', + `policy_updated_at` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', PRIMARY KEY (`id`), UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`), UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), @@ -268,9 +269,7 @@ CREATE TABLE `label_membership` ( `label_id` int(10) unsigned NOT NULL, `host_id` int(10) unsigned NOT NULL, PRIMARY KEY (`host_id`,`label_id`), - KEY `idx_lm_label_id` (`label_id`), - CONSTRAINT `fk_lm_host_id` FOREIGN KEY (`host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `fk_lm_label_id` FOREIGN KEY (`label_id`) REFERENCES `labels` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + KEY `idx_lm_label_id` (`label_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; @@ -310,9 +309,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=102 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=105 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -400,9 +399,12 @@ CREATE TABLE `policies` ( `query_id` int(10) unsigned NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `team_id` int(10) unsigned DEFAULT NULL, PRIMARY KEY (`id`), KEY `fk_policies_query_id` (`query_id`), - CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`) + KEY `fk_policies_team_id` (`team_id`), + CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`), + CONSTRAINT `policies_ibfk_2` FOREIGN KEY (`team_id`) REFERENCES `teams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; SET @saved_cs_client = @@character_set_client; diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index 16fb65c8fb..02e8273beb 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -70,7 +70,7 @@ func (d *Datastore) NewSession(ctx context.Context, session *fleet.Session) (*fl } func (d *Datastore) DestroySession(ctx context.Context, session *fleet.Session) error { - err := d.deleteEntity(ctx, "sessions", session.ID) + err := d.deleteEntity(ctx, sessionsTable, session.ID) if err != nil { return errors.Wrap(err, "deleting session") } diff --git a/server/datastore/mysql/sessions_test.go b/server/datastore/mysql/sessions_test.go index 14c225a8a8..446052e653 100644 --- a/server/datastore/mysql/sessions_test.go +++ b/server/datastore/mysql/sessions_test.go @@ -10,10 +10,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestSessionGetters(t *testing.T) { +func TestSessions(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Getters", testSessionsGetters}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testSessionsGetters(t *testing.T, ds *Datastore) { user, err := ds.NewUser(context.Background(), &fleet.User{ Password: []byte("supersecret"), Email: "other@bobcom", diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 04952635bf..c8d754e526 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -332,11 +332,18 @@ func (d *Datastore) AllSoftwareWithoutCPEIterator(ctx context.Context) (fleet.So } func (d *Datastore) AddCPEForSoftware(ctx context.Context, software fleet.Software, cpe string) error { + _, err := addCPEForSoftwareDB(ctx, d.writer, software, cpe) + return err +} + +func addCPEForSoftwareDB(ctx context.Context, exec sqlx.ExecerContext, software fleet.Software, cpe string) (uint, error) { sql := `INSERT INTO software_cpe (software_id, cpe) VALUES (?, ?)` - if _, err := d.writer.ExecContext(ctx, sql, software.ID, cpe); err != nil { - return errors.Wrap(err, "insert software cpe") + res, err := exec.ExecContext(ctx, sql, software.ID, cpe) + if err != nil { + return 0, errors.Wrap(err, "insert software cpe") } - return nil + id, _ := res.LastInsertId() // cannot fail with the mysql driver + return uint(id), nil } func (d *Datastore) AllCPEs(ctx context.Context) ([]string, error) { diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 9e52730939..099bdeb5a1 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -11,14 +11,37 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/test" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestSaveHostSoftware(t *testing.T) { +func TestSoftware(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"SaveHost", testSoftwareSaveHost}, + {"CPE", testSoftwareCPE}, + {"InsertCVEs", testSoftwareInsertCVEs}, + {"HostDuplicates", testSoftwareHostDuplicates}, + {"LoadVulnerabilities", testSoftwareLoadVulnerabilities}, + {"AllCPEs", testSoftwareAllCPEs}, + {"NothingChanged", testSoftwareNothingChanged}, + {"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs}, + {"List", testSoftwareList}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testSoftwareSaveHost(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) @@ -103,10 +126,7 @@ func TestSaveHostSoftware(t *testing.T) { test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software) } -func TestSoftwareCPE(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareCPE(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft1 := fleet.HostSoftware{ @@ -176,10 +196,7 @@ func TestSoftwareCPE(t *testing.T) { require.NoError(t, iterator.Close()) } -func TestInsertCVEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -197,10 +214,7 @@ func TestInsertCVEs(t *testing.T) { require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) } -func TestHostSoftwareDuplicates(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) longName := strings.Repeat("a", 260) @@ -232,10 +246,7 @@ func TestHostSoftwareDuplicates(t *testing.T) { require.NoError(t, tx.Commit()) } -func TestLoadSoftwareVulnerabilities(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -270,10 +281,7 @@ func TestLoadSoftwareVulnerabilities(t *testing.T) { require.Len(t, host.Software[1].Vulnerabilities, 0) } -func TestAllCPEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareAllCPEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -296,7 +304,7 @@ func TestAllCPEs(t *testing.T) { assert.ElementsMatch(t, cpes, []string{"somecpe", "someothercpewithoutvulns"}) } -func TestNothingChanged(t *testing.T) { +func testSoftwareNothingChanged(t *testing.T, ds *Datastore) { assert.False(t, nothingChanged(nil, []fleet.Software{{}})) assert.True(t, nothingChanged(nil, nil)) assert.True(t, nothingChanged( @@ -313,10 +321,7 @@ func TestNothingChanged(t *testing.T) { )) } -func TestLoadSupportsTonsOfCVEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -332,15 +337,26 @@ func TestLoadSupportsTonsOfCVEs(t *testing.T) { require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) sort.Slice(host.Software, func(i, j int) bool { return host.Software[i].Name < host.Software[j].Name }) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) - for i := 0; i < 1000; i++ { - part1 := rand.Intn(1000) - part2 := rand.Intn(1000) - part3 := rand.Intn(1000) - cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), cve, []string{"somecpe"})) - } + + require.NoError(t, ds.withTx(context.Background(), func(tx sqlx.ExtContext) error { + somecpeID, err := addCPEForSoftwareDB(context.Background(), tx, host.Software[0], "somecpe") + if err != nil { + return err + } + sql := `INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES (?, ?)` + for i := 0; i < 1000; i++ { + part1 := rand.Intn(1000) + part2 := rand.Intn(1000) + part3 := rand.Intn(1000) + cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) + if _, err := tx.ExecContext(context.Background(), sql, somecpeID, cve); err != nil { + return err + } + } + return nil + })) require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) @@ -363,10 +379,7 @@ func TestLoadSupportsTonsOfCVEs(t *testing.T) { } } -func TestListSoftware(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareList(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index 201175e7c2..34583b0ab7 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -10,13 +10,28 @@ import ( "github.com/stretchr/testify/require" ) -func TestShouldSendStatistics(t *testing.T) { +func TestStatistics(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"ShouldSend", testStatisticsShouldSend}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -48,6 +63,7 @@ func TestShouldSendStatistics(t *testing.T) { _, err = ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index dabd73a76d..1f2cd2000c 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -14,10 +14,27 @@ import ( "github.com/stretchr/testify/require" ) -func TestCountHostsInTargets(t *testing.T) { +func TestTargets(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"CountHosts", testTargetsCountHosts}, + {"HostStatus", testTargetsHostStatus}, + {"HostIDsInTargets", testTargetsHostIDsInTargets}, + {"HostIDsInTargetsTeam", testTargetsHostIDsInTargetsTeam}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testTargetsCountHosts(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -30,6 +47,7 @@ func TestCountHostsInTargets(t *testing.T) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, @@ -156,13 +174,9 @@ func TestCountHostsInTargets(t *testing.T) { assert.Equal(t, uint(0), metrics.OnlineHosts) assert.Equal(t, uint(5), metrics.OfflineHosts) assert.Equal(t, uint(1), metrics.MissingInActionHosts) - } -func TestHostStatus(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostStatus(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) mockClock := clock.NewMockClock() @@ -222,10 +236,7 @@ func TestHostStatus(t *testing.T) { } } -func TestHostIDsInTargets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -237,6 +248,7 @@ func TestHostIDsInTargets(t *testing.T) { NodeKey: strconv.Itoa(hostCount), DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) @@ -302,10 +314,7 @@ func TestHostIDsInTargets(t *testing.T) { assert.Equal(t, []uint{1, 3, 4, 5, 6}, ids) } -func TestHostIDsInTargetsTeam(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -318,6 +327,7 @@ func TestHostIDsInTargetsTeam(t *testing.T) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, diff --git a/server/datastore/mysql/teams.go b/server/datastore/mysql/teams.go index 122a734cc5..183f6c4dac 100644 --- a/server/datastore/mysql/teams.go +++ b/server/datastore/mysql/teams.go @@ -80,7 +80,7 @@ func saveTeamSecretsDB(ctx context.Context, exec sqlx.ExecerContext, team *fleet } func (d *Datastore) DeleteTeam(ctx context.Context, tid uint) error { - if err := d.deleteEntity(ctx, "teams", tid); err != nil { + if err := d.deleteEntity(ctx, teamsTable, tid); err != nil { return errors.Wrapf(err, "delete team id %d", tid) } return nil diff --git a/server/datastore/mysql/teams_test.go b/server/datastore/mysql/teams_test.go index 21bb004915..744299f0b9 100644 --- a/server/datastore/mysql/teams_test.go +++ b/server/datastore/mysql/teams_test.go @@ -13,10 +13,28 @@ import ( "github.com/stretchr/testify/require" ) -func TestTeamGetSetDelete(t *testing.T) { +func TestTeams(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"GetSetDelete", testTeamsGetSetDelete}, + {"Users", testTeamsUsers}, + {"List", testTeamsList}, + {"Search", testTeamsSearch}, + {"EnrollSecrets", testTeamsEnrollSecrets}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testTeamsGetSetDelete(t *testing.T, ds *Datastore) { var createTests = []struct { name, description string }{ @@ -25,7 +43,7 @@ func TestTeamGetSetDelete(t *testing.T) { } for _, tt := range createTests { - t.Run("", func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { team, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: tt.name, Description: tt.description, @@ -52,10 +70,7 @@ func TestTeamGetSetDelete(t *testing.T) { } } -func TestTeamUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsUsers(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID} @@ -111,10 +126,7 @@ func TestTeamUsers(t *testing.T) { assert.ElementsMatch(t, team2Users, team2.Users) } -func TestTeamListTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsList(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID, GlobalRole: ptr.String(fleet.RoleAdmin)} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID, GlobalRole: ptr.String(fleet.RoleObserver)} @@ -168,10 +180,7 @@ func TestTeamListTeams(t *testing.T) { assert.Equal(t, 1, teams[1].UserCount) } -func TestTeamSearchTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsSearch(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -207,10 +216,7 @@ func TestTeamSearchTeams(t *testing.T) { assert.Len(t, teams, 0) } -func TestTeamEnrollSecrets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsEnrollSecrets(t *testing.T, ds *Datastore) { secrets := []*fleet.EnrollSecret{{Secret: "secret1"}, {Secret: "secret2"}} team1, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: "team1", diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 6d84ca5e8a..e90a9a3302 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -233,7 +233,9 @@ func createMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto strings.TrimPrefix(details.Name(), "github.com/fleetdm/fleet/v4/"), "/", "_", ) cleanName = strings.ReplaceAll(cleanName, ".", "_") - return initializeDatabase(t, cleanName, opts) + ds := initializeDatabase(t, cleanName, opts) + t.Cleanup(func() { ds.Close() }) + return ds } func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datastore { @@ -243,3 +245,71 @@ func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto func CreateMySQLDS(t *testing.T) *Datastore { return createMySQLDSWithOptions(t, nil) } + +func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { + if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { + t.Skip("MySQL tests are disabled") + } + + t.Parallel() + ds := initializeDatabase(t, name, new(DatastoreTestOptions)) + t.Cleanup(func() { ds.Close() }) + return ds +} + +// TruncateTables truncates the specified tables, in order, using ds.writer. +// Note that the order is typically not important because FK checks are +// disabled while truncating. If no table is provided, all tables (except +// those that are seeded by the SQL schema file) are truncated. +func TruncateTables(t *testing.T, ds *Datastore, tables ...string) { + // those tables are seeded with the schema.sql and as such must not + // be truncated - a more precise approach must be used for those, e.g. + // delete where id > max before test, or something like that. + nonEmptyTables := map[string]bool{ + "app_config_json": true, + "app_configs": true, + "migration_status_tables": true, + "osquery_options": true, + } + + ctx := context.Background() + + require.NoError(t, ds.withTx(ctx, func(tx sqlx.ExtContext) error { + var skipSeeded bool + + if len(tables) == 0 { + skipSeeded = true + sql := ` + SELECT + table_name + FROM + information_schema.tables + WHERE + table_schema = database() AND + table_type = 'BASE TABLE' + ` + if err := sqlx.SelectContext(ctx, tx, &tables, sql); err != nil { + return err + } + } + + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=0`); err != nil { + return err + } + for _, tbl := range tables { + if nonEmptyTables[tbl] { + if skipSeeded { + continue + } + return fmt.Errorf("cannot truncate table %s, it contains seed data from schema.sql", tbl) + } + if _, err := tx.ExecContext(ctx, "TRUNCATE TABLE "+tbl); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=1`); err != nil { + return err + } + return nil + })) +} diff --git a/server/datastore/mysql/unicode_test.go b/server/datastore/mysql/unicode_test.go index 64bfb3adb7..a06a991bfa 100644 --- a/server/datastore/mysql/unicode_test.go +++ b/server/datastore/mysql/unicode_test.go @@ -33,6 +33,7 @@ func TestUnicode(t *testing.T) { Hostname: "🍌", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go index 29399b1114..4f801295ca 100644 --- a/server/datastore/mysql/users.go +++ b/server/datastore/mysql/users.go @@ -267,5 +267,5 @@ func saveTeamsForUserDB(ctx context.Context, tx sqlx.ExtContext, user *fleet.Use // DeleteUser deletes the associated user func (d *Datastore) DeleteUser(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "users", id) + return d.deleteEntity(ctx, usersTable, id) } diff --git a/server/datastore/mysql/users_test.go b/server/datastore/mysql/users_test.go index 86562c97f6..240e2a338e 100644 --- a/server/datastore/mysql/users_test.go +++ b/server/datastore/mysql/users_test.go @@ -15,10 +15,30 @@ import ( "github.com/stretchr/testify/require" ) -func TestCreateUser(t *testing.T) { +func TestUsers(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Create", testUsersCreate}, + {"ByID", testUsersByID}, + {"Save", testUsersSave}, + {"List", testUsersList}, + {"Teams", testUsersTeams}, + {"CreateWithTeams", testUsersCreateWithTeams}, + {"SaveMany", testUsersSaveMany}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testUsersCreate(t *testing.T, ds *Datastore) { var createTests = []struct { password, email string isAdmin, passwordReset, sso bool @@ -48,10 +68,7 @@ func TestCreateUser(t *testing.T) { } } -func TestUserByID(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersByID(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) for _, tt := range users { returned, err := ds.UserByID(context.Background(), tt.ID) @@ -92,10 +109,7 @@ func createTestUsers(t *testing.T, ds fleet.Datastore) []*fleet.User { return users } -func TestSaveUser(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersSave(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) testUserGlobalRole(t, ds, users) testEmailAttribute(t, ds, users) @@ -150,10 +164,7 @@ func testUserGlobalRole(t *testing.T, ds fleet.Datastore, users []*fleet.User) { assert.Equal(t, "Cannot specify both Global Role and Team Roles", flErr.Message) } -func TestListUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersList(t *testing.T, ds *Datastore) { createTestUsers(t, ds) users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) @@ -171,10 +182,7 @@ func TestListUsers(t *testing.T) { assert.Equal(t, "mike@fleet.co", users[0].Email) } -func TestUserTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersTeams(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -264,10 +272,7 @@ func TestUserTeams(t *testing.T) { assert.Len(t, users[1].Teams, 0) } -func TestUserCreateWithTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersCreateWithTeams(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -306,10 +311,7 @@ func TestUserCreateWithTeams(t *testing.T) { assert.Equal(t, "maintainer", user.Teams[2].Role) } -func TestSaveUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersSaveMany(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, t.Name()+"Admin1", t.Name()+"admin1@fleet.co", true) u2 := test.NewUser(t, ds, t.Name()+"Admin2", t.Name()+"admin2@fleet.co", true) u3 := test.NewUser(t, ds, t.Name()+"Admin3", t.Name()+"admin3@fleet.co", true) diff --git a/server/datastore/redis/redis.go b/server/datastore/redis/redis.go index d9046af6e8..3679b7a61a 100644 --- a/server/datastore/redis/redis.go +++ b/server/datastore/redis/redis.go @@ -1,9 +1,11 @@ package redis import ( + "net" "strings" "time" + "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" @@ -17,29 +19,64 @@ type standalonePool struct { addr string } +func (p *standalonePool) ConfigureDoer(conn redis.Conn) redis.Conn { + return conn +} + func (p *standalonePool) Stats() map[string]redis.PoolStats { return map[string]redis.PoolStats{ p.addr: p.Pool.Stats(), } } +type clusterPool struct { + *redisc.Cluster + followRedirs bool +} + +// ConfigureDoer configures conn to follow redirections if the redis +// configuration requested it. If the conn is already in error, or +// if it is not a redisc cluster connection, it is returned unaltered. +func (p *clusterPool) ConfigureDoer(conn redis.Conn) redis.Conn { + if err := conn.Err(); err == nil && p.followRedirs { + rc, err := redisc.RetryConn(conn, 3, 300*time.Millisecond) + if err == nil { + return rc + } + } + return conn +} + +// PoolConfig holds the redis pool configuration options. +type PoolConfig struct { + Server string + Password string + Database int + UseTLS bool + ConnTimeout time.Duration + KeepAlive time.Duration + ConnectRetryAttempts int + ClusterFollowRedirections bool + + // allows for testing dial retries and other dial-related scenarios + testRedisDialFunc func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) +} + // NewRedisPool creates a Redis connection pool using the provided server // address, password and database. -func NewRedisPool( - server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration, -) (fleet.RedisPool, error) { - cluster := newCluster(server, password, database, useTLS, connTimeout, keepAlive) +func NewRedisPool(config PoolConfig) (fleet.RedisPool, error) { + cluster := newCluster(config) if err := cluster.Refresh(); err != nil { if isClusterDisabled(err) || isClusterCommandUnknown(err) { // not a Redis Cluster setup, use a standalone Redis pool - pool, _ := cluster.CreatePool(server) + pool, _ := cluster.CreatePool(config.Server) cluster.Close() - return &standalonePool{pool, server}, nil + return &standalonePool{pool, config.Server}, nil } return nil, errors.Wrap(err, "refresh cluster") } - return cluster, nil + return &clusterPool{cluster, config.ClusterFollowRedirections}, nil } // SplitRedisKeysBySlot takes a list of redis keys and groups them by hash slot @@ -49,7 +86,7 @@ func NewRedisPool( // simply returns all keys in the same group (i.e. the top-level slice has a // length of 1). func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { - if _, isCluster := pool.(*redisc.Cluster); isCluster { + if _, isCluster := pool.(*clusterPool); isCluster { return redisc.SplitBySlot(keys...) } return [][]string{keys} @@ -61,7 +98,7 @@ func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { // of nodes stops and EachRedisNode returns that error. For standalone redis, // fn is called only once. func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { - if cluster, isCluster := pool.(*redisc.Cluster); isCluster { + if cluster, isCluster := pool.(*clusterPool); isCluster { return cluster.EachNode(false, func(_ string, conn redis.Conn) error { return fn(conn) }) @@ -72,35 +109,64 @@ func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { return fn(conn) } -func newCluster(server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration) *redisc.Cluster { +func newCluster(config PoolConfig) *redisc.Cluster { + opts := []redis.DialOption{ + redis.DialDatabase(config.Database), + redis.DialUseTLS(config.UseTLS), + redis.DialConnectTimeout(config.ConnTimeout), + redis.DialKeepAlive(config.KeepAlive), + // Read/Write timeouts not set here because we may see results + // only rarely on the pub/sub channel. + } + if config.Password != "" { + opts = append(opts, redis.DialPassword(config.Password)) + } + + dialFn := redis.Dial + if config.testRedisDialFunc != nil { + dialFn = config.testRedisDialFunc + } + return &redisc.Cluster{ - StartupNodes: []string{server}, - CreatePool: func(server string, opts ...redis.DialOption) (*redis.Pool, error) { + StartupNodes: []string{config.Server}, + CreatePool: func(server string, _ ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ MaxIdle: 3, IdleTimeout: 240 * time.Second, + Dial: func() (redis.Conn, error) { - c, err := redis.Dial( - "tcp", - server, - redis.DialDatabase(database), - redis.DialUseTLS(useTLS), - redis.DialConnectTimeout(connTimeout), - redis.DialKeepAlive(keepAlive), - // Read/Write timeouts not set here because we may see results - // only rarely on the pub/sub channel. - ) - if err != nil { - return nil, err + var conn redis.Conn + op := func() error { + c, err := dialFn("tcp", server, opts...) + + var netErr net.Error + if errors.As(err, &netErr) { + if netErr.Temporary() || netErr.Timeout() { + // retryable error + return err + } + } + if err != nil { + // at this point, this is a non-retryable error + return backoff.Permanent(err) + } + + // success, store the connection to use + conn = c + return nil } - if password != "" { - if _, err := c.Do("AUTH", password); err != nil { - c.Close() + + if config.ConnectRetryAttempts > 0 { + boff := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(config.ConnectRetryAttempts)) + if err := backoff.Retry(op, boff); err != nil { return nil, err } + } else if err := op(); err != nil { + return nil, err } - return c, err + return conn, nil }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Minute { return nil diff --git a/server/datastore/redis/redis_test.go b/server/datastore/redis/redis_test.go index ad3da7d6b5..30b55a3c16 100644 --- a/server/datastore/redis/redis_test.go +++ b/server/datastore/redis/redis_test.go @@ -2,15 +2,171 @@ package redis import ( "fmt" + "io" + "runtime" "testing" "time" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" + "github.com/pkg/errors" "github.com/stretchr/testify/require" ) +type netError struct { + error + timeout bool + temporary bool + allowedCalls int // once this reaches 0, mockDial does not return an error + countCalls int +} + +func (t *netError) Timeout() bool { return t.timeout } +func (t *netError) Temporary() bool { return t.temporary } + +var errFromConn = errors.New("SUCCESS") + +type redisConn struct{} + +func (redisConn) Close() error { return errFromConn } +func (redisConn) Err() error { return errFromConn } +func (redisConn) Do(_ string, _ ...interface{}) (interface{}, error) { return nil, errFromConn } +func (redisConn) Send(_ string, _ ...interface{}) error { return errFromConn } +func (redisConn) Flush() error { return errFromConn } +func (redisConn) Receive() (interface{}, error) { return nil, errFromConn } + +func TestConnectRetry(t *testing.T) { + mockDial := func(err error) func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { + return func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { + var ne *netError + if errors.As(err, &ne) { + ne.countCalls++ + if ne.allowedCalls <= 0 { + return redisConn{}, nil + } + ne.allowedCalls-- + } + return nil, err + } + } + + cases := []struct { + err error + retries int + wantCalls int + min, max time.Duration + }{ + // the min-max time intervals are based on the backoff default configuration as + // used in the Dial func of the redis pool. It starts with 500ms interval, + // multiplies by 1.5 on each attempt, and has a randomization of 0.5 that must + // be accounted for. Example ranges of intervals are given at + // https://github.com/fleetdm/fleet/pull/1962#issue-729635664 + // and were used to calculate the (approximate) expected range. + { + io.EOF, 0, 1, 0, 100 * time.Millisecond, + }, // non-retryable, no retry configured + { + &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 0, 1, 0, 100 * time.Millisecond, + }, // retryable, but no retry configured + { + io.EOF, 3, 1, 0, 100 * time.Millisecond, + }, // non-retryable, retry configured + { + &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, + }, // retryable, retry configured + { + &netError{error: io.EOF, temporary: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, + }, // retryable, retry configured + { + &netError{error: io.EOF, allowedCalls: 10}, 2, 1, 0, 100 * time.Millisecond, + }, // net error, but non-retryable + { + &netError{error: io.EOF, timeout: true, allowedCalls: 1}, 10, 2, 250 * time.Millisecond, 750 * time.Millisecond, + }, // retryable, but succeeded after one retry + } + for _, c := range cases { + t.Run(c.err.Error(), func(t *testing.T) { + start := time.Now() + _, err := NewRedisPool(PoolConfig{ + Server: "127.0.0.1:12345", + ConnectRetryAttempts: c.retries, + testRedisDialFunc: mockDial(c.err), + }) + diff := time.Since(start) + require.GreaterOrEqual(t, diff, c.min) + require.LessOrEqual(t, diff, c.max) + require.Error(t, err) + + wantErr := io.EOF + var ne *netError + if errors.As(c.err, &ne) { + require.Equal(t, c.wantCalls, ne.countCalls) + if ne.allowedCalls == 0 { + wantErr = errFromConn + } + } else { + require.Equal(t, c.wantCalls, 1) + } + + // the error is returned as part of the cluster.Refresh error, hence the + // check with Contains. + require.Contains(t, err.Error(), wantErr.Error()) + }) + } +} + +func TestRedisPoolConfigureDoer(t *testing.T) { + const prefix = "TestRedisPoolConfigureDoer:" + + t.Run("standalone", func(t *testing.T) { + pool, teardown := setupRedisForTest(t, false, false) + defer teardown() + + c1 := pool.Get() + defer c1.Close() + c2 := pool.ConfigureDoer(pool.Get()) + defer c2.Close() + + // both conns work equally well, get nil because keys do not exist, + // but no redirection error (this is standalone redis). + _, err := redis.String(c1.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c1.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + + _, err = redis.String(c2.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c2.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + }) + + t.Run("cluster", func(t *testing.T) { + pool, teardown := setupRedisForTest(t, true, true) + defer teardown() + + c1 := pool.Get() + defer c1.Close() + c2 := pool.ConfigureDoer(pool.Get()) + defer c2.Close() + + // unconfigured conn gets MOVED error on the second key + // (it is bound to {a}, {b} is on a different node) + _, err := redis.String(c1.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c1.Do("GET", prefix+"{b}")) + rerr := redisc.ParseRedir(err) + require.Error(t, rerr) + require.Equal(t, "MOVED", rerr.Type) + + // configured conn gets the nil value, it redirected automatically + _, err = redis.String(c2.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c2.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + }) +} + func TestEachRedisNode(t *testing.T) { const prefix = "TestEachRedisNode:" @@ -49,19 +205,23 @@ func TestEachRedisNode(t *testing.T) { } t.Run("standalone", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, false) + pool, teardown := setupRedisForTest(t, false, false) defer teardown() runTest(t, pool) }) t.Run("cluster", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, true) + pool, teardown := setupRedisForTest(t, true, false) defer teardown() runTest(t, pool) }) } -func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardown func()) { +func setupRedisForTest(t *testing.T, cluster, redir bool) (pool fleet.RedisPool, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -74,7 +234,15 @@ func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardo } addr += port - pool, err := NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := NewRedisPool(PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + ClusterFollowRedirections: redir, + }) require.NoError(t, err) conn := pool.Get() diff --git a/server/fleet/app.go b/server/fleet/app.go index 043dc0259d..93f7644607 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -243,6 +243,12 @@ type ListOptions struct { MatchQuery string } +type ListQueryOptions struct { + ListOptions + + OnlyObserverCanRun bool +} + // EnrollSecret contains information about an enroll secret, name, and active // status. Enroll secrets are used for osquery authentication. type EnrollSecret struct { @@ -316,6 +322,7 @@ type Logging struct { type UpdateIntervalConfig struct { OSQueryDetail time.Duration `json:"osquery_detail"` + OSQueryPolicy time.Duration `json:"osquery_policy"` } type LoggingPlugin struct { diff --git a/server/fleet/authz.go b/server/fleet/authz.go index 94cda02869..118dcce871 100644 --- a/server/fleet/authz.go +++ b/server/fleet/authz.go @@ -11,4 +11,6 @@ const ( ActionWriteRole = "write_role" // ActionRun is the action for running a live query. ActionRun = "run" + // ActionRunNew is the action for running a new live query. + ActionRunNew = "run_new" ) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index c333c010a4..50b960b84a 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -61,7 +61,7 @@ type Datastore interface { Query(ctx context.Context, id uint) (*Query, error) // ListQueries returns a list of queries with the provided sorting and paging options. Associated packs should also // be loaded. - ListQueries(ctx context.Context, opt ListOptions) ([]*Query, error) + ListQueries(ctx context.Context, opt ListQueryOptions) ([]*Query, error) // QueryByName looks up a query by name. QueryByName(ctx context.Context, name string, opts ...OptionalArg) (*Query, error) @@ -139,10 +139,9 @@ type Datastore interface { Label(ctx context.Context, lid uint) (*Label, error) ListLabels(ctx context.Context, filter TeamFilter, opt ListOptions) ([]*Label, error) - // LabelQueriesForHost returns the label queries that should be executed for the given host. The cutoff is the - // minimum timestamp a query execution should have to be considered "fresh". Executions that are not fresh will be - // repeated. Results are returned in a map of label id -> query - LabelQueriesForHost(ctx context.Context, host *Host, cutoff time.Time) (map[string]string, error) + // LabelQueriesForHost returns the label queries that should be executed for the given host. + // Results are returned in a map of label id -> query + LabelQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) // RecordLabelQueryExecutions saves the results of label queries. The results map is a map of label id -> whether or // not the label matches. The time parameter is the timestamp to save with the query execution. @@ -287,6 +286,7 @@ type Datastore interface { DeleteScheduledQuery(ctx context.Context, id uint) error ScheduledQuery(ctx context.Context, id uint) (*ScheduledQuery, error) CleanupOrphanScheduledQueryStats(ctx context.Context) error + CleanupOrphanLabelMembership(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// // TeamStore @@ -331,7 +331,8 @@ type Datastore interface { RecordStatisticsSent(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// - // GlobalPoliciesStore interface { + // GlobalPoliciesStore + NewGlobalPolicy(ctx context.Context, queryID uint) (*Policy, error) Policy(ctx context.Context, id uint) (*Policy, error) RecordPolicyQueryExecutions(ctx context.Context, host *Host, results map[uint]*bool, updated time.Time) error @@ -349,6 +350,14 @@ type Datastore interface { MigrationStatus(ctx context.Context) (MigrationStatus, error) ListSoftware(ctx context.Context, teamId *uint, opt ListOptions) ([]Software, error) + + /////////////////////////////////////////////////////////////////////////////// + // Team Policies + + NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) + ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) + DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } type MigrationStatus int diff --git a/server/fleet/global_policies.go b/server/fleet/global_policies.go index e41bacb87d..7034f0fd2b 100644 --- a/server/fleet/global_policies.go +++ b/server/fleet/global_policies.go @@ -6,10 +6,11 @@ type Policy struct { QueryName string `json:"query_name" db:"query_name"` PassingHostCount uint `json:"passing_host_count" db:"passing_host_count"` FailingHostCount uint `json:"failing_host_count" db:"failing_host_count"` + TeamID *uint `json:"team_id" db:"team_id"` UpdateCreateTimestamps } -func (Policy) AuthzType() string { +func (p Policy) AuthzType() string { return "policy" } diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 5c0e19f1d9..5ef79d568f 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -64,6 +64,7 @@ type Host struct { OsqueryHostID string `json:"-" db:"osquery_host_id"` DetailUpdatedAt time.Time `json:"detail_updated_at" db:"detail_updated_at"` // Time that the host details were last updated LabelUpdatedAt time.Time `json:"label_updated_at" db:"label_updated_at"` // Time that the host labels were last updated + PolicyUpdatedAt time.Time `json:"policy_updated_at" db:"policy_updated_at"` // Time that the host policies were last updated LastEnrolledAt time.Time `json:"last_enrolled_at" db:"last_enrolled_at"` // Time that the host last enrolled SeenTime time.Time `json:"seen_time" db:"seen_time"` // Time that the host was last "seen" RefetchRequested bool `json:"refetch_requested" db:"refetch_requested"` diff --git a/server/fleet/redis_pool.go b/server/fleet/redis_pool.go index 18fa088b77..e1d86daf2c 100644 --- a/server/fleet/redis_pool.go +++ b/server/fleet/redis_pool.go @@ -5,7 +5,17 @@ import "github.com/gomodule/redigo/redis" // RedisPool is the common interface for redigo's Pool for standalone Redis // and redisc's Cluster for Redis Cluster. type RedisPool interface { + // Get returns a redis connection. It must always be closed after use. Get() redis.Conn + + // Close closes the redis connection. Close() error + + // Stats returns a map of redis pool statistics for each server address. Stats() map[string]redis.PoolStats + + // ConfigureDoer returns a redis connection that is properly configured + // to execute Do commands. This should only be called when the actions + // to execute are all done with conn.Do. + ConfigureDoer(redis.Conn) redis.Conn } diff --git a/server/fleet/service.go b/server/fleet/service.go index 3f4e8c2b91..8c7f4568be 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -358,6 +358,7 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // ActivitiesService + ListActivities(ctx context.Context, opt ListOptions) ([]*Activity, error) /////////////////////////////////////////////////////////////////////////////// @@ -368,6 +369,7 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // GlobalScheduleService + GlobalScheduleQuery(ctx context.Context, sq *ScheduledQuery) (*ScheduledQuery, error) GetGlobalScheduledQueries(ctx context.Context, opts ListOptions) ([]*ScheduledQuery, error) ModifyGlobalScheduledQueries(ctx context.Context, id uint, q ScheduledQueryPayload) (*ScheduledQuery, error) @@ -375,10 +377,12 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // TranslatorService + Translate(ctx context.Context, payloads []TranslatePayload) ([]TranslatePayload, error) /////////////////////////////////////////////////////////////////////////////// // TeamScheduleService + TeamScheduleQuery(ctx context.Context, teamID uint, sq *ScheduledQuery) (*ScheduledQuery, error) GetTeamScheduledQueries(ctx context.Context, teamID uint, opts ListOptions) ([]*ScheduledQuery, error) ModifyTeamScheduledQueries( @@ -398,4 +402,12 @@ type Service interface { // Software ListSoftware(ctx context.Context, teamID *uint, opt ListOptions) ([]Software, error) + + /////////////////////////////////////////////////////////////////////////////// + // Team Policies + + NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) + ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) + DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } diff --git a/server/live_query/redis_live_query.go b/server/live_query/redis_live_query.go index b81b88b3fb..a54e480b37 100644 --- a/server/live_query/redis_live_query.go +++ b/server/live_query/redis_live_query.go @@ -194,7 +194,7 @@ func (r *redisLiveQuery) RunQuery(name, sql string, hostIDs []uint) error { } func (r *redisLiveQuery) StopQuery(name string) error { - conn := r.pool.Get() + conn := r.pool.ConfigureDoer(r.pool.Get()) defer conn.Close() targetKey, sqlKey := generateKeys(name) @@ -279,7 +279,7 @@ func (r *redisLiveQuery) collectBatchQueriesForHost(hostID uint, queryKeys []str } func (r *redisLiveQuery) QueryCompletedByHost(name string, hostID uint) error { - conn := r.pool.Get() + conn := r.pool.ConfigureDoer(r.pool.Get()) defer conn.Close() targetKey, _ := generateKeys(name) diff --git a/server/live_query/redis_live_query_test.go b/server/live_query/redis_live_query_test.go index bdabe609f2..5a13411b32 100644 --- a/server/live_query/redis_live_query_test.go +++ b/server/live_query/redis_live_query_test.go @@ -1,6 +1,7 @@ package live_query import ( + "runtime" "testing" "time" @@ -99,6 +100,10 @@ func TestMigrateKeys(t *testing.T) { } func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -111,7 +116,14 @@ func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, tea } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) store = NewRedisLiveQuery(pool) diff --git a/server/logging/filesystem.go b/server/logging/filesystem.go index ac82ade98b..5b9a7e98bf 100644 --- a/server/logging/filesystem.go +++ b/server/logging/filesystem.go @@ -24,35 +24,42 @@ type filesystemLogWriter struct { // NewFilesystemLogWriter creates a log file for osquery status/result logs. // The logFile can be rotated by sending a `SIGHUP` signal to Fleet if // enableRotation is true +// +// The enableCompression argument is only used when enableRotation is true. func NewFilesystemLogWriter(path string, appLogger log.Logger, enableRotation bool, enableCompression bool) (*filesystemLogWriter, error) { - if enableRotation { - // Use lumberjack logger that supports rotation - osquerydLogger := &lumberjack.Logger{ - Filename: path, - MaxSize: 500, // megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: enableCompression, - } - appLogger = log.With(appLogger, "component", "osqueryd-logger") - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGHUP) - go func() { - for { - <-sig //block on signal - if err := osquerydLogger.Rotate(); err != nil { - appLogger.Log("err", err) - } - } - }() - return &filesystemLogWriter{osquerydLogger}, nil - } - // no log rotation, use "raw" bufio implementation - writer, err := newRawLogWriter(path) + // Fail early if the process does not have the necessary + // permissions to open the file at path. + file, err := openFile(path) if err != nil { - return nil, errors.Wrap(err, "create new raw logger") + return nil, errors.Wrap(err, "perm check") } - return &filesystemLogWriter{writer}, nil + if !enableRotation { + // no log rotation, use "raw" bufio implementation + return &filesystemLogWriter{ + writer: newRawLogWriter(file), + }, nil + } + // Use lumberjack logger that supports rotation + file.Close() + osquerydLogger := &lumberjack.Logger{ + Filename: path, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: enableCompression, + } + appLogger = log.With(appLogger, "component", "osqueryd-logger") + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP) + go func() { + for { + <-sig //block on signal + if err := osquerydLogger.Rotate(); err != nil { + appLogger.Log("err", err) + } + } + }() + return &filesystemLogWriter{osquerydLogger}, nil } // If writer is based on bufio we want to flush after a batch of @@ -85,13 +92,9 @@ type rawLogWriter struct { mtx sync.Mutex } -func newRawLogWriter(path string) (*rawLogWriter, error) { - file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - if err != nil { - return nil, err - } +func newRawLogWriter(file *os.File) *rawLogWriter { buff := bufio.NewWriter(file) - return &rawLogWriter{file: file, buff: buff}, nil + return &rawLogWriter{file: file, buff: buff} } // Write bytes to file @@ -141,3 +144,7 @@ func (l *rawLogWriter) Close() error { return nil } + +func openFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) +} diff --git a/server/logging/filesystem_test.go b/server/logging/filesystem_test.go index ae6328ec44..912e3e5542 100644 --- a/server/logging/filesystem_test.go +++ b/server/logging/filesystem_test.go @@ -4,6 +4,8 @@ import ( "context" "crypto/rand" "encoding/json" + stderrors "errors" + "io/fs" "io/ioutil" "os" "path" @@ -59,6 +61,27 @@ func TestFilesystemLogger(t *testing.T) { } +// TestFilesystemLoggerPermission tests that NewFilesystemLogWriter fails +// if the process does not have permissions to write to the provided path. +func TestFilesystemLoggerPermission(t *testing.T) { + tempPath := t.TempDir() + require.NoError(t, os.Chmod(tempPath, 0000)) + fileName := path.Join(tempPath, "filesystemLogWriter") + for _, tc := range []struct { + name string + rotation bool + }{ + {name: "with-rotation", rotation: true}, + {name: "without-rotation", rotation: false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := NewFilesystemLogWriter(fileName, log.NewNopLogger(), tc.rotation, false) + require.Error(t, err) + require.True(t, stderrors.Is(err, fs.ErrPermission), err) + }) + } +} + func BenchmarkFilesystemLogger(b *testing.B) { ctx := context.Background() tempPath, err := ioutil.TempDir("", "test") diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 666db34709..206d4433f4 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -59,7 +59,7 @@ type DeleteQueriesFunc func(ctx context.Context, ids []uint) (uint, error) type QueryFunc func(ctx context.Context, id uint) (*fleet.Query, error) -type ListQueriesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) +type ListQueriesFunc func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) type QueryByNameFunc func(ctx context.Context, name string, opts ...fleet.OptionalArg) (*fleet.Query, error) @@ -115,7 +115,7 @@ type LabelFunc func(ctx context.Context, lid uint) (*fleet.Label, error) type ListLabelsFunc func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Label, error) -type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) +type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) type RecordLabelQueryExecutionsFunc func(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error @@ -221,6 +221,8 @@ type ScheduledQueryFunc func(ctx context.Context, id uint) (*fleet.ScheduledQuer type CleanupOrphanScheduledQueryStatsFunc func(ctx context.Context) error +type CleanupOrphanLabelMembershipFunc func(ctx context.Context) error + type NewTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) type SaveTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) @@ -277,6 +279,14 @@ type MigrationStatusFunc func(ctx context.Context) (fleet.MigrationStatus, error type ListSoftwareFunc func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) +type NewTeamPolicyFunc func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) + +type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) + +type DeleteTeamPoliciesFunc func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + +type TeamPolicyFunc func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) + type DataStore struct { NewCarveFunc NewCarveFunc NewCarveFuncInvoked bool @@ -593,6 +603,9 @@ type DataStore struct { CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFuncInvoked bool + CleanupOrphanLabelMembershipFunc CleanupOrphanLabelMembershipFunc + CleanupOrphanLabelMembershipFuncInvoked bool + NewTeamFunc NewTeamFunc NewTeamFuncInvoked bool @@ -676,6 +689,18 @@ type DataStore struct { ListSoftwareFunc ListSoftwareFunc ListSoftwareFuncInvoked bool + + NewTeamPolicyFunc NewTeamPolicyFunc + NewTeamPolicyFuncInvoked bool + + ListTeamPoliciesFunc ListTeamPoliciesFunc + ListTeamPoliciesFuncInvoked bool + + DeleteTeamPoliciesFunc DeleteTeamPoliciesFunc + DeleteTeamPoliciesFuncInvoked bool + + TeamPolicyFunc TeamPolicyFunc + TeamPolicyFuncInvoked bool } func (s *DataStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { @@ -798,7 +823,7 @@ func (s *DataStore) Query(ctx context.Context, id uint) (*fleet.Query, error) { return s.QueryFunc(ctx, id) } -func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { +func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { s.ListQueriesFuncInvoked = true return s.ListQueriesFunc(ctx, opt) } @@ -938,9 +963,9 @@ func (s *DataStore) ListLabels(ctx context.Context, filter fleet.TeamFilter, opt return s.ListLabelsFunc(ctx, filter, opt) } -func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { +func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { s.LabelQueriesForHostFuncInvoked = true - return s.LabelQueriesForHostFunc(ctx, host, cutoff) + return s.LabelQueriesForHostFunc(ctx, host) } func (s *DataStore) RecordLabelQueryExecutions(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error { @@ -1203,6 +1228,11 @@ func (s *DataStore) CleanupOrphanScheduledQueryStats(ctx context.Context) error return s.CleanupOrphanScheduledQueryStatsFunc(ctx) } +func (s *DataStore) CleanupOrphanLabelMembership(ctx context.Context) error { + s.CleanupOrphanLabelMembershipFuncInvoked = true + return s.CleanupOrphanLabelMembershipFunc(ctx) +} + func (s *DataStore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { s.NewTeamFuncInvoked = true return s.NewTeamFunc(ctx, team) @@ -1342,3 +1372,23 @@ func (s *DataStore) ListSoftware(ctx context.Context, teamId *uint, opt fleet.Li s.ListSoftwareFuncInvoked = true return s.ListSoftwareFunc(ctx, teamId, opt) } + +func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + s.NewTeamPolicyFuncInvoked = true + return s.NewTeamPolicyFunc(ctx, teamID, queryID) +} + +func (s *DataStore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + s.ListTeamPoliciesFuncInvoked = true + return s.ListTeamPoliciesFunc(ctx, teamID) +} + +func (s *DataStore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + s.DeleteTeamPoliciesFuncInvoked = true + return s.DeleteTeamPoliciesFunc(ctx, teamID, ids) +} + +func (s *DataStore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + s.TeamPolicyFuncInvoked = true + return s.TeamPolicyFunc(ctx, teamID, policyID) +} diff --git a/server/pubsub/testing_utils.go b/server/pubsub/testing_utils.go index 2b8a02cc41..9ff5b39c1d 100644 --- a/server/pubsub/testing_utils.go +++ b/server/pubsub/testing_utils.go @@ -1,6 +1,7 @@ package pubsub import ( + "runtime" "testing" "time" @@ -10,6 +11,10 @@ import ( ) func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -23,7 +28,14 @@ func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, te } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) store = NewRedisQueryResults(pool, dupResults) diff --git a/server/service/client.go b/server/service/client.go index abb98c15af..4e7a86b500 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -230,7 +230,7 @@ func (l *logRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { fmt.Fprintf(os.Stderr, "Read body error: %v", err) return nil, err } - res.Body = ioutil.NopCloser(resBody) + res.Body = io.NopCloser(resBody) return res, nil } diff --git a/server/service/global_policies_test.go b/server/service/global_policies_test.go new file mode 100644 index 0000000000..284cccb4eb --- /dev/null +++ b/server/service/global_policies_test.go @@ -0,0 +1,84 @@ +package service + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" +) + +func TestGlobalPoliciesAuth(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + ds.NewGlobalPolicyFunc = func(ctx context.Context, queryID uint) (*fleet.Policy, error) { + return nil, nil + } + ds.ListGlobalPoliciesFunc = func(ctx context.Context) ([]*fleet.Policy, error) { + return nil, nil + } + ds.PolicyFunc = func(ctx context.Context, id uint) (*fleet.Policy, error) { + return nil, nil + } + ds.DeleteGlobalPoliciesFunc = func(ctx context.Context, ids []uint) ([]uint, error) { + return nil, nil + } + + var testCases = []struct { + name string + user *fleet.User + shouldFailWrite bool + shouldFailRead bool + }{ + { + "global admin", + &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + false, + false, + }, + { + "global maintainer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, + false, + false, + }, + { + "global observer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + true, + false, + }, + { + "team maintainer", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, + true, + true, + }, + { + "team observer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, + true, + true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + + _, err := svc.NewGlobalPolicy(ctx, 2) + checkAuthErr(t, tt.shouldFailWrite, err) + + _, err = svc.ListGlobalPolicies(ctx) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.GetPolicyByIDQueries(ctx, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.DeleteGlobalPolicies(ctx, []uint{1}) + checkAuthErr(t, tt.shouldFailWrite, err) + }) + } +} diff --git a/server/service/handler.go b/server/service/handler.go index 90518ab52b..edbb178d86 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -695,16 +695,35 @@ func attachNewStyleFleetAPIRoutes(r *mux.Router, svc fleet.Service, opts []kitht e.POST("/api/v1/fleet/translate", translatorEndpoint, translatorRequest{}) e.POST("/api/v1/fleet/spec/teams", applyTeamSpecsEndpoint, applyTeamSpecsRequest{}) + // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ e.GET("/api/v1/fleet/team/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) e.POST("/api/v1/fleet/team/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) e.PATCH("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) e.DELETE("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) + // End alias + + e.GET("/api/v1/fleet/teams/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) + e.POST("/api/v1/fleet/teams/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) + e.PATCH("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) + e.DELETE("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) e.POST("/api/v1/fleet/global/policies", globalPolicyEndpoint, globalPolicyRequest{}) e.GET("/api/v1/fleet/global/policies", listGlobalPoliciesEndpoint, nil) e.GET("/api/v1/fleet/global/policies/{policy_id}", getPolicyByIDEndpoint, getPolicyByIDRequest{}) e.POST("/api/v1/fleet/global/policies/delete", deleteGlobalPoliciesEndpoint, deleteGlobalPoliciesRequest{}) + // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ + e.POST("/api/v1/fleet/team/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) + e.GET("/api/v1/fleet/team/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/team/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) + e.POST("/api/v1/fleet/team/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) + // End alias + + e.POST("/api/v1/fleet/teams/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) + e.GET("/api/v1/fleet/teams/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/teams/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) + e.POST("/api/v1/fleet/teams/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/software", listSoftwareEndpoint, listSoftwareRequest{}) } diff --git a/server/service/http_auth.go b/server/service/http_auth.go index bf61a4c616..7a1c1756ef 100644 --- a/server/service/http_auth.go +++ b/server/service/http_auth.go @@ -18,9 +18,11 @@ func setRequestsContexts(svc fleet.Service) kithttp.RequestFunc { return func(ctx context.Context, r *http.Request) context.Context { bearer := token.FromHTTPRequest(r) ctx = token.NewContext(ctx, bearer) - v, err := authViewer(ctx, string(bearer), svc) - if err == nil { - ctx = viewer.NewContext(ctx, *v) + if bearer != "" { + v, err := authViewer(ctx, string(bearer), svc) + if err == nil { + ctx = viewer.NewContext(ctx, *v) + } } // get the user-id for request diff --git a/server/service/http_auth_test.go b/server/service/http_auth_test.go index 3c9533de85..eea04c6164 100644 --- a/server/service/http_auth_test.go +++ b/server/service/http_auth_test.go @@ -21,7 +21,6 @@ import ( func TestLogin(t *testing.T) { ds, users, server := setupAuthTest(t) - defer server.Close() var loginTests = []struct { email string status int @@ -60,7 +59,7 @@ func TestLogin(t *testing.T) { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := &nopCloser{bytes.NewBuffer(j)} + requestBody := io.NopCloser(bytes.NewBuffer(j)) resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, tt.status, resp.StatusCode) @@ -173,7 +172,7 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := &nopCloser{bytes.NewBuffer(j)} + requestBody := io.NopCloser(bytes.NewBuffer(j)) resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) @@ -191,7 +190,6 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { func TestNoHeaderErrorsDifferently(t *testing.T) { _, _, server := setupAuthTest(t) - defer server.Close() req, _ := http.NewRequest("GET", server.URL+"/api/v1/fleet/users", nil) client := &http.Client{} @@ -230,10 +228,3 @@ func TestNoHeaderErrorsDifferently(t *testing.T) { } `, string(bodyBytes)) } - -// an io.ReadCloser for new request body -type nopCloser struct { - io.Reader -} - -func (nopCloser) Close() error { return nil } diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go new file mode 100644 index 0000000000..25dafc96ca --- /dev/null +++ b/server/service/integration_core_test.go @@ -0,0 +1,394 @@ +package service + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type integrationTestSuite struct { + suite.Suite + + withServer +} + +func (s *integrationTestSuite) SetupSuite() { + s.withServer.SetupSuite("integrationTestSuite") +} + +func TestIntegrations(t *testing.T) { + testingSuite := new(integrationTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +func (s *integrationTestSuite) TestDoubleUserCreationErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String(fleet.RoleObserver), + } + + s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusOK) + respSecond := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusConflict) + + assertBodyContains(t, respSecond, `Error 1062: Duplicate entry 'email@asd.com'`) +} + +func (s *integrationTestSuite) TestUserWithoutRoleErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + } + + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") +} + +func (s *integrationTestSuite) TestUserWithWrongRoleErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String("wrongrole"), + } + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") +} + +func (s *integrationTestSuite) TestUserCreationWrongTeamErrors() { + t := s.T() + + teams := []fleet.UserTeam{ + { + Team: fleet.Team{ + ID: 9999, + }, + Role: fleet.RoleObserver, + }, + } + + params := fleet.UserPayload{ + Name: ptr.String("user2"), + Email: ptr.String("email2@asd.com"), + Password: ptr.String("pass"), + Teams: &teams, + } + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) +} + +func (s *integrationTestSuite) TestQueryCreationLogsActivity() { + t := s.T() + + admin1 := s.users["admin1@example.com"] + admin1.GravatarURL = "http://iii.com" + err := s.ds.SaveUser(context.Background(), &admin1) + require.NoError(t, err) + + params := fleet.QueryPayload{ + Name: ptr.String("user1"), + Query: ptr.String("select * from time;"), + } + s.Do("POST", "/api/v1/fleet/queries", ¶ms, http.StatusOK) + + activities := listActivitiesResponse{} + s.DoJSON("GET", "/api/v1/fleet/activities", nil, http.StatusOK, &activities) + + assert.Len(t, activities.Activities, 1) + assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0].ActorFullName) + require.NotNil(t, activities.Activities[0].ActorGravatar) + assert.Equal(t, "http://iii.com", *activities.Activities[0].ActorGravatar) + assert.Equal(t, "created_saved_query", activities.Activities[0].Type) +} +func (s *integrationTestSuite) TestAppConfigAdditionalQueriesCanBeRemoved() { + t := s.T() + + spec := []byte(` + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 0 + host_settings: + additional_queries: + time: SELECT * FROM time + enable_host_users: true +`) + s.applyConfig(spec) + + spec = []byte(` + host_settings: + enable_host_users: true + additional_queries: null +`) + s.applyConfig(spec) + + config := s.getConfig() + assert.Nil(t, config.HostSettings.AdditionalQueries) + assert.True(t, config.HostExpirySettings.HostExpiryEnabled) +} + +func (s *integrationTestSuite) TestAppConfigDefaultValues() { + config := s.getConfig() + s.Run("Update interval", func() { + require.Equal(s.T(), 1*time.Hour, config.UpdateInterval.OSQueryDetail) + }) + + s.Run("has logging", func() { + require.NotNil(s.T(), config.Logging) + }) +} + +func (s *integrationTestSuite) TestUserRolesSpec() { + t := s.T() + + _, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + email := t.Name() + "@asd.com" + u := &fleet.User{ + Password: []byte("asd"), + Name: t.Name(), + Email: email, + GravatarURL: "http://asd.com", + GlobalRole: ptr.String(fleet.RoleObserver), + } + user, err := s.ds.NewUser(context.Background(), u) + require.NoError(t, err) + assert.Len(t, user.Teams, 0) + + spec := []byte(fmt.Sprintf(` + roles: + %s: + global_role: null + teams: + - role: maintainer + team: team1 +`, + email)) + + var userRoleSpec applyUserRoleSpecsRequest + err = yaml.Unmarshal(spec, &userRoleSpec.Spec) + require.NoError(t, err) + + s.Do("POST", "/api/v1/fleet/users/roles/spec", &userRoleSpec, http.StatusOK) + + user, err = s.ds.UserByEmail(context.Background(), email) + require.NoError(t, err) + require.Len(t, user.Teams, 1) + assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) +} + +func (s *integrationTestSuite) TestGlobalSchedule() { + t := s.T() + + gs := fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 0) + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery1", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} + r := globalScheduleQueryResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/schedule", gsParams, http.StatusOK, &r) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) + assert.Equal(t, "TestQuery1", gs.GlobalSchedule[0].Name) + id := gs.GlobalSchedule[0].ID + + gs = fleet.GlobalSchedulePayload{} + gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} + s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), gsParams, http.StatusOK, &gs) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) + + r = globalScheduleQueryResponse{} + s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), nil, http.StatusOK, &r) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 0) +} + +func (s *integrationTestSuite) TestTranslator() { + t := s.T() + + payload := translatorResponse{} + params := translatorRequest{List: []fleet.TranslatePayload{ + { + Type: fleet.TranslatorTypeUserEmail, + Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, + }, + }} + s.DoJSON("POST", "/api/v1/fleet/translate", ¶ms, http.StatusOK, &payload) + require.Len(t, payload.List, 1) + + assert.Equal(t, s.users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) +} + +func (s *integrationTestSuite) TestVulnerableSoftware() { + t := s.T() + + host, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1", + UUID: t.Name() + "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + require.NotNil(t, host) + + soft := fleet.HostSoftware{ + Modified: true, + Software: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "apps"}, + }, + } + host.HostSoftware = soft + require.NoError(t, s.ds.SaveHostSoftware(context.Background(), host)) + require.NoError(t, s.ds.LoadHostSoftware(context.Background(), host)) + + soft1 := host.Software[0] + if soft1.Name != "bar" { + soft1 = host.Software[1] + } + + require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) + require.NoError(t, s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) + + resp := s.Do("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), nil, http.StatusOK) + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + expectedJSONSoft2 := `"name": "bar", + "version": "0.0.3", + "source": "apps", + "generated_cpe": "somecpe", + "vulnerabilities": [ + { + "cve": "cve-123-123-132", + "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" + } + ]` + expectedJSONSoft1 := `"name": "foo", + "version": "0.0.1", + "source": "chrome_extensions", + "generated_cpe": "", + "vulnerabilities": null` + // We are doing Contains instead of equals to test the output for software in particular + // ignoring other things like timestamps and things that are outside the cope of this ticket + assert.Contains(t, string(bodyBytes), expectedJSONSoft2) + assert.Contains(t, string(bodyBytes), expectedJSONSoft1) +} + +func (s *integrationTestSuite) TestGlobalPolicies() { + t := s.T() + + for i := 0; i < 3; i++ { + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), + OsqueryHostID: strconv.Itoa(i), + NodeKey: fmt.Sprintf("%d", i), + UUID: fmt.Sprintf("%d", i), + Hostname: fmt.Sprintf("foo.local%d", i), + }) + require.NoError(t, err) + } + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery3", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gpParams := globalPolicyRequest{QueryID: qr.ID} + gpResp := globalPolicyResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/policies", gpParams, http.StatusOK, &gpResp) + require.NotNil(t, gpResp.Policy) + assert.Equal(t, qr.ID, gpResp.Policy.QueryID) + + policiesResponse := listGlobalPoliciesResponse{} + s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 1) + assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) + + singlePolicyResponse := getPolicyByIDResponse{} + singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) + s.DoJSON("GET", singlePolicyURL, nil, http.StatusOK, &singlePolicyResponse) + assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) + assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) + + listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) + listHostsResp := listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 3) + + h1 := listHostsResp.Hosts[0] + h2 := listHostsResp.Hosts[1] + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 0) + + require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) + require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 1) + + deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} + deletePolicyResp := deleteGlobalPoliciesResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp) + + policiesResponse = listGlobalPoliciesResponse{} + s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 0) +} diff --git a/server/service/integration_ds_only_test.go b/server/service/integration_ds_only_test.go new file mode 100644 index 0000000000..780a281fb2 --- /dev/null +++ b/server/service/integration_ds_only_test.go @@ -0,0 +1,61 @@ +package service + +import ( + "net/http" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type integrationDSTestSuite struct { + withDS + suite.Suite +} + +func TestIntegrationDSTestSuite(t *testing.T) { + testingSuite := new(integrationDSTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +func (s *integrationDSTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationDSTestSuite") +} + +func (s *integrationDSTestSuite) TestLicenseExpiration() { + testCases := []struct { + name string + tier string + expiration time.Time + shouldHaveHeader bool + }{ + {"basic expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, + {"basic not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, + {"core expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, + {"core not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, + } + + createTestUsers(s.T(), s.ds) + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + + license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} + _, server := RunServerForTestsWithDS(t, s.ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) + + ts := withServer{server: server} + ts.s = &s.Suite + ts.token = ts.getTestAdminToken() + + resp := ts.Do("GET", "/api/v1/fleet/config", nil, http.StatusOK) + if tt.shouldHaveHeader { + require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) + } else { + require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) + } + }) + } +} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go new file mode 100644 index 0000000000..5370f50b82 --- /dev/null +++ b/server/service/integration_enterprise_test.go @@ -0,0 +1,206 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestIntegrationsEnterprise(t *testing.T) { + testingSuite := new(integrationEnterpriseTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +type integrationEnterpriseTestSuite struct { + withServer + suite.Suite +} + +func (s *integrationEnterpriseTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationEnterpriseTestSuite") + + users, server := RunServerForTestsWithDS( + s.T(), s.ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) + s.server = server + s.users = users + s.token = s.getTestAdminToken() +} + +func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { + t := s.T() + + // create a team through the service so it initializes the agent ops + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + Description: "desc team1", + } + + s.Do("POST", "/api/v1/fleet/teams", team, http.StatusOK) + + // updates a team + agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) + teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: teamName, AgentOptions: &agentOpts}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + team, err := s.ds.TeamByName(context.Background(), teamName) + require.NoError(t, err) + + assert.Len(t, team.Secrets, 0) + require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) + + // creates a team with default agent options + user, err := s.ds.UserByEmail(context.Background(), "admin1@example.com") + require.NoError(t, err) + + teams, err := s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + require.True(t, len(teams) >= 1) + + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + teams, err = s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + assert.True(t, len(teams) >= 2) + + team, err = s.ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` + assert.Len(t, team.Secrets, 0) + require.NotNil(t, team.AgentOptions) + require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) + + // updates secrets + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + team, err = s.ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + require.Len(t, team.Secrets, 1) + assert.Equal(t, "ABC", team.Secrets[0].Secret) +} + +func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { + t := s.T() + + team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + ts := getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 0) + + qr, err := s.ds.NewQuery( + context.Background(), + &fleet.Query{Name: "TestQueryTeamPolicy", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}, + ) + require.NoError(t, err) + + gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} + r := teamScheduleQueryResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), gsParams, http.StatusOK, &r) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(42), ts.Scheduled[0].Interval) + assert.Equal(t, "TestQueryTeamPolicy", ts.Scheduled[0].Name) + assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) + id := ts.Scheduled[0].ID + + modifyResp := modifyTeamScheduleResponse{} + modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} + s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), modifyParams, http.StatusOK, &modifyResp) + + // just to satisfy my paranoia, wanted to make sure the contents of the json would work + s.DoRaw("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), []byte(`{"interval": 77}`), http.StatusOK) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(77), ts.Scheduled[0].Interval) + + deleteResp := deleteTeamScheduleResponse{} + s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), nil, http.StatusOK, &deleteResp) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 0) +} + +func (s *integrationEnterpriseTestSuite) TestTeamPolicies() { + t := s.T() + + team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1" + t.Name(), + Description: "desc team1", + }) + require.NoError(t, err) + + oldToken := s.token + t.Cleanup(func() { + s.token = oldToken + }) + + password := "garbage" + email := "testteam@user.com" + + u := &fleet.User{ + Name: "test team user", + Email: email, + GlobalRole: nil, + Teams: []fleet.UserTeam{ + { + Team: *team1, + Role: fleet.RoleMaintainer, + }, + }, + } + require.NoError(t, u.SetPassword(password, 10, 10)) + _, err = s.ds.NewUser(context.Background(), u) + require.NoError(t, err) + + s.token = s.getTestToken(email, password) + + ts := listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 0) + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) + require.NoError(t, err) + + tpParams := teamPolicyRequest{QueryID: qr.ID} + r := teamPolicyResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), tpParams, http.StatusOK, &r) + + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 1) + assert.Equal(t, "TestQuery2", ts.Policies[0].QueryName) + assert.Equal(t, qr.ID, ts.Policies[0].QueryID) + + deletePolicyParams := deleteTeamPoliciesRequest{IDs: []uint{ts.Policies[0].ID}} + deletePolicyResp := deleteTeamPoliciesResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies/delete", team1.ID), deletePolicyParams, http.StatusOK, &deletePolicyResp) + + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 0) +} diff --git a/server/service/integration_logger_test.go b/server/service/integration_logger_test.go new file mode 100644 index 0000000000..2f39f97496 --- /dev/null +++ b/server/service/integration_logger_test.go @@ -0,0 +1,186 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestIntegrationLoggerTestSuite(t *testing.T) { + testingSuite := new(integrationLoggerTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +type integrationLoggerTestSuite struct { + withServer + suite.Suite + + buf *bytes.Buffer +} + +func (s *integrationLoggerTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationLoggerTestSuite") + + s.buf = new(bytes.Buffer) + logger := log.NewJSONLogger(s.buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + users, server := RunServerForTestsWithDS(s.T(), s.ds, TestServerOpts{Logger: logger}) + s.server = server + s.users = users +} + +func (s *integrationLoggerTestSuite) TearDownTest() { + s.buf.Reset() +} + +func (s *integrationLoggerTestSuite) TestLogger() { + t := s.T() + + s.token = getTestAdminToken(t, s.server) + + s.getConfig() + + params := fleet.QueryPayload{ + Name: ptr.String("somequery"), + Description: ptr.String("desc"), + Query: ptr.String("select 1 from osquery;"), + } + payload := createQueryRequest{} + s.DoJSON("POST", "/api/v1/fleet/queries", params, http.StatusOK, &payload) + + logs := s.buf.String() + parts := strings.Split(strings.TrimSpace(logs), "\n") + assert.Len(t, parts, 3) + for i, part := range parts { + kv := make(map[string]string) + err := json.Unmarshal([]byte(part), &kv) + require.NoError(t, err) + + assert.NotEqual(t, "", kv["took"]) + + switch i { + case 0: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) + case 1: + assert.Equal(t, "debug", kv["level"]) + assert.Equal(t, "GET", kv["method"]) + assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + case 2: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + assert.Equal(t, "somequery", kv["name"]) + assert.Equal(t, "select 1 from osquery;", kv["sql"]) + default: + t.Fail() + } + } +} + +func (s *integrationLoggerTestSuite) TestOsqueryEndpointsLogErrors() { + t := s.T() + + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1234", + UUID: "1", + Hostname: "foo.local", + OsqueryHostID: t.Name(), + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + requestBody := io.NopCloser(bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))) + req, _ := http.NewRequest("POST", s.server.URL+"/api/v1/osquery/log", requestBody) + client := &http.Client{} + _, err = client.Do(req) + require.Nil(t, err) + + logString := s.buf.String() + assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} +`, logString) +} + +func (s *integrationLoggerTestSuite) TestSubmitStatusLog() { + t := s.T() + + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + OsqueryHostID: t.Name(), + }) + require.NoError(t, err) + + req := submitLogsRequest{ + NodeKey: "1234", + LogType: "status", + Data: nil, + } + res := submitLogsResponse{} + s.DoJSON("POST", "/api/v1/osquery/log", req, http.StatusOK, &res) + + logString := s.buf.String() + assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) + assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) +} + +func (s *integrationLoggerTestSuite) TestEnrollAgentLogsErrors() { + t := s.T() + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + j, err := json.Marshal(&enrollAgentRequest{ + EnrollSecret: "1234", + HostIdentifier: "4321", + HostDetails: nil, + }) + require.NoError(t, err) + + s.DoRawNoAuth("POST", "/api/v1/osquery/enroll", j, http.StatusUnauthorized) + + parts := strings.Split(strings.TrimSpace(s.buf.String()), "\n") + require.Len(t, parts, 1) + logData := make(map[string]json.RawMessage) + require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) + assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) +} diff --git a/server/service/integration_test.go b/server/service/integration_test.go deleted file mode 100644 index ca24a626c2..0000000000 --- a/server/service/integration_test.go +++ /dev/null @@ -1,931 +0,0 @@ -package service - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/datastore/mysql" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/fleetdm/fleet/v4/server/test" - "github.com/ghodss/yaml" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDoubleUserCreationErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String(fleet.RoleObserver), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - requestBody = &nopCloser{bytes.NewBuffer(j)} - req, _ = http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - resp, err = client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusConflict, resp.StatusCode) - assertBodyContains(t, resp, `Error 1062: Duplicate entry 'email@asd.com'`) -} - -func TestUserWithoutRoleErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") -} - -func TestUserWithWrongRoleErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String("wrongrole"), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") -} - -func TestUserCreationWrongTeamErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - teams := []fleet.UserTeam{ - { - Team: fleet.Team{ - ID: 9999, - }, - Role: fleet.RoleObserver, - }, - } - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - Teams: &teams, - } - method := "POST" - path := "/api/v1/fleet/users/admin" - expectedStatusCode := http.StatusUnprocessableEntity - - resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) - defer closeFunc() - assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) -} - -func doReq( - t *testing.T, - params interface{}, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, -) (*http.Response, func()) { - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest(method, server.URL+path, requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, expectedStatusCode, resp.StatusCode) - return resp, func() { - thisResp := resp - thisResp.Body.Close() - } -} - -func doRawReq( - t *testing.T, - body []byte, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, -) *http.Response { - requestBody := &nopCloser{bytes.NewBuffer(body)} - req, _ := http.NewRequest(method, server.URL+path, requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, expectedStatusCode, resp.StatusCode) - return resp -} - -func doJSONReq( - t *testing.T, - params interface{}, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, - v interface{}, -) { - resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) - defer closeFunc() - err := json.NewDecoder(resp.Body).Decode(v) - require.Nil(t, err) -} - -func assertBodyContains(t *testing.T, resp *http.Response, expectedError string) { - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err) - bodyString := string(bodyBytes) - assert.Contains(t, bodyString, expectedError) -} - -func TestQueryCreationLogsActivity(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - users, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - admin1 := users["admin1@example.com"] - admin1.GravatarURL = "http://iii.com" - err := ds.SaveUser(context.Background(), &admin1) - require.NoError(t, err) - - params := fleet.QueryPayload{ - Name: ptr.String("user1"), - Query: ptr.String("select * from time;"), - } - _, closeFunc := doReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK) - defer closeFunc() - type activitiesRespose struct { - Activities []map[string]interface{} `json:"activities"` - } - activities := activitiesRespose{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/activities", token, http.StatusOK, &activities) - - assert.Len(t, activities.Activities, 1) - assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0]["actor_full_name"]) - assert.Equal(t, "http://iii.com", activities.Activities[0]["actor_gravatar"]) - assert.Equal(t, "created_saved_query", activities.Activities[0]["type"]) -} - -func getJSON(r *http.Response, target interface{}) error { - return json.NewDecoder(r.Body).Decode(target) -} - -func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { - err := &fleet.Error{} - require.Nil(t, getJSON(resp, err)) - assert.Equal(t, code, err.Code) - assert.Equal(t, message, err.Message) -} - -func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - spec := []byte(` - host_expiry_settings: - host_expiry_enabled: true - host_expiry_window: 0 - host_settings: - additional_queries: - time: SELECT * FROM time - enable_host_users: true -`) - applyConfig(t, spec, server, token) - - spec = []byte(` - host_settings: - enable_host_users: true - additional_queries: null -`) - applyConfig(t, spec, server, token) - - config := getConfig(t, server, token) - assert.Nil(t, config.HostSettings.AdditionalQueries) - assert.True(t, config.HostExpirySettings.HostExpiryEnabled) -} - -func TestAppConfigUpdateInterval(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - config := getConfig(t, server, token) - require.Equal(t, 1*time.Hour, config.UpdateInterval.OSQueryDetail) -} - -func TestAppConfigHasLogging(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - config := getConfig(t, server, token) - require.NotNil(t, config.Logging) -} - -func applyConfig(t *testing.T, spec []byte, server *httptest.Server, token string) { - var appConfigSpec interface{} - err := yaml.Unmarshal(spec, &appConfigSpec) - require.NoError(t, err) - - _, closeFunc := doReq(t, appConfigSpec, "PATCH", server, "/api/v1/fleet/config", token, http.StatusOK) - closeFunc() -} - -func getConfig(t *testing.T, server *httptest.Server, token string) *appConfigResponse { - var responseBody *appConfigResponse - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK, &responseBody) - return responseBody -} - -func TestUserRolesSpec(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - _, err := ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - token := getTestAdminToken(t, server) - - user, err := ds.UserByEmail(context.Background(), "user1@example.com") - require.NoError(t, err) - assert.Len(t, user.Teams, 0) - - spec := []byte(` - roles: - user1@example.com: - global_role: null - teams: - - role: maintainer - team: team1 -`) - - var userRoleSpec applyUserRoleSpecsRequest - err = yaml.Unmarshal(spec, &userRoleSpec.Spec) - require.NoError(t, err) - - _, closeFunc := doReq(t, userRoleSpec, "POST", server, "/api/v1/fleet/users/roles/spec", token, http.StatusOK) - closeFunc() - - user, err = ds.UserByEmail(context.Background(), "user1@example.com") - require.NoError(t, err) - require.Len(t, user.Teams, 1) - assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) - - // But users are not deleted - users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) - require.NoError(t, err) - assert.Len(t, users, 3) -} - -func TestGlobalSchedule(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - test.AddAllHostsLabel(t, ds) - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - gs := fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - assert.Len(t, gs.GlobalSchedule, 0) - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} - r := globalScheduleQueryResponse{} - doJSONReq(t, gsParams, "POST", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &r) - require.Nil(t, r.Err) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) - assert.Equal(t, "TestQuery", gs.GlobalSchedule[0].Name) - id := gs.GlobalSchedule[0].ID - - gs = fleet.GlobalSchedulePayload{} - gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} - doJSONReq( - t, gsParams, "PATCH", server, - fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), - token, http.StatusOK, &gs, - ) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) - - r = globalScheduleQueryResponse{} - doJSONReq( - t, nil, "DELETE", server, - fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), - token, http.StatusOK, &r, - ) - require.Nil(t, r.Err) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 0) -} - -func TestTeamSpecs(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) - defer server.Close() - token := getTestAdminToken(t, server) - - // create a team through the service so it initializes the agent ops - team := &fleet.Team{ - Name: "team1", - Description: "desc team1", - } - _, closeFunc := doReq(t, team, "POST", server, "/api/v1/fleet/teams", token, http.StatusOK) - defer closeFunc() - - // updates a team - agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) - teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team1", AgentOptions: &agentOpts}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - team, err := ds.TeamByName(context.Background(), "team1") - require.NoError(t, err) - - assert.Len(t, team.Secrets, 0) - require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) - - // creates a team with default agent options - user, err := ds.UserByEmail(context.Background(), "admin1@example.com") - require.NoError(t, err) - - teams, err := ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - assert.Len(t, teams, 1) - - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - teams, err = ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - assert.Len(t, teams, 2) - - team, err = ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` - assert.Len(t, team.Secrets, 0) - require.NotNil(t, team.AgentOptions) - require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) - - // updates secrets - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - team, err = ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - require.Len(t, team.Secrets, 1) - assert.Equal(t, "ABC", team.Secrets[0].Secret) -} - -func TestTranslator(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - users, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - payload := translatorResponse{} - params := translatorRequest{List: []fleet.TranslatePayload{ - { - Type: fleet.TranslatorTypeUserEmail, - Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, - }, - }} - doJSONReq(t, ¶ms, "POST", server, "/api/v1/fleet/translate", token, http.StatusOK, &payload) - - require.Nil(t, payload.Err) - assert.Len(t, payload.List, 1) - - assert.Equal(t, users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) -} - -func TestTeamSchedule(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - test.AddAllHostsLabel(t, ds) - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - team1, err := ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - - ts := getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 0) - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) - require.NoError(t, err) - - gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} - r := teamScheduleQueryResponse{} - doJSONReq(t, gsParams, "POST", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &r) - require.Nil(t, r.Err) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(42), ts.Scheduled[0].Interval) - assert.Equal(t, "TestQuery", ts.Scheduled[0].Name) - assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) - id := ts.Scheduled[0].ID - - modifyResp := modifyTeamScheduleResponse{} - modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} - doJSONReq( - t, modifyParams, "PATCH", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK, &modifyResp, - ) - - // just to satisfy my paranoia, wanted to make sure the contents of the json would work - doRawReq(t, []byte(`{"interval": 77}`), "PATCH", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(77), ts.Scheduled[0].Interval) - - deleteResp := deleteTeamScheduleResponse{} - doJSONReq( - t, nil, "DELETE", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK, &deleteResp, - ) - require.Nil(t, r.Err) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 0) -} - -func TestLogger(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - token := getTestAdminToken(t, server) - - getConfig(t, server, token) - params := fleet.QueryPayload{ - Name: ptr.String("somequery"), - Description: ptr.String("desc"), - Query: ptr.String("select 1 from osquery;"), - } - payload := createQueryRequest{} - doJSONReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK, &payload) - - logs := buf.String() - parts := strings.Split(strings.TrimSpace(logs), "\n") - assert.Len(t, parts, 3) - for i, part := range parts { - kv := make(map[string]string) - err := json.Unmarshal([]byte(part), &kv) - require.NoError(t, err) - - assert.NotEqual(t, "", kv["took"]) - - switch i { - case 0: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) - case 1: - assert.Equal(t, "debug", kv["level"]) - assert.Equal(t, "GET", kv["method"]) - assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - case 2: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - assert.Equal(t, "somequery", kv["name"]) - assert.Equal(t, "select 1 from osquery;", kv["sql"]) - default: - t.Fail() - } - } -} - -func TestVulnerableSoftware(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - host, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - require.NotNil(t, host) - - soft := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "bar", Version: "0.0.3", Source: "apps"}, - }, - } - host.HostSoftware = soft - require.NoError(t, ds.SaveHostSoftware(context.Background(), host)) - require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) - - soft1 := host.Software[0] - if soft1.Name != "bar" { - soft1 = host.Software[1] - } - - require.NoError(t, ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) - - path := fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID) - resp, closeFunc := doReq(t, nil, "GET", server, path, token, http.StatusOK) - defer closeFunc() - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - expectedJSONSoft2 := `"name": "bar", - "version": "0.0.3", - "source": "apps", - "generated_cpe": "somecpe", - "vulnerabilities": [ - { - "cve": "cve-123-123-132", - "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" - } - ]` - expectedJSONSoft1 := `"name": "foo", - "version": "0.0.1", - "source": "chrome_extensions", - "generated_cpe": "", - "vulnerabilities": null` - // We are doing Contains instead of equals to test the output for software in particular - // ignoring other things like timestamps and things that are outside the cope of this ticket - assert.Contains(t, string(bodyBytes), expectedJSONSoft2) - assert.Contains(t, string(bodyBytes), expectedJSONSoft1) -} - -func TestGlobalPolicies(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - for i := 0; i < 3; i++ { - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), - OsqueryHostID: strconv.Itoa(i), - NodeKey: fmt.Sprintf("%d", i), - UUID: fmt.Sprintf("%d", i), - Hostname: fmt.Sprintf("foo.local%d", i), - }) - require.NoError(t, err) - } - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gpParams := globalPolicyRequest{QueryID: qr.ID} - gpResp := globalPolicyResponse{} - doJSONReq(t, gpParams, "POST", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &gpResp) - require.NotNil(t, gpResp.Policy) - assert.Equal(t, qr.ID, gpResp.Policy.QueryID) - - policiesResponse := listGlobalPoliciesResponse{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 1) - assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) - - singlePolicyResponse := getPolicyByIDResponse{} - singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) - doJSONReq(t, nil, "GET", server, singlePolicyURL, token, http.StatusOK, &singlePolicyResponse) - assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) - assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) - - listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) - listHostsResp := listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 3) - - h1 := listHostsResp.Hosts[0] - h2 := listHostsResp.Hosts[1] - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 0) - - require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) - require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 1) - - deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} - deletePolicyResp := deleteGlobalPoliciesResponse{} - doJSONReq(t, deletePolicyParams, "POST", server, "/api/v1/fleet/global/policies/delete", token, http.StatusOK, &deletePolicyResp) - - policiesResponse = listGlobalPoliciesResponse{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 0) -} - -func TestOsqueryEndpointsLogErrors(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - requestBody := &nopCloser{bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/log", requestBody) - client := &http.Client{} - _, err = client.Do(req) - require.Nil(t, err) - - logString := buf.String() - assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} -`, logString) -} - -func TestSubmitStatusLog(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - token := getTestAdminToken(t, server) - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - req := submitLogsRequest{ - NodeKey: "1234", - LogType: "status", - Data: nil, - } - res := submitLogsResponse{} - doJSONReq(t, req, "POST", server, "/api/v1/osquery/log", token, http.StatusOK, &res) - - logString := buf.String() - assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) - assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) -} - -func TestEnrollAgentLogsErrors(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - j, err := json.Marshal(&enrollAgentRequest{ - EnrollSecret: "1234", - HostIdentifier: "4321", - HostDetails: nil, - }) - require.NoError(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/enroll", requestBody) - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - - parts := strings.Split(strings.TrimSpace(buf.String()), "\n") - require.Len(t, parts, 1) - logData := make(map[string]json.RawMessage) - require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) - assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) -} - -func TestLicenseExpiration(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - testCases := []struct { - name string - tier string - expiration time.Time - shouldHaveHeader bool - }{ - {"premium expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, - {"premium not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, - {"free expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, - {"free not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, - } - - _ = createTestUsers(t, ds) - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) - defer server.Close() - - token := getTestAdminToken(t, server) - - resp, closeFunc := doReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK) - defer closeFunc() - if tt.shouldHaveHeader { - require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) - } else { - require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) - } - }) - } -} diff --git a/server/service/service.go b/server/service/service.go index e9cf216b89..62cec4c931 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -38,10 +38,19 @@ type Service struct { } // NewService creates a new service from the config struct -func NewService(ds fleet.Datastore, resultStore fleet.QueryResultStore, - logger kitlog.Logger, osqueryLogger *logging.OsqueryLogger, config config.FleetConfig, mailService fleet.MailService, - c clock.Clock, sso sso.SessionStore, lq fleet.LiveQueryStore, carveStore fleet.CarveStore, - license fleet.LicenseInfo) (fleet.Service, error) { +func NewService( + ds fleet.Datastore, + resultStore fleet.QueryResultStore, + logger kitlog.Logger, + osqueryLogger *logging.OsqueryLogger, + config config.FleetConfig, + mailService fleet.MailService, + c clock.Clock, + sso sso.SessionStore, + lq fleet.LiveQueryStore, + carveStore fleet.CarveStore, + license fleet.LicenseInfo, +) (fleet.Service, error) { var svc fleet.Service authorizer, err := authz.NewAuthorizer() diff --git a/server/service/service_appconfig.go b/server/service/service_appconfig.go index e15ecf851d..1c9ce979e7 100644 --- a/server/service/service_appconfig.go +++ b/server/service/service_appconfig.go @@ -181,7 +181,10 @@ func (svc *Service) SetupRequired(ctx context.Context) (bool, error) { } func (svc *Service) UpdateIntervalConfig(ctx context.Context) (*fleet.UpdateIntervalConfig, error) { - return &fleet.UpdateIntervalConfig{OSQueryDetail: svc.config.Osquery.DetailUpdateInterval}, nil + return &fleet.UpdateIntervalConfig{ + OSQueryDetail: svc.config.Osquery.DetailUpdateInterval, + OSQueryPolicy: svc.config.Osquery.PolicyUpdateInterval, + }, nil } func (svc *Service) LoggingConfig(ctx context.Context) (*fleet.Logging, error) { diff --git a/server/service/service_campaign_test.go b/server/service/service_campaign_test.go index 6a6039cac5..5437cca55f 100644 --- a/server/service/service_campaign_test.go +++ b/server/service/service_campaign_test.go @@ -36,7 +36,7 @@ func TestStreamCampaignResultsClosesReditOnWSClose(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { diff --git a/server/service/service_campaigns.go b/server/service/service_campaigns.go index 0802a2f99c..ec1cd90e17 100644 --- a/server/service/service_campaigns.go +++ b/server/service/service_campaigns.go @@ -61,7 +61,7 @@ func (svc Service) NewDistributedQueryCampaign(ctx context.Context, queryString } queryString = query.Query } else { - if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionWrite); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRunNew); err != nil { return nil, err } query = &fleet.Query{ diff --git a/server/service/service_hosts_test.go b/server/service/service_hosts_test.go index 7f276443b7..d288d2ff6e 100644 --- a/server/service/service_hosts_test.go +++ b/server/service/service_hosts_test.go @@ -32,6 +32,7 @@ func TestListHosts(t *testing.T) { SeenTime: storedTime, DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), }) require.NoError(t, err) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index fc5ab0bfdd..6787aa7c7b 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -2,8 +2,10 @@ package service import ( "context" + "crypto/rand" "encoding/json" "fmt" + "math/big" "strconv" "strings" "time" @@ -404,7 +406,7 @@ const hostDistributedQueryPrefix = "fleet_distributed_query_" // osqueryd to fill in the host details func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map[string]string, error) { queries := make(map[string]string) - if host.DetailUpdatedAt.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) && !host.RefetchRequested { + if !svc.shouldUpdate(host.DetailUpdatedAt, svc.config.Osquery.DetailUpdateInterval) && !host.RefetchRequested { // No need to update already fresh details return queries, nil } @@ -438,6 +440,19 @@ func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map return queries, nil } +func (svc *Service) shouldUpdate(lastUpdated time.Time, interval time.Duration) bool { + var jitter time.Duration + if svc.config.Osquery.MaxJitterPercent > 0 { + maxJitter := time.Duration(svc.config.Osquery.MaxJitterPercent) * interval / time.Duration(100.0) + randDuration, err := rand.Int(rand.Reader, big.NewInt(int64(maxJitter))) + if err == nil { + jitter = time.Duration(randDuration.Int64()) + } + } + cutoff := svc.clock.Now().Add(-(interval + jitter)) + return lastUpdated.Before(cutoff) +} + func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) { // skipauth: Authorization is currently for user endpoints only. svc.authz.SkipAuthorization(ctx) @@ -455,14 +470,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin } // Retrieve the label queries that should be updated - cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval) - labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host, cutoff) - if err != nil { - return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} - } + if svc.shouldUpdate(host.LabelUpdatedAt, svc.config.Osquery.LabelUpdateInterval) { + labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host) + if err != nil { + return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} + } - for name, query := range labelQueries { - queries[hostLabelQueryPrefix+name] = query + for name, query := range labelQueries { + queries[hostLabelQueryPrefix+name] = query + } } liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID) @@ -474,13 +490,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin queries[hostDistributedQueryPrefix+name] = query } - policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) - if err != nil { - return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} - } + if svc.shouldUpdate(host.PolicyUpdatedAt, svc.config.Osquery.PolicyUpdateInterval) { + policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) + if err != nil { + return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} + } - for name, query := range policyQueries { - queries[hostPolicyQueryPrefix+name] = query + for name, query := range policyQueries { + queries[hostPolicyQueryPrefix+name] = query + } } accelerate := uint(0) @@ -691,6 +709,8 @@ func (svc *Service) SubmitDistributedQueryResults( } if len(policyResults) > 0 { + host.Modified = true + host.PolicyUpdatedAt = svc.clock.Now() err = svc.ds.RecordPolicyQueryExecutions(ctx, &host, policyResults, svc.clock.Now()) if err != nil { logging.WithErr(ctx, err) diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 30d7b9fa56..2f0d3f0fe9 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -324,7 +324,7 @@ func TestLabelQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { @@ -361,7 +361,7 @@ func TestLabelQueries(t *testing.T) { assert.Len(t, queries, 0) assert.Zero(t, acc) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{ "label1": "query1", "label2": "query2", @@ -539,7 +539,7 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -715,7 +715,7 @@ func TestDetailQueries(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -920,7 +920,7 @@ func TestNewDistributedQueryCampaign(t *testing.T) { mockClock := clock.NewMockClock() svc := newTestServiceWithClock(ds, rs, lq, mockClock) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { @@ -989,7 +989,7 @@ func TestDistributedQueryResults(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -1721,7 +1721,7 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { }, nil } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { return nil } @@ -1746,6 +1746,63 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { require.NoError(t, err) } +func TestTeamMaintainerCanRunNewDistributedCampaigns(t *testing.T) { + ds := new(mock.Store) + rs := &mock.QueryResultStore{ + HealthCheckFunc: func() error { + return nil + }, + } + lq := &live_query.MockLiveQuery{} + mockClock := clock.NewMockClock() + svc := newTestServiceWithClock(ds, rs, lq, mockClock) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{}, nil + } + + ds.NewDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) (*fleet.DistributedQueryCampaign, error) { + return camp, nil + } + ds.QueryFunc = func(ctx context.Context, id uint) (*fleet.Query, error) { + return &fleet.Query{ + ID: 42, + Name: "query", + Query: "select 1;", + ObserverCanRun: false, + }, nil + } + viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{ + User: &fleet.User{ID: 0, Teams: []fleet.UserTeam{{Role: fleet.RoleMaintainer}}}, + }) + + q := "select year, month, day, hour, minutes, seconds from time" + ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { + return nil + } + //var gotQuery *fleet.Query + ds.NewQueryFunc = func(ctx context.Context, query *fleet.Query, opts ...fleet.OptionalArg) (*fleet.Query, error) { + //gotQuery = query + query.ID = 42 + return query, nil + } + ds.NewDistributedQueryCampaignTargetFunc = func(ctx context.Context, target *fleet.DistributedQueryCampaignTarget) (*fleet.DistributedQueryCampaignTarget, error) { + return target, nil + } + ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) { + return fleet.TargetMetrics{}, nil + } + ds.HostIDsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets) ([]uint, error) { + return []uint{1, 3, 5}, nil + } + ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { + return nil + } + lq.On("RunQuery", "0", "select year, month, day, hour, minutes, seconds from time", []uint{1, 3, 5}).Return(nil) + _, err := svc.NewDistributedQueryCampaign(viewerCtx, q, nil, fleet.HostTargets{HostIDs: []uint{2}, LabelIDs: []uint{1}}) + require.NoError(t, err) +} + func TestPolicyQueries(t *testing.T) { mockClock := clock.NewMockClock() ds := new(mock.Store) @@ -1756,13 +1813,14 @@ func TestPolicyQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { return host, nil } - ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { + ds.SaveHostFunc = func(ctx context.Context, gotHost *fleet.Host) error { + host = gotHost return nil } ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { @@ -1815,4 +1873,40 @@ func TestPolicyQueries(t *testing.T) { require.NotNil(t, recordedResults[1]) require.Equal(t, true, *recordedResults[1]) require.Nil(t, recordedResults[2]) + + ctx = hostctx.NewContext(context.Background(), *host) + queries, _, err = svc.GetDistributedQueries(ctx) + require.NoError(t, err) + require.Len(t, queries, expectedDetailQueries) + + // After the first time we get policies and update the host, then there shouldn't be any policies + hasAnyPolicy := false + for name := range queries { + if strings.HasPrefix(name, hostPolicyQueryPrefix) { + hasAnyPolicy = true + break + } + } + assert.False(t, hasAnyPolicy) + + // Let's move time forward, there should be policies now + mockClock.AddTime(2 * time.Hour) + + queries, _, err = svc.GetDistributedQueries(ctx) + require.NoError(t, err) + require.Len(t, queries, expectedDetailQueries+2) + + hasPolicy1, hasPolicy2 = false, false + for name := range queries { + if strings.HasPrefix(name, hostPolicyQueryPrefix) { + if name[len(hostPolicyQueryPrefix):] == "1" { + hasPolicy1 = true + } + if name[len(hostPolicyQueryPrefix):] == "2" { + hasPolicy2 = true + } + } + } + assert.True(t, hasPolicy1) + assert.True(t, hasPolicy2) } diff --git a/server/service/service_queries.go b/server/service/service_queries.go index 65ea5722df..3eefd83733 100644 --- a/server/service/service_queries.go +++ b/server/service/service_queries.go @@ -66,7 +66,7 @@ func (svc Service) GetQuerySpecs(ctx context.Context) ([]*fleet.QuerySpec, error return nil, err } - queries, err := svc.ds.ListQueries(ctx, fleet.ListOptions{}) + queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{}) if err != nil { return nil, errors.Wrap(err, "getting queries") } @@ -90,12 +90,39 @@ func (svc Service) GetQuerySpec(ctx context.Context, name string) (*fleet.QueryS return specFromQuery(query), nil } +func onlyShowObserverCanRunQueries(user *fleet.User) bool { + if user.GlobalRole != nil && *user.GlobalRole == fleet.RoleObserver { + return true + } else if len(user.Teams) > 0 { + allObserver := true + for _, team := range user.Teams { + if team.Role != fleet.RoleObserver { + allObserver = false + break + } + } + return allObserver + } + return false +} + func (svc Service) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRead); err != nil { return nil, err } - return svc.ds.ListQueries(ctx, opt) + user := authz.UserFromContext(ctx) + onlyShowObserverCanRun := onlyShowObserverCanRunQueries(user) + + queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{ + ListOptions: opt, + OnlyObserverCanRun: onlyShowObserverCanRun, + }) + if err != nil { + return nil, err + } + + return queries, nil } func (svc *Service) GetQuery(ctx context.Context, id uint) (*fleet.Query, error) { diff --git a/server/service/service_queries_test.go b/server/service/service_queries_test.go index 5167d6b6f7..5f3087a81a 100644 --- a/server/service/service_queries_test.go +++ b/server/service/service_queries_test.go @@ -4,8 +4,11 @@ import ( "context" "testing" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,3 +24,61 @@ func TestNewQueryAttach(t *testing.T) { ) require.Error(t, err) } + +func TestFilterQueriesForObserver(t *testing.T) { + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)})) + + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleObserver}}})) + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ + {Role: fleet.RoleObserver}, + {Role: fleet.RoleObserver}, + }})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ + {Role: fleet.RoleObserver}, + {Role: fleet.RoleMaintainer}, + }})) +} + +func TestListQueries(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + cases := [...]struct { + title string + user *fleet.User + expectedOpts fleet.ListQueryOptions + }{ + { + title: "global admin", + user: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, + }, + { + title: "global observer", + user: &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: true}, + }, + { + title: "team admin", + user: &fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleAdmin}}}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, + }, + } + + var calledWithOpts fleet.ListQueryOptions + ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { + calledWithOpts = opt + return []*fleet.Query{}, nil + } + + for _, tt := range cases { + t.Run(tt.title, func(t *testing.T) { + viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + _, err := svc.ListQueries(viewerCtx, fleet.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, tt.expectedOpts, calledWithOpts) + }) + } +} diff --git a/server/service/team_policies.go b/server/service/team_policies.go new file mode 100644 index 0000000000..9ea4febc28 --- /dev/null +++ b/server/service/team_policies.go @@ -0,0 +1,156 @@ +package service + +import ( + "context" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" +) + +///////////////////////////////////////////////////////////////////////////////// +// Add +///////////////////////////////////////////////////////////////////////////////// + +type teamPolicyRequest struct { + TeamID uint `url:"team_id"` + QueryID uint `json:"query_id"` +} + +type teamPolicyResponse struct { + Policy *fleet.Policy `json:"policy,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r teamPolicyResponse) error() error { return r.Err } + +func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*teamPolicyRequest) + resp, err := svc.NewTeamPolicy(ctx, req.TeamID, req.QueryID) + if err != nil { + return teamPolicyResponse{Err: err}, nil + } + return teamPolicyResponse{Policy: resp}, nil +} + +func (svc Service) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } + + return svc.ds.NewTeamPolicy(ctx, teamID, queryID) +} + +///////////////////////////////////////////////////////////////////////////////// +// List +///////////////////////////////////////////////////////////////////////////////// + +type listTeamPoliciesRequest struct { + TeamID uint `url:"team_id"` +} + +type listTeamPoliciesResponse struct { + Policies []*fleet.Policy `json:"policies,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r listTeamPoliciesResponse) error() error { return r.Err } + +func listTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*listTeamPoliciesRequest) + resp, err := svc.ListTeamPolicies(ctx, req.TeamID) + if err != nil { + return listTeamPoliciesResponse{Err: err}, nil + } + return listTeamPoliciesResponse{Policies: resp}, nil +} + +func (svc Service) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } + + return svc.ds.ListTeamPolicies(ctx, teamID) +} + +///////////////////////////////////////////////////////////////////////////////// +// Get by id +///////////////////////////////////////////////////////////////////////////////// + +type getTeamPolicyByIDRequest struct { + TeamID uint `url:"team_id"` + PolicyID uint `url:"policy_id"` +} + +type getTeamPolicyByIDResponse struct { + Policy *fleet.Policy `json:"policy"` + Err error `json:"error,omitempty"` +} + +func (r getTeamPolicyByIDResponse) error() error { return r.Err } + +func getTeamPolicyByIDEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*getTeamPolicyByIDRequest) + teamPolicy, err := svc.GetTeamPolicyByIDQueries(ctx, req.TeamID, req.PolicyID) + if err != nil { + return getTeamPolicyByIDResponse{Err: err}, nil + } + return getTeamPolicyByIDResponse{Policy: teamPolicy}, nil +} + +func (svc Service) GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } + + teamPolicy, err := svc.ds.TeamPolicy(ctx, teamID, policyID) + if err != nil { + return nil, err + } + + return teamPolicy, nil +} + +///////////////////////////////////////////////////////////////////////////////// +// Delete +///////////////////////////////////////////////////////////////////////////////// + +type deleteTeamPoliciesRequest struct { + TeamID uint `url:"team_id"` + IDs []uint `json:"ids"` +} + +type deleteTeamPoliciesResponse struct { + Deleted []uint `json:"deleted,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r deleteTeamPoliciesResponse) error() error { return r.Err } + +func deleteTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*deleteTeamPoliciesRequest) + resp, err := svc.DeleteTeamPolicies(ctx, req.TeamID, req.IDs) + if err != nil { + return deleteTeamPoliciesResponse{Err: err}, nil + } + return deleteTeamPoliciesResponse{Deleted: resp}, nil +} + +func (svc Service) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } + + return svc.ds.DeleteTeamPolicies(ctx, teamID, ids) +} diff --git a/server/service/team_policies_test.go b/server/service/team_policies_test.go new file mode 100644 index 0000000000..662dcce3ea --- /dev/null +++ b/server/service/team_policies_test.go @@ -0,0 +1,107 @@ +package service + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/authz" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/require" +) + +func TestTeamPoliciesAuth(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + ds.NewTeamPolicyFunc = func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + return &fleet.Policy{}, nil + } + ds.ListTeamPoliciesFunc = func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + return nil, nil + } + ds.TeamPolicyFunc = func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + return nil, nil + } + ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + return nil, nil + } + + var testCases = []struct { + name string + user *fleet.User + shouldFailWrite bool + shouldFailRead bool + }{ + { + "global admin", + &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + false, + false, + }, + { + "global maintainer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, + true, + false, + }, + { + "global observer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + true, + false, + }, + { + "team maintainer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, + false, + false, + }, + { + "team observer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, + true, + false, + }, + { + "team maintainer, DOES NOT belong to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleMaintainer}}}, + true, + true, + }, + { + "team observer, DOES NOT belong to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleObserver}}}, + true, + true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + + _, err := svc.NewTeamPolicy(ctx, 1, 2) + checkAuthErr(t, tt.shouldFailWrite, err) + + _, err = svc.ListTeamPolicies(ctx, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.GetTeamPolicyByIDQueries(ctx, 1, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.DeleteTeamPolicies(ctx, 1, []uint{1}) + checkAuthErr(t, tt.shouldFailWrite, err) + }) + } +} + +func checkAuthErr(t *testing.T, shouldFail bool, err error) { + if shouldFail { + require.Error(t, err) + require.Equal(t, (&authz.Forbidden{}).Error(), err.Error()) + } else { + require.NoError(t, err) + } +} diff --git a/server/service/team_schedule.go b/server/service/team_schedule.go index f1578d966f..9561cd2595 100644 --- a/server/service/team_schedule.go +++ b/server/service/team_schedule.go @@ -39,6 +39,9 @@ func (svc Service) GetTeamScheduledQueries(ctx context.Context, teamID uint, opt if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -102,6 +105,9 @@ func (svc Service) TeamScheduleQuery(ctx context.Context, teamID uint, q *fleet. if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -143,6 +149,9 @@ func (svc Service) ModifyTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -183,6 +192,8 @@ func (svc Service) DeleteTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return err } - _ = teamID + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return err + } return svc.DeleteScheduledQuery(ctx, scheduledQueryID) } diff --git a/server/service/testing_client.go b/server/service/testing_client.go new file mode 100644 index 0000000000..89cbaf28eb --- /dev/null +++ b/server/service/testing_client.go @@ -0,0 +1,148 @@ +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + + "github.com/fleetdm/fleet/v4/server/datastore/mysql" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type withDS struct { + s *suite.Suite + ds *mysql.Datastore +} + +func (ts *withDS) SetupSuite(dbName string) { + ts.ds = mysql.CreateNamedMySQLDS(ts.s.T(), dbName) + test.AddAllHostsLabel(ts.s.T(), ts.ds) +} + +func (ts *withDS) TearDownSuite() { + ts.ds.Close() +} + +type withServer struct { + withDS + + server *httptest.Server + users map[string]fleet.User + token string +} + +func (ts *withServer) SetupSuite(dbName string) { + ts.withDS.SetupSuite(dbName) + + users, server := RunServerForTestsWithDS(ts.s.T(), ts.ds) + ts.server = server + ts.users = users + ts.token = ts.getTestAdminToken() +} + +func (ts *withServer) TearDownSuite() { + ts.withDS.TearDownSuite() +} + +func (ts *withServer) Do(verb, path string, params interface{}, expectedStatusCode int) *http.Response { + t := ts.s.T() + + j, err := json.Marshal(params) + require.NoError(t, err) + + resp := ts.DoRaw(verb, path, j, expectedStatusCode) + + t.Cleanup(func() { + resp.Body.Close() + }) + return resp +} + +func (ts *withServer) DoRawWithHeaders(verb string, path string, rawBytes []byte, expectedStatusCode int, headers map[string]string) *http.Response { + t := ts.s.T() + + requestBody := io.NopCloser(bytes.NewBuffer(rawBytes)) + req, _ := http.NewRequest(verb, ts.server.URL+path, requestBody) + for key, val := range headers { + req.Header.Add(key, val) + } + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + require.Equal(t, expectedStatusCode, resp.StatusCode) + + return resp +} + +func (ts *withServer) DoRaw(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { + return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", ts.token), + }) +} + +func (ts *withServer) DoRawNoAuth(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { + return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, nil) +} + +func (ts *withServer) DoJSON(verb, path string, params interface{}, expectedStatusCode int, v interface{}) { + resp := ts.Do(verb, path, params, expectedStatusCode) + err := json.NewDecoder(resp.Body).Decode(v) + require.NoError(ts.s.T(), err) + if e, ok := v.(errorer); ok { + require.NoError(ts.s.T(), e.error()) + } +} + +func (ts *withServer) getTestAdminToken() string { + testUser := testUsers["admin1"] + + return ts.getTestToken(testUser.Email, testUser.PlaintextPassword) +} + +func (ts *withServer) getTestToken(email string, password string) string { + params := loginRequest{ + Email: email, + Password: password, + } + j, err := json.Marshal(¶ms) + require.NoError(ts.s.T(), err) + + requestBody := io.NopCloser(bytes.NewBuffer(j)) + resp, err := http.Post(ts.server.URL+"/api/v1/fleet/login", "application/json", requestBody) + require.NoError(ts.s.T(), err) + defer resp.Body.Close() + assert.Equal(ts.s.T(), http.StatusOK, resp.StatusCode) + + var jsn = struct { + User *fleet.User `json:"user"` + Token string `json:"token"` + Err []map[string]string `json:"errors,omitempty"` + }{} + err = json.NewDecoder(resp.Body).Decode(&jsn) + require.NoError(ts.s.T(), err) + require.Len(ts.s.T(), jsn.Err, 0) + + return jsn.Token +} + +func (ts *withServer) applyConfig(spec []byte) { + var appConfigSpec interface{} + err := yaml.Unmarshal(spec, &appConfigSpec) + require.NoError(ts.s.T(), err) + + ts.Do("PATCH", "/api/v1/fleet/config", appConfigSpec, http.StatusOK) +} + +func (ts *withServer) getConfig() *appConfigResponse { + var responseBody *appConfigResponse + ts.DoJSON("GET", "/api/v1/fleet/config", nil, http.StatusOK, &responseBody) + return responseBody +} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 3b2ab6316a..8d57e4f344 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -2,6 +2,9 @@ package service import ( "context" + "encoding/json" + "io/ioutil" + "net/http" "net/http/httptest" "os" "strings" @@ -9,6 +12,7 @@ import ( eeservice "github.com/fleetdm/fleet/v4/ee/server/service" "github.com/fleetdm/fleet/v4/server/logging" + "github.com/stretchr/testify/assert" "github.com/WatchBeam/clock" "github.com/fleetdm/fleet/v4/server/config" @@ -163,6 +167,9 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...TestServe limitStore, _ := memstore.New(0) r := MakeHandler(svc, config.FleetConfig{}, logger, limitStore) server := httptest.NewServer(r) + t.Cleanup(func() { + server.Close() + }) return users, server } @@ -235,3 +242,21 @@ func testStdoutPluginConfig() config.FleetConfig { c.Osquery.StatusLogPlugin = "stdout" return c } + +func assertBodyContains(t *testing.T, resp *http.Response, expected string) { + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + bodyString := string(bodyBytes) + assert.Contains(t, bodyString, expected) +} + +func getJSON(r *http.Response, target interface{}) error { + return json.NewDecoder(r.Body).Decode(target) +} + +func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { + err := &fleet.Error{} + require.Nil(t, getJSON(resp, err)) + assert.Equal(t, code, err.Code) + assert.Equal(t, message, err.Message) +} diff --git a/server/sso/session_store.go b/server/sso/session_store.go index b2277c5619..7b0bb36fdc 100644 --- a/server/sso/session_store.go +++ b/server/sso/session_store.go @@ -46,7 +46,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin if len(requestID) < 8 { return errors.New("request id must be 8 or more characters in length") } - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() sess := Session{OriginalURL: originalURL, Metadata: metadata} var writer bytes.Buffer @@ -59,7 +59,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin } func (s *store) Get(requestID string) (*Session, error) { - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() val, err := redis.String(conn.Do("GET", requestID)) if err != nil { @@ -81,7 +81,7 @@ func (s *store) Get(requestID string) (*Session, error) { var ErrSessionNotFound = errors.New("session not found") func (s *store) Expire(requestID string) error { - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() _, err := conn.Do("DEL", requestID) return err diff --git a/server/sso/session_store_test.go b/server/sso/session_store_test.go index 45b46af866..318ba12acd 100644 --- a/server/sso/session_store_test.go +++ b/server/sso/session_store_test.go @@ -2,6 +2,7 @@ package sso import ( "os" + "runtime" "testing" "time" @@ -12,6 +13,10 @@ import ( ) func newPool(t *testing.T, cluster bool) fleet.RedisPool { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + if _, ok := os.LookupEnv("REDIS_TEST"); ok { var ( addr = "127.0.0.1:" @@ -25,7 +30,14 @@ func newPool(t *testing.T, cluster bool) fleet.RedisPool { } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) conn := pool.Get() defer conn.Close() diff --git a/server/test/new_objects.go b/server/test/new_objects.go index aaa09a7f11..171239ad79 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -105,6 +105,7 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t UUID: uuid, DetailUpdatedAt: now, LabelUpdatedAt: now, + PolicyUpdatedAt: now, SeenTime: now, OsqueryHostID: osqueryHostID, }) diff --git a/server/vulnerabilities/cpe_test.go b/server/vulnerabilities/cpe_test.go index e0e35e9170..ed004902bf 100644 --- a/server/vulnerabilities/cpe_test.go +++ b/server/vulnerabilities/cpe_test.go @@ -53,6 +53,10 @@ func TestCpeFromSoftware(t *testing.T) { } func TestSyncCPEDatabase(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + // Disabling vcr because the resulting file exceeds the 100mb limit for github r, err := recorder.NewAsMode("fixtures/nvd-cpe-release", recorder.ModeDisabled, http.DefaultTransport) require.NoError(t, err) @@ -144,6 +148,10 @@ func (f *fakeSoftwareIterator) Err() error { return nil } func (f *fakeSoftwareIterator) Close() error { f.closed = true; return nil } func TestTranslateSoftwareToCPE(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + tempDir, err := os.MkdirTemp(os.TempDir(), "TestTranslateSoftwareToCPE-*") require.NoError(t, err) defer os.RemoveAll(tempDir) diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index adc83e8082..cbf984b600 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -26,6 +26,10 @@ var cvetests = []struct { } func TestTranslateCPEToCVE(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + tempDir := t.TempDir() ds := new(mock.Store) diff --git a/tools/backup_db/README.md b/tools/backup_db/README.md index 39f52ea042..11d7ad61a8 100644 --- a/tools/backup_db/README.md +++ b/tools/backup_db/README.md @@ -1,3 +1,3 @@ These scripts are for backing up and restore the Docker development MySQL database. -Usage is documented [here](../../docs/3-Contributing/2-Testing.md#database-backuprestore). +Usage is documented [here](../../docs/03-Contributing/02-Testing.md#database-backuprestore). diff --git a/tools/fleetctl-npm/README.md b/tools/fleetctl-npm/README.md index 52ff50a021..e303e10c50 100644 --- a/tools/fleetctl-npm/README.md +++ b/tools/fleetctl-npm/README.md @@ -10,4 +10,4 @@ Simply install `fleetctl` with `npm install -g fleetctl`. ## Usage -See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. +See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json index 3d03c517c5..8634b01ca6 100644 --- a/tools/fleetctl-npm/package.json +++ b/tools/fleetctl-npm/package.json @@ -1,6 +1,6 @@ { "name": "fleetctl", - "version": "v4.3.0", + "version": "v4.3.1", "description": "Installer for the fleetctl CLI tool", "bin": { "fleetctl": "./run.js" diff --git a/tools/terraform/.terraform-version b/tools/terraform/.terraform-version new file mode 100644 index 0000000000..a6a3a43c3a --- /dev/null +++ b/tools/terraform/.terraform-version @@ -0,0 +1 @@ +1.0.4 \ No newline at end of file diff --git a/tools/terraform/ecs-iam.tf b/tools/terraform/ecs-iam.tf new file mode 100644 index 0000000000..c726f85398 --- /dev/null +++ b/tools/terraform/ecs-iam.tf @@ -0,0 +1,54 @@ +data "aws_iam_policy_document" "fleet" { + statement { + effect = "Allow" + actions = ["cloudwatch:PutMetricData"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn] + } + + statement { + effect = "Allow" + actions = [ + "firehose:DescribeDeliveryStream", + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + resources = [aws_kinesis_firehose_delivery_stream.osquery_logs.arn] + } +} + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "main" { + name = "fleetdm-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "role_attachment" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + role = aws_iam_role.main.name +} + +resource "aws_iam_policy" "main" { + name = "fleet-iam-policy" + policy = data.aws_iam_policy_document.fleet.json +} + +resource "aws_iam_role_policy_attachment" "attachment" { + policy_arn = aws_iam_policy.main.arn + role = aws_iam_role.main.name +} \ No newline at end of file diff --git a/tools/terraform/ecs-sgs.tf b/tools/terraform/ecs-sgs.tf new file mode 100644 index 0000000000..c5d3222dbd --- /dev/null +++ b/tools/terraform/ecs-sgs.tf @@ -0,0 +1,78 @@ +# Security group for the public internet facing load balancer +resource "aws_security_group" "lb" { + name = "${var.prefix} load balancer" + description = "${var.prefix} Load balancer security group" + vpc_id = module.vpc.vpc_id +} + +# Allow traffic from public internet +resource "aws_security_group_rule" "lb-ingress" { + description = "${var.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "443" + to_port = "443" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +resource "aws_security_group_rule" "lb-http-ingress" { + description = "${var.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "80" + to_port = "80" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +# Allow outbound traffic +resource "aws_security_group_rule" "lb-egress" { + description = "${var.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +# Security group for the backends that run the application. +# Allows traffic from the load balancer +resource "aws_security_group" "backend" { + name = "${var.prefix} backend" + description = "${var.prefix} Backend security group" + vpc_id = module.vpc.vpc_id + +} + +# Allow traffic from the load balancer to the backends +resource "aws_security_group_rule" "backend-ingress" { + description = "${var.prefix}: allow traffic from load balancer" + type = "ingress" + + from_port = "8080" + to_port = "8080" + protocol = "tcp" + source_security_group_id = aws_security_group.lb.id + security_group_id = aws_security_group.backend.id +} + +# Allow outbound traffic from the backends +resource "aws_security_group_rule" "backend-egress" { + description = "${var.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.backend.id +} \ No newline at end of file diff --git a/tools/terraform/ecs.tf b/tools/terraform/ecs.tf new file mode 100644 index 0000000000..090cc6a2da --- /dev/null +++ b/tools/terraform/ecs.tf @@ -0,0 +1,328 @@ +//resource "aws_route53_record" "record" { +// name = "fleetdm" +// type = "A" +// zone_id = "Z046188311R47QSK245X" +// alias { +// evaluate_target_health = false +// name = aws_alb.main.dns_name +// zone_id = aws_alb.main.zone_id +// } +//} + +resource "aws_alb" "main" { + name = "fleetdm" + internal = false + security_groups = [aws_security_group.lb.id, aws_security_group.backend.id] + subnets = module.vpc.public_subnets +} + +resource "aws_alb_target_group" "main" { + name = "fleetdm" + protocol = "HTTP" + target_type = "ip" + port = "8080" + vpc_id = module.vpc.vpc_id + deregistration_delay = 30 + + load_balancing_algorithm_type = "least_outstanding_requests" + + health_check { + path = "/healthz" + matcher = "200" + timeout = 10 + interval = 15 + healthy_threshold = 5 + unhealthy_threshold = 5 + } + + depends_on = [aws_alb.main] +} + +resource "aws_alb_listener" "https-fleetdm" { + load_balancer_arn = aws_alb.main.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_alb_target_group.main.arn + type = "forward" + } +} + +resource "aws_alb_listener" "http" { + load_balancer_arn = aws_alb.main.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_ecs_cluster" "fleet" { + name = "${var.prefix}-backend" + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_service" "fleet" { + name = "fleet" + launch_type = "FARGATE" + cluster = aws_ecs_cluster.fleet.id + task_definition = aws_ecs_task_definition.backend.arn + desired_count = 1 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + health_check_grace_period_seconds = 30 + + load_balancer { + target_group_arn = aws_alb_target_group.main.arn + container_name = "fleet" + container_port = 8080 + } + + network_configuration { + subnets = module.vpc.private_subnets + security_groups = [aws_security_group.backend.id] + } + + depends_on = [aws_alb_listener.http, aws_alb_listener.https-fleetdm] +} + +resource "aws_cloudwatch_log_group" "backend" { + name = "fleetdm" + retention_in_days = 1 +} + +data "aws_region" "current" {} + +data "aws_secretsmanager_secret" "license" { + name = "/fleet/license" +} + +resource "aws_ecs_task_definition" "backend" { + family = "fleet" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 512 + memory = 4096 + container_definitions = jsonencode( + [ + { + name = "fleet" + image = "fleetdm/fleet" + cpu = 512 + memory = 4096 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + protocol = "tcp" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet" + } + }, + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_MYSQL_READ_REPLICA_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_LICENSE_KEY" + valueFrom = data.aws_secretsmanager_secret.license.arn + } + ] + environment = [ + { + name = "FLEET_MYSQL_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_reader_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + }, + { + name = "FLEET_FIREHOSE_STATUS_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_logs.name + }, + { + name = "FLEET_FIREHOSE_RESULT_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_logs.name + }, + { + name = "FLEET_FIREHOSE_REGION" + value = data.aws_region.current.name + }, + { + name = "FLEET_OSQUERY_STATUS_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_OSQUERY_RESULT_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_SERVER_TLS" + value = "false" + }, + { + name = "FLEET_BETA_SOFTWARE_INVENTORY" + value = "1" + }, + { + name = "FLEET_VULNERABILITIES_DATABASES_PATH" + value = "/home/fleet" + } + ] + } + ]) +} + +resource "aws_ecs_task_definition" "migration" { + family = "fleet-migrate" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 256 + memory = 512 + container_definitions = jsonencode( + [ + { + name = "fleet-prepare-db" + image = "fleetdm/fleet" + cpu = 256 + memory = 512 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + protocol = "tcp" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet" + } + }, + command = ["fleet", "prepare", "db"] + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + } + ] + environment = [ + { + name = "FLEET_MYSQL_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + } + ] + } + ]) +} + +resource "aws_appautoscaling_target" "ecs_target" { + max_capacity = 5 + min_capacity = 1 + resource_id = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}" + scalable_dimension = "ecs:service:DesiredCount" + service_namespace = "ecs" +} + +resource "aws_appautoscaling_policy" "ecs_policy_memory" { + name = "fleet-memory-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageMemoryUtilization" + } + target_value = 80 + } +} + +resource "aws_appautoscaling_policy" "ecs_policy_cpu" { + name = "fleet-cpu-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + + target_value = 60 + } +} diff --git a/tools/terraform/firehose.tf b/tools/terraform/firehose.tf new file mode 100644 index 0000000000..ceed78ac42 --- /dev/null +++ b/tools/terraform/firehose.tf @@ -0,0 +1,69 @@ +resource "aws_s3_bucket" "osquery" { + bucket = "fleet-osquery-logs-archive" + acl = "private" + + lifecycle_rule { + enabled = true + expiration { + days = 1 + } + } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + } + } +} + +// allow firehose to write to bucket +data "aws_iam_policy_document" "osquery_logs_policy_doc" { + statement { + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ] + resources = [aws_s3_bucket.osquery.arn, "${aws_s3_bucket.osquery.arn}/*"] + } +} + +resource "aws_iam_policy" "firehose" { + name = "osquery_logs_firehose_policy" + policy = data.aws_iam_policy_document.osquery_logs_policy_doc.json +} + +resource "aws_iam_role" "firehose" { + assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json +} + +resource "aws_iam_role_policy_attachment" "firehose" { + policy_arn = aws_iam_policy.firehose.arn + role = aws_iam_role.firehose.name +} + +data "aws_iam_policy_document" "osquery_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["firehose.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_kinesis_firehose_delivery_stream" "osquery_logs" { + name = "osquery_logs" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.osquery.arn + } +} \ No newline at end of file diff --git a/tools/terraform/main.tf b/tools/terraform/main.tf new file mode 100644 index 0000000000..95bb588b3e --- /dev/null +++ b/tools/terraform/main.tf @@ -0,0 +1,19 @@ +variable "region" { + default = "us-east-2" +} + +provider "aws" { + region = var.region +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.54.0" + } + } +} + +data "aws_caller_identity" "current" {} + diff --git a/tools/terraform/outputs.tf b/tools/terraform/outputs.tf new file mode 100644 index 0000000000..caa6ef2314 --- /dev/null +++ b/tools/terraform/outputs.tf @@ -0,0 +1,7 @@ +output "nameservers_fleetctl" { + value = aws_route53_zone.dogfood_fleetctl_com.name_servers +} + +output "nameservers_fleetdm" { + value = aws_route53_zone.dogfood_fleetdm_com.name_servers +} diff --git a/tools/terraform/r53.tf b/tools/terraform/r53.tf new file mode 100644 index 0000000000..619483a545 --- /dev/null +++ b/tools/terraform/r53.tf @@ -0,0 +1,93 @@ +resource "aws_route53_zone" "dogfood_fleetctl_com" { + name = "dogfood.fleetctl.com" +} + +resource "aws_route53_zone" "dogfood_fleetdm_com" { + name = "dogfood.fleetdm.com" +} + +resource "aws_route53_record" "dogfood_fleetctl_com" { + zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id + name = "dogfood.fleetctl.com" + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_route53_record" "dogfood_fleetdm_com" { + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id + name = "dogfood.fleetdm.com" + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_acm_certificate" "dogfood_fleetctl_com" { + domain_name = "dogfood.fleetctl.com" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_acm_certificate" "dogfood_fleetdm_com" { + domain_name = "dogfood.fleetdm.com" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "dogfood_fleetctl_com_validation" { + for_each = { + for dvo in aws_acm_certificate.dogfood_fleetctl_com.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id +} + +resource "aws_route53_record" "dogfood_fleetdm_com_validation" { + for_each = { + for dvo in aws_acm_certificate.dogfood_fleetdm_com.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id +} + +resource "aws_acm_certificate_validation" "dogfood_fleetctl_com" { + certificate_arn = aws_acm_certificate.dogfood_fleetctl_com.arn + validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetctl_com_validation : record.fqdn] +} + +resource "aws_acm_certificate_validation" "dogfood_fleetdm_com" { + certificate_arn = aws_acm_certificate.dogfood_fleetdm_com.arn + validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetdm_com_validation : record.fqdn] +} \ No newline at end of file diff --git a/tools/terraform/rds.tf b/tools/terraform/rds.tf new file mode 100644 index 0000000000..c4a13466d6 --- /dev/null +++ b/tools/terraform/rds.tf @@ -0,0 +1,113 @@ +locals { + name = "fleetdm" +} + +resource "random_password" "database_password" { + length = 16 + special = false +} + +resource "aws_secretsmanager_secret" "database_password_secret" { + name = "/fleet/database/password/master" +} + +resource "aws_secretsmanager_secret_version" "database_password_secret_version" { + secret_id = aws_secretsmanager_secret.database_password_secret.id + secret_string = random_password.database_password.result +} + +// if you want to use RDS Serverless option prefer the following commented block +//module "aurora_mysql_serverless" { +// source = "terraform-aws-modules/rds-aurora/aws" +// version = "5.2.0" +// +// name = "${local.name}-mysql" +// engine = "aurora-mysql" +// engine_mode = "serverless" +// storage_encrypted = true +// username = "fleet" +// password = random_password.database_password.result +// create_random_password = false +// database_name = "fleet" +// enable_http_endpoint = true +// +// vpc_id = module.vpc.vpc_id +// subnets = module.vpc.database_subnets +// create_security_group = true +// allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks +// +// replica_scale_enabled = false +// replica_count = 0 +// +// monitoring_interval = 60 +// +// apply_immediately = true +// skip_final_snapshot = true +// +// db_parameter_group_name = aws_db_parameter_group.example_mysql.id +// db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id +// +// scaling_configuration = { +// auto_pause = true +// min_capacity = 2 +// max_capacity = 16 +// seconds_until_auto_pause = 300 +// timeout_action = "ForceApplyCapacityChange" +// } +//} + +module "aurora_mysql" { + source = "terraform-aws-modules/rds-aurora/aws" + version = "5.2.0" + + name = "${local.name}-mysql-iam" + engine = "aurora-mysql" + engine_version = "5.7.mysql_aurora.2.10.0" + instance_type = "db.t4g.medium" + instance_type_replica = "db.t4g.medium" + + iam_database_authentication_enabled = true + storage_encrypted = true + username = "fleet" + password = random_password.database_password.result + create_random_password = false + database_name = "fleet" + enable_http_endpoint = false + #performance_insights_enabled = true + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.database_subnets + create_security_group = true + allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks + + replica_count = 1 + replica_scale_enabled = true + replica_scale_min = 1 + replica_scale_max = 3 + + monitoring_interval = 60 + iam_role_name = "${local.name}-rds-enhanced-monitoring" + iam_role_use_name_prefix = true + iam_role_description = "${local.name} RDS enhanced monitoring IAM role" + iam_role_path = "/autoscaling/" + iam_role_max_session_duration = 7200 + + apply_immediately = true + skip_final_snapshot = true + + db_parameter_group_name = aws_db_parameter_group.example_mysql.id + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id + +} + +resource "aws_db_parameter_group" "example_mysql" { + name = "${local.name}-aurora-db-mysql-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-db-mysql-parameter-group" +} + +resource "aws_rds_cluster_parameter_group" "example_mysql" { + name = "${local.name}-aurora-mysql-cluster-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-mysql-cluster-parameter-group" +} \ No newline at end of file diff --git a/tools/terraform/readme.md b/tools/terraform/readme.md new file mode 100644 index 0000000000..5c6cb4baae --- /dev/null +++ b/tools/terraform/readme.md @@ -0,0 +1,44 @@ +## Terraform + +`terraform init && terraform workspace new dev` + +`terraform plan` + +`terraform apply` + +### Configuration + +Typical settings to override in an existing environment: + +`module.vpc.vpc_id` -- the VPC ID output from VPC module. If you are introducing fleet to an existing VPC, you could replace all instances with your VPC ID. + +In this reference architecture we are placing ECS, RDS MySQL, and Redis (ElastiCache) in separate subnets, each associated to a route table, allowing communication between. +This is not required, as long as Fleet can resolve the MySQL and Redis hosts, that should be adequate. + +#### HTTPS + +The ALB is in the public subnet with an ENI to bridge into the private subnet. SSL is terminated at the ALB and `fleet serve` is launched with `FLEET_SERVER_TLS=false` as an +environment variable. + +Replace `cert_arn` with the **certificate ARN** that applies to your environment. This is the **certificate ARN** used in the **ALB HTTPS Listener**. + +### Migrating the DB + +After applying terraform run the following to migrate the database: +``` +aws ecs run-task --cluster fleet-backend --task-definition fleet-migrate: --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets=[],securityGroups=[]}" +``` + +### Connecting a Host + +Build orbit: + +``` + fleetctl package --type=msi --fleet-url= --enroll-secret= +``` + +Run orbit: + +``` + "C:\Program Files\Orbit\bin\orbit\orbit.exe" --root-dir "C:\Program Files\Orbit\." --log-file "C:\Program Files\Orbit\orbit-log.txt" --fleet-url "http://" --enroll-secret-path "C:\Program Files\Orbit\secret.txt" --update-url "https://tuf.fleetctl.com" --orbit-channel "stable" --osqueryd-channel "stable" +``` \ No newline at end of file diff --git a/tools/terraform/redis.tf b/tools/terraform/redis.tf new file mode 100644 index 0000000000..d289472318 --- /dev/null +++ b/tools/terraform/redis.tf @@ -0,0 +1,65 @@ +variable "maintenance_window" { + default = "" +} +variable "engine_version" { + default = "5.0.6" +} +variable "node_type" { + default = "cache.t2.micro" +} +variable "number_cache_clusters" { + default = 3 +} +resource "aws_elasticache_replication_group" "default" { + availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] + engine = "redis" + parameter_group_name = aws_elasticache_parameter_group.default.name + subnet_group_name = module.vpc.elasticache_subnet_group_name + security_group_ids = [aws_security_group.redis.id] + replication_group_id = "fleetdm-redis" + number_cache_clusters = var.number_cache_clusters + node_type = var.node_type + engine_version = var.engine_version + port = "6379" + maintenance_window = var.maintenance_window + snapshot_retention_limit = 0 + automatic_failover_enabled = false + at_rest_encryption_enabled = false + transit_encryption_enabled = false + apply_immediately = true + replication_group_description = "fleetdm-redis" +} + +resource "aws_elasticache_parameter_group" "default" { + name = "fleetdm-redis" + family = "redis5.0" + description = "for fleet" +} + +resource "aws_security_group" "redis" { + name = local.security_group_name + vpc_id = module.vpc.vpc_id +} + +locals { + security_group_name = "${var.prefix}-elasticache-redis" +} + +resource "aws_security_group_rule" "ingress" { + type = "ingress" + from_port = "6379" + to_port = "6379" + protocol = "tcp" + cidr_blocks = module.vpc.private_subnets_cidr_blocks + security_group_id = aws_security_group.redis.id +} + +resource "aws_security_group_rule" "egress" { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.redis.id +} + diff --git a/tools/terraform/variables.tf b/tools/terraform/variables.tf new file mode 100644 index 0000000000..40d60b7d4d --- /dev/null +++ b/tools/terraform/variables.tf @@ -0,0 +1,3 @@ +variable "prefix" { + default = "fleet" +} \ No newline at end of file diff --git a/tools/terraform/vpc.tf b/tools/terraform/vpc.tf new file mode 100644 index 0000000000..ed9fa8b4a7 --- /dev/null +++ b/tools/terraform/vpc.tf @@ -0,0 +1,24 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "fleet-vpc" + cidr = "10.10.0.0/16" + + azs = ["us-east-2a", "us-east-2b", "us-east-2c"] + private_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] + public_subnets = ["10.10.11.0/24", "10.10.12.0/24", "10.10.13.0/24"] + database_subnets = ["10.10.21.0/24", "10.10.22.0/24", "10.10.23.0/24"] + elasticache_subnets = ["10.10.31.0/24", "10.10.32.0/24", "10.10.33.0/24"] + + create_database_subnet_group = true + create_database_subnet_route_table = true + + create_elasticache_subnet_group = true + create_elasticache_subnet_route_table = true + + enable_vpn_gateway = false + one_nat_gateway_per_az = false + + single_nat_gateway = true + enable_nat_gateway = true +} \ No newline at end of file diff --git a/website/.sailsrc b/website/.sailsrc index 9f58d730b0..7499bf8d2f 100644 --- a/website/.sailsrc +++ b/website/.sailsrc @@ -12,224 +12,264 @@ "url": "/docs", "title": "Readme.md", "lastModifiedAt": 1624049901000, - "htmlId": "docs--readme--9f534d32b2", - "meta": {} - }, - { - "url": "/docs/using-fleet/learn-how-to-use-fleet", - "title": "Learn how to use Fleet", - "lastModifiedAt": 1631573400000, - "htmlId": "docs--0-learn-how-to-use-f--1b80658ae8", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleet-ui", - "title": "Fleet UI", - "lastModifiedAt": 1631640519000, - "htmlId": "docs--1-fleet-ui--ed954948be", - "meta": {} - }, - { - "url": "/docs/using-fleet/teams", - "title": "Teams", - "lastModifiedAt": 1629395421000, - "htmlId": "docs--10-teams--782f2af710", - "meta": {} - }, - { - "url": "/docs/using-fleet/usage-statistics", - "title": "Usage statistics", - "lastModifiedAt": 1624989594000, - "htmlId": "docs--11-usage-statistics--3ed9f3101b", - "meta": {} - }, - { - "url": "/docs/using-fleet/supported-browsers", - "title": "Supported browsers", - "lastModifiedAt": 1630452786000, - "htmlId": "docs--12-supported-browser--6f8b591603", - "meta": {} - }, - { - "url": "/docs/using-fleet/vulnerability-processing", - "title": "Vulnerability processing", - "lastModifiedAt": 1629761820000, - "htmlId": "docs--13-vulnerability-pro--edb754352c", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleetctl-cli", - "title": "Fleetctl CLI", - "lastModifiedAt": 1631040955000, - "htmlId": "docs--2-fleetctl-cli--b4a4f6b08c", - "meta": {} - }, - { - "url": "/docs/using-fleet/rest-api", - "title": "REST API", - "lastModifiedAt": 1631555916000, - "htmlId": "docs--3-rest-api--0370e3eaff", - "meta": {} - }, - { - "url": "/docs/using-fleet/adding-hosts", - "title": "Adding hosts", - "lastModifiedAt": 1625588060000, - "htmlId": "docs--4-adding-hosts--f25bc11364", - "meta": {} - }, - { - "url": "/docs/using-fleet/osquery-logs", - "title": "Osquery logs", - "lastModifiedAt": 1624631015000, - "htmlId": "docs--5-osquery-logs--b2e649cc1f", - "meta": {} - }, - { - "url": "/docs/using-fleet/monitoring-fleet", - "title": "Monitoring Fleet", - "lastModifiedAt": 1630357234000, - "htmlId": "docs--6-monitoring-fleet--b1fa6e4a69", - "meta": {} - }, - { - "url": "/docs/using-fleet/security-best-practices", - "title": "Security best practices", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--7-security-best-prac--ad931bb00b", - "meta": {} - }, - { - "url": "/docs/using-fleet/updating-fleet", - "title": "Updating Fleet", - "lastModifiedAt": 1630641746000, - "htmlId": "docs--8-updating-fleet--1887128e93", - "meta": {} - }, - { - "url": "/docs/using-fleet/permissions", - "title": "Permissions", - "lastModifiedAt": 1630415095000, - "htmlId": "docs--9-permissions--905e9c08da", - "meta": {} - }, - { - "url": "/docs/using-fleet/faq", - "title": "FAQ", - "lastModifiedAt": 1627511403000, - "htmlId": "docs--faq--f96c7228ae", - "meta": {} - }, - { - "url": "/docs/using-fleet", - "title": "Using Fleet", - "lastModifiedAt": 1626938622000, - "htmlId": "docs--readme--b097d08746", + "htmlId": "docs--readme--27004f4448", + "sectionRelativeRepoPath": "README.md", "meta": {} }, { "url": "/docs/deploying/installation", "title": "Installation", - "lastModifiedAt": 1625173055000, - "htmlId": "docs--1-installation--fe7d4e2e74", - "meta": {} - }, - { - "url": "/docs/deploying/example-deployment-scenarios", - "title": "Example deployment scenarios", - "lastModifiedAt": 1625588060000, - "htmlId": "docs--3-example-deployment--b850738ae0", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-installation--c1f7b7262d", + "sectionRelativeRepoPath": "02-Deploying/01-Installation.md", "meta": {} }, { "url": "/docs/deploying/configuration", "title": "Configuration", - "lastModifiedAt": 1631134512000, - "htmlId": "docs--2-configuration--a242085fa7", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-configuration--25bb47a163", + "sectionRelativeRepoPath": "02-Deploying/02-Configuration.md", + "meta": {} + }, + { + "url": "/docs/deploying/example-deployment-scenarios", + "title": "Example deployment scenarios", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--03-example-deploymen--1d32b988ab", + "sectionRelativeRepoPath": "02-Deploying/03-Example-deployment-scenarios.md", "meta": {} }, { "url": "/docs/deploying/fleetctl-agent-updates", "title": "Fleetctl agent updates", - "lastModifiedAt": 1631165652000, - "htmlId": "docs--4-fleetctl-agent-upd--f6d6a601d4", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-fleetctl-agent-up--92c6890fa9", + "sectionRelativeRepoPath": "02-Deploying/04-fleetctl-agent-updates.md", "meta": {} }, { "url": "/docs/deploying/faq", "title": "FAQ", - "lastModifiedAt": 1627511403000, - "htmlId": "docs--faq--7abb678d36", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--3ad91393ce", + "sectionRelativeRepoPath": "02-Deploying/FAQ.md", "meta": {} }, { "url": "/docs/deploying", "title": "Deploying", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--readme--a0a26f55e2", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--fb635b427f", + "sectionRelativeRepoPath": "02-Deploying/README.md", "meta": {} }, { "url": "/docs/contributing/building-fleet", "title": "Building Fleet", - "lastModifiedAt": 1629140343000, - "htmlId": "docs--1-building-fleet--a0d05ce171", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-building-fleet--abcea456d8", + "sectionRelativeRepoPath": "03-Contributing/01-Building-Fleet.md", "meta": {} }, { "url": "/docs/contributing/testing", "title": "Testing", - "lastModifiedAt": 1630685123000, - "htmlId": "docs--2-testing--20bd58879c", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-testing--2f307719a6", + "sectionRelativeRepoPath": "03-Contributing/02-Testing.md", "meta": {} }, { "url": "/docs/contributing/migrations", "title": "Migrations", - "lastModifiedAt": 1629743194000, - "htmlId": "docs--3-migrations--ee672f0676", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--03-migrations--b553b6254f", + "sectionRelativeRepoPath": "03-Contributing/03-Migrations.md", "meta": {} }, { "url": "/docs/contributing/committing-changes", "title": "Committing changes", - "lastModifiedAt": 1629140343000, - "htmlId": "docs--4-committing-changes--62d8075df1", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-committing-change--9b92fdc560", + "sectionRelativeRepoPath": "03-Contributing/04-Committing-Changes.md", "meta": {} }, { "url": "/docs/contributing/releasing-fleet", "title": "Releasing Fleet", - "lastModifiedAt": 1629929664000, - "htmlId": "docs--5-releasing-fleet--2b2a696ea0", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--05-releasing-fleet--1f39f77c64", + "sectionRelativeRepoPath": "03-Contributing/05-Releasing-Fleet.md", + "meta": {} + }, + { + "url": "/docs/contributing/seeding-data", + "title": "Seeding data", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--06-seeding-data--af5ac86a99", + "sectionRelativeRepoPath": "03-Contributing/06-Seeding-Data.md", "meta": {} }, { "url": "/docs/contributing/faq", "title": "FAQ", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--faq--92e0006bf2", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--1b33e57806", + "sectionRelativeRepoPath": "03-Contributing/FAQ.md", "meta": {} }, { "url": "/docs/contributing", "title": "Contributing", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--readme--d5e4f68946", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--6de1bc799d", + "sectionRelativeRepoPath": "03-Contributing/README.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleet-ui", + "title": "Fleet UI", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-fleet-ui--4b5755ee58", + "sectionRelativeRepoPath": "01-Using-Fleet/01-Fleet-UI.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleetctl-cli", + "title": "Fleetctl CLI", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-fleetctl-cli--2a521b49d6", + "sectionRelativeRepoPath": "01-Using-Fleet/02-fleetctl-CLI.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/rest-api", + "title": "REST API", + "lastModifiedAt": 1632174198000, + "htmlId": "docs--03-rest-api--f0b4e26bae", + "sectionRelativeRepoPath": "01-Using-Fleet/03-REST-API.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/adding-hosts", + "title": "Adding hosts", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-adding-hosts--9ffccb2221", + "sectionRelativeRepoPath": "01-Using-Fleet/04-Adding-hosts.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/osquery-logs", + "title": "Osquery logs", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--05-osquery-logs--7fbf2c5c5a", + "sectionRelativeRepoPath": "01-Using-Fleet/05-Osquery-logs.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/monitoring-fleet", + "title": "Monitoring Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--06-monitoring-fleet--83f7cca9f9", + "sectionRelativeRepoPath": "01-Using-Fleet/06-Monitoring-Fleet.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/security-best-practices", + "title": "Security best practices", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--07-security-best-pra--7ba1af6048", + "sectionRelativeRepoPath": "01-Using-Fleet/07-Security-best-practices.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/updating-fleet", + "title": "Updating Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--08-updating-fleet--3b4e821ee3", + "sectionRelativeRepoPath": "01-Using-Fleet/08-Updating-Fleet.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/permissions", + "title": "Permissions", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--09-permissions--eb9ac05ff5", + "sectionRelativeRepoPath": "01-Using-Fleet/09-Permissions.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/teams", + "title": "Teams", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--10-teams--bd0bdf9444", + "sectionRelativeRepoPath": "01-Using-Fleet/10-Teams.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/usage-statistics", + "title": "Usage statistics", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--11-usage-statistics--ccd73f532c", + "sectionRelativeRepoPath": "01-Using-Fleet/11-Usage-statistics.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/supported-browsers", + "title": "Supported browsers", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--12-supported-browser--c3a9c18d40", + "sectionRelativeRepoPath": "01-Using-Fleet/12-Supported-browsers.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/vulnerability-processing", + "title": "Vulnerability processing", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--13-vulnerability-pro--7a9b62b621", + "sectionRelativeRepoPath": "01-Using-Fleet/13-Vulnerability-Processing.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/faq", + "title": "FAQ", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--75e099695e", + "sectionRelativeRepoPath": "01-Using-Fleet/FAQ.md", + "meta": {} + }, + { + "url": "/docs/using-fleet", + "title": "Using Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--0b226f5257", + "sectionRelativeRepoPath": "01-Using-Fleet/README.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/learn-how-to-use-fleet", + "title": "Learn how to use Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--00-learn-how-to-use---95b515dfd1", + "sectionRelativeRepoPath": "01-Using-Fleet/00-Learn-how-to-use-Fleet.md", "meta": {} }, { "url": "/docs/using-fleet/configuration-files", "title": "Configuration files", - "lastModifiedAt": 1631296113000, - "htmlId": "docs--readme--7908cef8a3", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--dc5df431cb", + "sectionRelativeRepoPath": "01-Using-Fleet/configuration-files/README.md", "meta": {} }, { "url": "/docs/using-fleet/standard-query-library", "title": "Standard query library", - "lastModifiedAt": 1624049901000, - "htmlId": "docs--readme--d3c7d96146", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--db16aa6f37", + "sectionRelativeRepoPath": "01-Using-Fleet/standard-query-library/README.md", "meta": {} } ], @@ -264,11 +304,11 @@ "remediation": "N/A" }, { - "name": "Detect Linux hosts with high severity vulnerable versions of OpenSSL", + "name": "Get OpenSSL versions", "platforms": "Linux", "description": "Retrieves the OpenSSL version.", "query": "SELECT name AS name, version AS version, 'deb_packages' AS source FROM deb_packages WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'apt_sources' AS source FROM apt_sources WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'rpm_packages' AS source FROM rpm_packages WHERE name LIKE 'openssl%';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Zach Wasserman", @@ -277,15 +317,15 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "detect-linux-hosts-with-high-severity-vulnerable-versions-of-open-ssl", + "slug": "get-open-ssl-versions", "remediation": "N/A" }, { - "name": "Detect machines with Gatekeeper disabled", + "name": "Get whether Gatekeeper is disabled", "platforms": "macOS", "description": "Gatekeeper tries to ensure only trusted software is run on a mac machine.", "query": "SELECT * FROM gatekeeper WHERE assessments_enabled = 0;", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Zach Wasserman", @@ -294,16 +334,16 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "detect-machines-with-gatekeeper-disabled", + "slug": "get-whether-gatekeeper-is-disabled", "remediation": "N/A" }, { - "name": "Detect presence of authorized SSH keys", + "name": "Get authorized SSH keys", "platforms": "macOS, Linux", "description": "Presence of authorized SSH keys may be unusual on laptops. Could be completely normal on servers, but may be worth auditing for unusual keys and/or changes.", "query": "SELECT username, authorized_keys. * FROM users CROSS JOIN authorized_keys USING (uid);", - "purpose": "Detection", - "remediation": "Check out the linked table (https://github.com/fleetdm/fleet/blob/32b4d53e7f1428ce43b0f9fa52838cbe7b413eed/handbook/queries/detect-hosts-with-high-severity-vulnerable-versions-of-openssl.md#table-of-vulnerable-openssl-versions) to determine if the installed version is a high severity vulnerability and view the corresponding CVE(s)", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Mike Thomas", @@ -312,7 +352,7 @@ "htmlUrl": "https://github.com/mike-j-thomas" } ], - "slug": "detect-presence-of-authorized-ssh-keys" + "slug": "get-authorized-ssh-keys" }, { "name": "Get authorized keys for Local Accounts", @@ -706,12 +746,12 @@ "remediation": "N/A" }, { - "name": "Detect unencrypted SSH keys for local accounts", + "name": "Get unencrypted SSH keys for local accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0;", - "purpose": "Detection", - "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -720,15 +760,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-unencrypted-ssh-keys-for-local-accounts" + "slug": "get-unencrypted-ssh-keys-for-local-accounts" }, { - "name": "Detect unencrypted SSH keys for domain joined accounts", + "name": "Get unencrypted SSH keys for domain joined accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0 and username in (SELECT distinct(username) FROM last);", - "purpose": "Detection", - "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -737,7 +777,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-unencrypted-ssh-keys-for-domain-joined-accounts" + "slug": "get-unencrypted-ssh-keys-for-domain-joined-accounts" }, { "name": "Get crontab jobs", @@ -774,12 +814,12 @@ "remediation": "N/A" }, { - "name": "Detect dynamic linker hijacking on Linux (MITRE. T1574.006)", + "name": "Get dynamic linker hijacking on Linux (MITRE. T1574.006)", "platforms": "Linux", "description": "Detect any processes that run with LD_PRELOAD environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='LD_PRELOAD';", - "purpose": "Detection", - "remediation": "Identify the process/binary detected and confirm with the system's owner.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -788,15 +828,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" + "slug": "get-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" }, { - "name": "Detect dynamic linker hijacking on macOS (MITRE. T1574.006)", + "name": "Get dynamic linker hijacking on macOS (MITRE. T1574.006)", "platforms": "macOS", "description": "Detect any processes that run with DYLD_INSERT_LIBRARIES environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='DYLD_INSERT_LIBRARIES';", - "purpose": "Detection", - "remediation": "Identify the process/binary detected and confirm with the system's owner.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -805,7 +845,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" + "slug": "get-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" }, { "name": "Get etc hosts entries", @@ -859,11 +899,11 @@ "remediation": "N/A" }, { - "name": "Detect active user accounts on servers", + "name": "Get active user accounts on servers", "platforms": "Linux", "description": "Domain Joined environment normally have root or other service account only and users are SSH-ing using their Domain Accounts.", "query": "SELECT * FROM shadow WHERE password_status='active' and username!='root';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Ahmed Elshaer", @@ -872,15 +912,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-active-user-accounts-on-servers", + "slug": "get-active-user-accounts-on-servers", "remediation": "N/A" }, { - "name": "Detect Nmap scanner", + "name": "Get Nmap scanner", "platforms": "macOS, Linux, Windows, FreeBSD", - "description": "Detect Nmap scanner process, identify the user, parent, process details.", + "description": "Get Nmap scanner process, as well as its user, parent, and process details.", "query": "SELECT p.pid, name, p.path, cmdline, cwd, start_time, parent, (SELECT name FROM processes WHERE pid=p.parent) AS parent_name, (SELECT username FROM users WHERE uid=p.uid) AS username FROM processes as p WHERE cmdline like 'nmap%';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Ahmed Elshaer", @@ -889,7 +929,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-nmap-scanner", + "slug": "get-nmap-scanner", "remediation": "N/A" }, { @@ -944,11 +984,11 @@ "remediation": "N/A" }, { - "name": "Detect Windows print spooler remote code execution vulnerability", + "name": "Get Windows print spooler remote code execution vulnerability", "platforms": "Windows", "description": "Detects devices that are potentially vulnerable to CVE-2021-1675 because the print spooler service is not disabled.", "query": "SELECT CASE cnt WHEN 2 THEN \"TRUE\" ELSE \"FALSE\" END \"Vulnerable\" FROM (SELECT name start_type, COUNT(name) AS cnt FROM services WHERE name = 'NTDS' or (name = 'Spooler' and start_type <> 'DISABLED')) WHERE cnt = 2;", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": null, @@ -957,7 +997,7 @@ "htmlUrl": "https://github.com/maravedi" } ], - "slug": "detect-windows-print-spooler-remote-code-execution-vulnerability", + "slug": "get-windows-print-spooler-remote-code-execution-vulnerability", "remediation": "N/A" }, { @@ -978,7 +1018,7 @@ "remediation": "N/A" }, { - "name": "Find deleted files from disk", + "name": "Get processes that no longer exist on disk", "platforms": "Linux, macOS, Windows", "description": "Lists all processes of which the binary which launched them no longer exists on disk. Attackers often delete files from disk after launching process to mask presence.", "query": "SELECT name, path, pid FROM processes WHERE on_disk = 0;", @@ -991,11 +1031,96 @@ "htmlUrl": "https://github.com/alphabrevity" } ], - "slug": "find-deleted-files-from-disk", + "slug": "get-processes-that-no-longer-exist-on-disk", + "remediation": "N/A" + }, + { + "name": "Get user files matching a specific hash", + "platforms": "macOS, Linux", + "description": "Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.)", + "query": "SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE \"\"/Users/%/Documents/%%\"\") AND sha256 = \"\"16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75\"\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-user-files-matching-a-specific-hash", + "remediation": "N/A" + }, + { + "name": "Get local administrator accounts on macOS", + "platforms": "macOS", + "description": "The query allows you to check macOS systems for local administrator accounts.", + "query": "SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid;", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-local-administrator-accounts-on-mac-os", + "remediation": "N/A" + }, + { + "name": "Get all listening ports, by process", + "platforms": "Linux, macOS, Windows", + "description": "List ports that are listening on all interfaces, along with the process to which they are attached.", + "query": "SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = \"0.0.0.0\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-all-listening-ports-by-process", + "remediation": "N/A" + }, + { + "name": "Get whether TeamViewer is installed/running", + "platforms": "Windows", + "description": "Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine.", + "query": "SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE \"%teamviewer%\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-whether-team-viewer-is-installed-running", + "remediation": "N/A" + }, + { + "name": "Get malicious Python backdoors", + "platforms": "macOS, Linux, Windows", + "description": "Watches for the backdoored Python packages installed on system. See (http://www.nbu.gov.sk/skcsirt-sa-20170909-pypi/index.html)", + "query": "select case cnt when 0 then \"NONE_INSTALLED\" else \"INSTALLED\" end as \"Malicious Python Packages\",package_name,package_version from (select count(name) as cnt,nameas package_name,version as package_version,path as package_pathfrom python_packages where package_name in ('acqusition','apidev-coop','bzip','crypt','django-server','pwd','setup-tools','telnet','urlib3','urllib'));", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-malicious-python-backdoors", "remediation": "N/A" } ], - "queryLibraryYmlRepoPath": "docs/1-Using-Fleet/standard-query-library/standard-query-library.yml", + "queryLibraryYmlRepoPath": "docs/01-Using-Fleet/standard-query-library/standard-query-library.yml", "compiledPagePartialsAppPath": "views/partials/built-from-markdown" } } diff --git a/website/api/controllers/docs/view-basic-documentation.js b/website/api/controllers/docs/view-basic-documentation.js index 769474f0ad..463d134991 100644 --- a/website/api/controllers/docs/view-basic-documentation.js +++ b/website/api/controllers/docs/view-basic-documentation.js @@ -44,14 +44,26 @@ module.exports = { // console.log('pageUrlSuffix:',pageUrlSuffix); // console.log('SECTION_URL_PREFIX + "/" + _.trim(pageUrlSuffix, "/"):',SECTION_URL_PREFIX + '/' + _.trim(pageUrlSuffix, '/')); // console.log('thisPage:',thisPage); - if (!thisPage) { - throw 'notFound'; - } - if (false) { - // TODO: add "redirect" exit and handle mismatched capitalization / extra slashes by redirecting to the correct URL. e.g. "http://localhost:2024/docs//usiNG-fleet///" Partial example of this: https://github.com/sailshq/sailsjs.com/blob/b53c6e6a90c9afdf89e5cae00b9c9dd3f391b0e7/api/controllers/documentation/view-documentation.js#L161-L166 - let revisedUrl = 'todo'; - throw {redirect: revisedUrl}; + // Setting a flag if the pageUrlSuffix doesn't match any existing page, or if the page it matches doesn't exactly match the pageUrlSuffix provided + // Note: because this also handles the docs landing page and a pageUrlSuffix might not have provided, we set this flag to false if the url is just '/docs' + let needsRedirectMaybe = (!thisPage || (thisPage.url !== '/docs/'+pageUrlSuffix && thisPage.url !== '/docs')); + + if (needsRedirectMaybe) { + // Creating a lower case, repeating-slashless pageUrlSuffix + let multipleSlashesRegex = /\/{2,}/g; + let modifiedPageUrlSuffix = pageUrlSuffix.toLowerCase().replace(multipleSlashesRegex, '/'); + // Finding the appropriate page content using the modified pageUrlSuffix. + let revisedPage = _.find(sails.config.builtStaticContent.markdownPages, { + url: _.trimRight(SECTION_URL_PREFIX + '/' + _.trim(modifiedPageUrlSuffix, '/'), '/') + }); + if(revisedPage) { + // If we matched a page with the modified pageUrlSuffix, then redirect to that. + throw {redirect: revisedPage.url}; + } else { + // If no page was found, throw a 404 error. + throw 'notFound'; + } } // Respond with view. diff --git a/website/api/controllers/webhooks/receive-from-github.js b/website/api/controllers/webhooks/receive-from-github.js index 190b899f88..20e5abd03e 100644 --- a/website/api/controllers/webhooks/receive-from-github.js +++ b/website/api/controllers/webhooks/receive-from-github.js @@ -25,8 +25,24 @@ module.exports = { let GitHub = require('machinepack-github'); let GREEN_LABEL_COLOR = 'C2E0C6';// « Used in multiple places below. - let GITHUB_USERNAMES_OF_BOTS = [// « Used in multiple places below. - 'sailsbot' + let GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS = [// « Used in multiple places below. + 'sailsbot', + 'fleet-release', + 'noahtalerman', + 'mike-j-thomas', + 'mikermcneil', + 'lukeheath', + 'zwass', + 'rlynnj11', + 'martavis', + 'rachelelysia', + 'gillespi314', + 'chiiph', + 'mna', + 'edwardsb', + 'alphabrevity', + 'eashaw', + 'drewbakerfdm' ]; let GITHUB_USERNAME_OF_DRI_FOR_LABELS = 'noahtalerman';// « Used below @@ -94,7 +110,7 @@ module.exports = { // `For help with questions about Sails, [click here](http://sailsjs.com/support).\n`; // } // } else { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); // if (wasReopenedByBot) { // newBotComment = '';// « checked below // } else { @@ -146,7 +162,7 @@ module.exports = { // if (action === 'edited' && pr.state !== 'open') { // // If this is an edit to an already-closed pull request, then do nothing. // } else if (action === 'reopened') { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); // if (!wasReopenedByBot) { // let newBotComment = // `Oh hey again, @${issueOrPr.user.login}. Now that this pull request is reopened, it's on our radar. Please let us know if there's any new information we should be aware of!\n`+ @@ -209,7 +225,7 @@ module.exports = { let repo = repository.name; let issueNumber = issueOrPr.number; - let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); if (!wasPostedByBot) { let greenLabels = _.filter(issueOrPr.labels, ({color}) => color === GREEN_LABEL_COLOR); await sails.helpers.flow.simultaneouslyForEach(greenLabels, async(greenLabel)=>{ @@ -217,10 +233,10 @@ module.exports = { });//∞ß }//fi } else if ( - (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS.includes(comment.user.login))|| + (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(comment.user.login))|| (ghNoun === 'commit_comment' && ['created'].includes(action))|| (ghNoun === 'label' && ['created','edited','deleted'].includes(action) && GITHUB_USERNAME_OF_DRI_FOR_LABELS !== sender.login)||//« exempt label changes made by the directly responsible individual for labels, because otherwise when process changes/fiddlings happen, they can otherwise end up making too much noise in Slack - (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 24*60*60*1000 ) ) + (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 7*24*60*60*1000 ) ) ) { // ██╗███╗ ██╗███████╗ ██████╗ ██████╗ ███╗ ███╗ ██╗ ██╗███████╗ // ██║████╗ ██║██╔════╝██╔═══██╗██╔══██╗████╗ ████║ ██║ ██║██╔════╝ @@ -254,7 +270,7 @@ module.exports = { : (ghNoun === 'label') ? `@${sender.login} just ${action} a GitHub label "*${label.name}*" (#${label.color}) in ${repository.owner.login}/${repository.name}.\n\n> To manage labels in ${repository.owner.login}/${repository.name}, visit https://github.com/${encodeURIComponent(repository.owner.login)}/${encodeURIComponent(repository.name)}/labels` : - `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >24 hours (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` + `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >7 days (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` )+`\n` }, {'Content-Type':'application/json'} diff --git a/website/api/controllers/webhooks/receive-usage-analytics.js b/website/api/controllers/webhooks/receive-usage-analytics.js index 5b3fc6bcb9..7895e1ca3e 100644 --- a/website/api/controllers/webhooks/receive-usage-analytics.js +++ b/website/api/controllers/webhooks/receive-usage-analytics.js @@ -4,7 +4,7 @@ module.exports = { friendlyName: 'Receive usage analytics', - description: '', + description: 'Receive anonymous usage analytics from deployments of Fleet running in production. (Not fleetctl preview or dev-mode deployments.)', inputs: { diff --git a/website/api/helpers/strings/to-html.js b/website/api/helpers/strings/to-html.js index 09bfabc6df..99f0dc8f96 100644 --- a/website/api/helpers/strings/to-html.js +++ b/website/api/helpers/strings/to-html.js @@ -73,8 +73,14 @@ module.exports = { smartLists: true, smartypants: false, }; - - if (inputs.addIdsToHeadings === false) { + if (inputs.addIdsToHeadings === true) { + var headingRenderer = new marked.Renderer(); + headingRenderer.heading = function (text, level) { + var headingID = _.kebabCase(text); + return ''+text+'\n'; + }; + markedOpts.renderer = headingRenderer; + } else { var renderer = new marked.Renderer(); renderer.heading = function (text, level) { return ''+text+''; diff --git a/website/assets/images/icon-link-16x16@2x.png b/website/assets/images/icon-link-16x16@2x.png new file mode 100644 index 0000000000..41120a4844 Binary files /dev/null and b/website/assets/images/icon-link-16x16@2x.png differ diff --git a/website/assets/js/pages/docs/basic-documentation.page.js b/website/assets/js/pages/docs/basic-documentation.page.js index e7fcdfe35d..c5b6e046fc 100644 --- a/website/assets/js/pages/docs/basic-documentation.page.js +++ b/website/assets/js/pages/docs/basic-documentation.page.js @@ -104,11 +104,13 @@ parasails.registerPage('basic-documentation', { // console.log(subtopics); this.subtopics = (() => { - let subtopics = $('#body-content').find('h2').map((_, el) => el.innerHTML); + let subtopics = $('#body-content').find('h2').map((_, el) => el.innerText); subtopics = $.makeArray(subtopics).map((title) => { + // Removing all apostrophes from the title to keep _.kebabCase() from turning words like 'user’s' into 'user-s' + let kebabCaseFriendlyTitle = title.replace(/[\’]/g, ''); return { title, - url: '#' + _.kebabCase(title), + url: '#' + _.kebabCase(kebabCaseFriendlyTitle), }; }); return subtopics; diff --git a/website/assets/styles/pages/docs/basic-documentation.less b/website/assets/styles/pages/docs/basic-documentation.less index 6f83ce2d41..4d9956c5dd 100644 --- a/website/assets/styles/pages/docs/basic-documentation.less +++ b/website/assets/styles/pages/docs/basic-documentation.less @@ -38,6 +38,17 @@ } } + .markdown-heading:hover { + + .markdown-link { + height: 16px; + vertical-align: middle; + margin-left: 8px; + content: url('/images/icon-link-16x16@2x.png'); + } + + } + [purpose='search'] { @@ -187,6 +198,56 @@ color: @core-fleet-black; } + } + .edit-button-container { + position: relative; + + .edit-button { + color: @core-vibrant-blue; + opacity: 0; + font-size: 12px; + position: absolute; + right: 4px; + top: 8px; + cursor: pointer; + border: 1px solid @core-vibrant-blue; + border-radius: 4px; + padding: 4px 8px; + text-decoration: none; + + .edit-link { + color: @core-vibrant-blue; + + } + .edit-pencil { + height: 16px; + padding-right: 5px; + } + + } + + .edit-button:hover { + + background: @core-vibrant-blue; + + a { + text-decoration: none; + } + .edit-link { + color: @accent-white; + + } + } + + } + + &:hover { + + .edit-button { + opacity: 1; + + } + } [purpose='search'] { diff --git a/website/assets/styles/pages/docs/code-blocks.less b/website/assets/styles/pages/docs/code-blocks.less index 4a01c5a023..d7caa9932c 100644 --- a/website/assets/styles/pages/docs/code-blocks.less +++ b/website/assets/styles/pages/docs/code-blocks.less @@ -1,6 +1,6 @@ // lesshint-disable spaceAroundComma, trailingWhitespace - code:not(.nohighlight, .hljs) { + code:not(.hljs):not(.nohighlight) { background-color: @ui-off-white; border: 1px solid @border-lt-gray; color: @core-fleet-black-75; @@ -16,7 +16,7 @@ } - pre:not(.algolia-autocomplete, .ds-dropdown-menu) { + pre:not(.algolia-autocomplete):not(.ds-dropdown-menu):not(.json) { padding: 24px; border: 1px solid @border-lt-gray; border-radius: 6px; diff --git a/website/config/routes.js b/website/config/routes.js index 6a1b6ffdeb..c107ee8700 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -13,19 +13,77 @@ module.exports.routes = { // ╦ ╦╔═╗╔╗ ╔═╗╔═╗╔═╗╔═╗╔═╗ // ║║║║╣ ╠╩╗╠═╝╠═╣║ ╦║╣ ╚═╗ // ╚╩╝╚═╝╚═╝╩ ╩ ╩╚═╝╚═╝╚═╝ - 'GET /': { action: 'view-homepage-or-redirect', locals: { isHomepage: true } }, - 'GET /company/contact': { action: 'view-contact' }, - 'GET /get-started': { action: 'view-get-started' }, - 'GET /pricing': { action: 'view-pricing' }, - 'GET /press-kit': { action: 'view-press-kit' }, + 'GET /': { + action: 'view-homepage-or-redirect', + locals: { isHomepage: true } + }, - 'GET /queries': { action: 'view-query-library' }, - 'GET /queries/:slug': { action: 'view-query-detail' }, + 'GET /company/contact': { + action: 'view-contact', + locals:{ + title: 'Contact us | Fleet for osquery', + description: 'Get in touch with our team.' + } + }, - 'GET /docs/?*': { skipAssets: false, action: 'docs/view-basic-documentation' },// handles /docs and /docs/foo/bar - // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar + 'GET /get-started': { + action: 'view-get-started' , + locals:{ + title: 'Get Started | Fleet for osquery', + description: 'Learn about getting started with Fleet.' + } + }, - 'GET /transparency': { action: 'view-transparency' }, + 'GET /pricing': { + action: 'view-pricing', + locals:{ + title: 'Pricing | Fleet for osquery', + description: 'View Fleet plans and pricing details.' + } + }, + + 'GET /press-kit': { + action: 'view-press-kit', + locals:{ + title: 'Press kit | Fleet for osquery', + description: 'Download Fleet logos, wallpapers, and screenshots.' + } + }, + + 'GET /queries': { + action: 'view-query-library', + locals:{ + title: 'Queries | Fleet for osquery', + description: 'A growing collection of useful queries for organizations deploying Fleet and osquery.' + } + }, + + 'GET /queries/:slug': { + action: 'view-query-detail', + locals:{ + title: 'Query details | Fleet for osquery', + description: 'View more information about a query in Fleet\'s standard query library', + } + }, + + 'GET /docs/?*': { + skipAssets: false, + action: 'docs/view-basic-documentation', + locals:{ + title: 'Documentation | Fleet for osquery', + description: 'Documentation for Fleet for osquery.', + } + },// handles /docs and /docs/foo/bar + + // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar + + 'GET /transparency': { + action: 'view-transparency', + locals:{ + title: 'Transparency | Fleet for osquery', + description: 'Learn what data osquery can see.', + } + }, diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index 2d9b24a15e..f52a6044bf 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -25,8 +25,8 @@ module.exports = { await sails.helpers.flow.simultaneously([ async()=>{// Parse query library from YAML and prepare to bake them into the Sails app's configuration. - let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/1-Using-Fleet/standard-query-library/standard-query-library.yml'; - let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)); + let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml'; + let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)).intercept('doesNotExist', (err)=>new Error(`Could not find standard query library YAML file at "${RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO}". Was it accidentally moved? Raw error: `+err.message)); let queriesWithProblematicRemediations = []; let queriesWithProblematicContributors = []; @@ -226,13 +226,19 @@ module.exports = { // to some page on the destination site where this will be hosted, like `(*.)?fleetdm.com`. // If external, add target="_blank" so the link will open in a new tab. let isExternal = ! hrefString.match(/^href=\"https?:\/\/([^\.]+\.)*fleetdm\.com/g);// « FUTURE: make this smarter with sails.config.baseUrl + _.escapeRegExp() + // Check if this link is to fleetdm.com or www.fleetdm.com. + let isBaseUrl = hrefString.match(/^(href="https?:\/\/)([^\.]+\.)*fleetdm\.com"$/g); if (isExternal) { return hrefString.replace(/(href="https?:\/\/([^"]+)")/g, '$1 target="_blank"'); } else { // Otherwise, change the link to be web root relative. // (e.g. 'href="http://sailsjs.com/documentation/concepts"'' becomes simply 'href="/documentation/concepts"'') // > Note: See the Git version history of "compile-markdown-content.js" in the sailsjs.com website repo for examples of ways this can work across versioned subdomains. - return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); + if (isBaseUrl) { + return hrefString.replace(/href="https?:\/\//, '').replace(/([^\.]+\.)*fleetdm\.com/, 'href="/'); + } else { + return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); + } } });//∞ @@ -293,12 +299,17 @@ module.exports = { await sails.helpers.fs.write(htmlOutputPath, htmlString); } + // Determine the path of the file in the fleet repo so we can link to + // the file on github from fleetdm.com (e.g. 1-Using-Fleet/2-fleetctl-CLI.md) + let sectionRelativeRepoPath = path.relative(path.join(topLvlRepoPath, sectionRepoPath), path.resolve(pageSourcePath)); + // Append to what will become configuration for the Sails app. builtStaticContent.markdownPages.push({ url: rootRelativeUrlPath, title: pageTitle, lastModifiedAt: lastModifiedAt, htmlId: htmlId, + sectionRelativeRepoPath: sectionRelativeRepoPath, meta: _.omit(embeddedMetadata, 'title') }); } diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index 2f956ccbbd..4c3854afb4 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -7,14 +7,15 @@ %> - Fleet for osquery | Open source device management + <%= typeof title !== 'undefined' ? title : 'Fleet for osquery | Open source device management' %> + <% /* Viewport tag for sensible mobile support */ %> - - + + <% /* Script tags should normally be included further down the page- but any scripts that load fonts (e.g. Fontawesome ≥v5) are special exceptions to the @@ -134,7 +135,7 @@ Blog Jobs Contribute - Press Kit + Logos/artwork Hall of fame
diff --git a/website/views/pages/docs/basic-documentation.ejs b/website/views/pages/docs/basic-documentation.ejs index c0b7434fa7..7cc744d5d3 100644 --- a/website/views/pages/docs/basic-documentation.ejs +++ b/website/views/pages/docs/basic-documentation.ejs @@ -189,6 +189,13 @@
+ +
+
+ Edit page +
+
+ <%- partial( path.relative( path.dirname(__filename), @@ -205,7 +212,7 @@

Is there something missing?

- If you notice something we've missed or could be improved on, please follow this link and submit a pull request to the Fleet repo. + If you notice something we’ve missed, or that could be improved, please click here to edit this page.

diff --git a/website/views/pages/homepage.ejs b/website/views/pages/homepage.ejs index b877c20cc6..18792d2001 100644 --- a/website/views/pages/homepage.ejs +++ b/website/views/pages/homepage.ejs @@ -12,8 +12,8 @@
Quickly deploy osquery and scale your fleet to 100,000+ devices on top of a stable core technology.
-
- Get started with Fleet +
+ Try it out Watch video @@ -36,7 +36,7 @@

- Inventory management + Single source or truth

Track and segment your enrolled hosts. Search by important details, and zoom in on individual targets. @@ -67,7 +67,7 @@

- Deployment + Deploy anywhere

Fleet is self-hosted and self-managed, and can be run within your own data centers or in the cloud. @@ -136,7 +136,7 @@ class="btn btn-block btn-md btn-primary mx-4" href="/get-started" > - Get started + Try it out - Get started + Try it out

Contributors

Want to add your own query? Please submit a pull request - + over on GitHub .

diff --git a/yarn.lock b/yarn.lock index ae51c26324..4afebf1bc4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1122,10 +1122,10 @@ date-fns "^1.27.2" figures "^1.7.0" -"@cypress/request@^2.88.5": - version "2.88.5" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.5.tgz#8d7ecd17b53a849cfd5ab06d5abe7d84976375d7" - integrity sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA== +"@cypress/request@^2.88.5", "@cypress/request@^2.88.6": + version "2.88.6" + resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.6.tgz#a970dd675befc6bdf8a8921576c01f51cc5798e9" + integrity sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ== dependencies: aws-sign2 "~0.7.0" aws4 "^1.8.0" @@ -1140,13 +1140,12 @@ isstream "~0.1.2" json-stringify-safe "~5.0.1" mime-types "~2.1.19" - oauth-sign "~0.9.0" performance-now "^2.1.0" qs "~6.5.2" safe-buffer "^5.1.2" tough-cookie "~2.5.0" tunnel-agent "^0.6.0" - uuid "^3.3.2" + uuid "^8.3.2" "@cypress/xvfb@^1.2.4": version "1.2.4" @@ -1692,6 +1691,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.50.tgz#e9b2e85fafc15f2a8aa8fdd41091b983da5fd6ee" integrity sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w== +"@types/node@^14.14.31": + version "14.17.16" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.16.tgz#2b9252bd4fdf0393696190cd9550901dd967c777" + integrity sha512-WiFf2izl01P1CpeY8WqFAeKWwByMueBEkND38EcN8N68qb0aDG3oIS1P5MhAX5kUdr469qRyqsY/MjanLjsFbQ== + "@types/node@^14.14.37": version "14.14.37" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.37.tgz#a3dd8da4eb84a996c36e331df98d82abd76b516e" @@ -1784,15 +1788,15 @@ "@types/prop-types" "*" csstype "^3.0.2" -"@types/sinonjs__fake-timers@^6.0.1": - version "6.0.2" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz#3a84cf5ec3249439015e14049bd3161419bf9eae" - integrity sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg== +"@types/sinonjs__fake-timers@^6.0.1", "@types/sinonjs__fake-timers@^6.0.2": + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08" + integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g== "@types/sizzle@^2.3.2": - version "2.3.2" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.2.tgz#a811b8c18e2babab7d542b3365887ae2e4d9de47" - integrity sha512-7EJYyKTL7tFR8+gDbB6Wwz/arpGa0Mywk1TJbNzKzHtzbwVmY4HR9WqS5VV7dsBUKQmPNr192jHr/VpBluj/hg== + version "2.3.3" + resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" + integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== "@types/sockjs-client@^1.5.1": version "1.5.1" @@ -1828,6 +1832,13 @@ dependencies: "@types/yargs-parser" "*" +"@types/yauzl@^2.9.1": + version "2.9.2" + resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" + integrity sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA== + dependencies: + "@types/node" "*" + "@typescript-eslint/eslint-plugin@^4.15.2": version "4.15.2" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.15.2.tgz#981b26b4076c62a5a55873fbef3fe98f83360c61" @@ -2110,6 +2121,14 @@ acorn@^8.1.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.1.1.tgz#fb0026885b9ac9f48bac1e185e4af472971149ff" integrity sha512-xYiIVjNuqtKXMxlRMDc6mZUhXehod4a3gbZ1qRlM7icK4EbxUFNLhWoPblCvFtB2Y9CIqHP3CF/rdxLItaQv8g== +aggregate-error@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" + integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== + dependencies: + clean-stack "^2.0.0" + indent-string "^4.0.0" + airbnb-prop-types@^2.16.0: version "2.16.0" resolved "https://registry.yarnpkg.com/airbnb-prop-types/-/airbnb-prop-types-2.16.0.tgz#b96274cefa1abb14f623f804173ee97c13971dc2" @@ -2202,6 +2221,13 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.11.0" +ansi-escapes@^4.3.0: + version "4.3.2" + resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== + dependencies: + type-fest "^0.21.3" + ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" @@ -2223,9 +2249,9 @@ ansi-regex@^4.1.0: integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^2.2.1: version "2.2.1" @@ -2277,7 +2303,7 @@ aproba@^1.0.3, aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -arch@^2.1.2: +arch@^2.1.2, arch@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== @@ -2487,9 +2513,9 @@ async@^2.5.0: lodash "^4.17.11" async@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== + version "3.2.1" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8" + integrity sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg== asynckit@^0.4.0: version "0.4.0" @@ -2537,9 +2563,9 @@ aws-sign2@~0.7.0: integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= aws4@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" - integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== axios@^0.21.1: version "0.21.1" @@ -3391,9 +3417,9 @@ balanced-match@^0.4.2: integrity sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg= balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64-js@^1.0.2: version "1.3.0" @@ -3445,7 +3471,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -blob-util@2.0.2: +blob-util@2.0.2, blob-util@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== @@ -3667,9 +3693,9 @@ buffer-crc32@~0.2.3: integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer-xor@^1.0.3: version "1.0.3" @@ -3874,9 +3900,9 @@ chalk@^3.0.0: supports-color "^7.1.0" chalk@^4.0.0, chalk@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" - integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" @@ -3965,6 +3991,11 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== +ci-info@^3.1.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.2.0.tgz#2876cb948a498797b5236f0095bc057d0dca38b6" + integrity sha512-dVqRX7fLUm8J6FgHJ418XuIgDLZDkYcDFTeL6TA2gt5WlIZUQrrH6EZrNClwT/H0FateUsZkGIOPRrLbP+PR9A== + cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -4012,6 +4043,11 @@ clean-css@4.2.x: dependencies: source-map "~0.6.0" +clean-stack@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== + cli-cursor@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" @@ -4026,6 +4062,13 @@ cli-cursor@^2.0.0, cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" +cli-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== + dependencies: + restore-cursor "^3.1.0" + cli-table3@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.0.tgz#b7b1bc65ca8e7b5cef9124e13dc2b21e2ce4faee" @@ -4044,6 +4087,14 @@ cli-truncate@^0.2.1: slice-ansi "0.0.4" string-width "^1.0.1" +cli-truncate@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" + integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== + dependencies: + slice-ansi "^3.0.0" + string-width "^4.2.0" + cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -4166,6 +4217,11 @@ colorette@^1.2.1: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== +colorette@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" + integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== + colormin@^1.0.5: version "1.1.2" resolved "https://registry.yarnpkg.com/colormin/-/colormin-1.1.2.tgz#ea2f7420a72b96881a38aae59ec124a6f7298133" @@ -4176,9 +4232,9 @@ colormin@^1.0.5: has "^1.0.1" colors@^1.1.2: - version "1.3.3" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d" - integrity sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg== + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== colors@~1.1.2: version "1.1.2" @@ -4353,11 +4409,16 @@ core-js@^2.4.0, core-js@^2.5.0: resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.9.tgz#6b4b214620c834152e179323727fc19741b084f2" integrity sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A== -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a" @@ -4676,47 +4737,48 @@ cypress@*: url "^0.11.0" yauzl "^2.10.0" -cypress@^6.9.1: - version "6.9.1" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.9.1.tgz#ce1106bfdc47f8d76381dba63f943447883f864c" - integrity sha512-/RVx6sOhsyTR9sd9v0BHI4tnDZAhsH9rNat7CIKCUEr5VPWxyfGH0EzK4IHhAqAH8vjFcD4U14tPiJXshoUrmQ== +cypress@8.4.0: + version "8.4.0" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-8.4.0.tgz#09ec06a73f1cb10121c103cba15076e659e24876" + integrity sha512-RtVgGFR06ikyMaq/VqapeqOjGaIA42PpK7F0qe1MCiFArfUuJECsLmeYaOA+1TlmNUgJNMSF5fWKkZIJr5Uc7w== dependencies: - "@cypress/listr-verbose-renderer" "^0.4.1" - "@cypress/request" "^2.88.5" + "@cypress/request" "^2.88.6" "@cypress/xvfb" "^1.2.4" - "@types/node" "12.12.50" - "@types/sinonjs__fake-timers" "^6.0.1" + "@types/node" "^14.14.31" + "@types/sinonjs__fake-timers" "^6.0.2" "@types/sizzle" "^2.3.2" - arch "^2.1.2" - blob-util "2.0.2" + arch "^2.2.0" + blob-util "^2.0.2" bluebird "^3.7.2" cachedir "^2.3.0" chalk "^4.1.0" check-more-types "^2.24.0" + cli-cursor "^3.1.0" cli-table3 "~0.6.0" commander "^5.1.0" common-tags "^1.8.0" - dayjs "^1.9.3" - debug "4.3.2" - eventemitter2 "^6.4.2" - execa "^4.0.2" + dayjs "^1.10.4" + debug "^4.3.2" + enquirer "^2.3.6" + eventemitter2 "^6.4.3" + execa "4.1.0" executable "^4.1.1" - extract-zip "^1.7.0" - fs-extra "^9.0.1" + extract-zip "2.0.1" + figures "^3.2.0" + fs-extra "^9.1.0" getos "^3.2.1" - is-ci "^2.0.0" - is-installed-globally "^0.3.2" + is-ci "^3.0.0" + is-installed-globally "~0.4.0" lazy-ass "^1.6.0" - listr "^0.14.3" - lodash "^4.17.19" + listr2 "^3.8.3" + lodash "^4.17.21" log-symbols "^4.0.0" minimist "^1.2.5" - moment "^2.29.1" ospath "^1.2.2" - pretty-bytes "^5.4.1" + pretty-bytes "^5.6.0" ramda "~0.27.1" request-progress "^3.0.0" - supports-color "^7.2.0" + supports-color "^8.1.1" tmp "~0.2.1" untildify "^4.0.0" url "^0.11.0" @@ -4753,10 +4815,10 @@ date-now@^0.1.4: resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" integrity sha1-6vQ5/U1ISK105cx9vvIAZyueNFs= -dayjs@^1.9.3: - version "1.10.4" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.4.tgz#8e544a9b8683f61783f570980a8a80eaf54ab1e2" - integrity sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw== +dayjs@^1.10.4, dayjs@^1.9.3: + version "1.10.7" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.7.tgz#2cf5f91add28116748440866a0a1d26f3a6ce468" + integrity sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig== debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: version "2.6.9" @@ -4765,7 +4827,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4.3.2: +debug@4.3.2, debug@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -5176,13 +5238,20 @@ encoding@^0.1.11: dependencies: iconv-lite "^0.6.2" -end-of-stream@^1.0.0, end-of-stream@^1.1.0: +end-of-stream@^1.0.0: version "1.4.1" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" integrity sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q== dependencies: once "^1.4.0" +end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + enhanced-resolve@4.1.0, enhanced-resolve@^4.0.0, enhanced-resolve@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" @@ -5201,7 +5270,7 @@ enhanced-resolve@~0.9.0: memory-fs "^0.2.0" tapable "^0.1.8" -enquirer@^2.3.5: +enquirer@^2.3.5, enquirer@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -5717,7 +5786,7 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -eventemitter2@^6.4.2: +eventemitter2@^6.4.2, eventemitter2@^6.4.3: version "6.4.4" resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== @@ -5747,20 +5816,7 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" integrity sha512-sEFIkc61v75sWeOe72qyrqg2Qg0OuLESziUDk/O/z2qgS15y2gWVFrI6f2Qn/qw/0/NCfCEsmNA4zOjkwEZT1A== -execa@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" - integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== - dependencies: - cross-spawn "^6.0.0" - get-stream "^4.0.0" - is-stream "^1.1.0" - npm-run-path "^2.0.0" - p-finally "^1.0.0" - signal-exit "^3.0.0" - strip-eof "^1.0.0" - -execa@^4.0.0, execa@^4.0.2: +execa@4.1.0, execa@^4.0.0, execa@^4.0.2: version "4.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== @@ -5775,6 +5831,19 @@ execa@^4.0.0, execa@^4.0.2: signal-exit "^3.0.2" strip-final-newline "^2.0.0" +execa@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" + integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== + dependencies: + cross-spawn "^6.0.0" + get-stream "^4.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + executable@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" @@ -5933,6 +6002,17 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" +extract-zip@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + extract-zip@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -5981,9 +6061,9 @@ fast-glob@^3.1.1: picomatch "^2.2.1" fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I= + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.4: version "2.0.6" @@ -6056,6 +6136,13 @@ figures@^2.0.0: dependencies: escape-string-regexp "^1.0.5" +figures@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" + integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== + dependencies: + escape-string-regexp "^1.0.5" + file-entry-cache@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -6287,7 +6374,7 @@ from2@^2.1.0: inherits "^2.0.1" readable-stream "^2.0.0" -fs-extra@^9.0.1: +fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== @@ -6439,7 +6526,7 @@ get-stream@^4.0.0: dependencies: pump "^3.0.0" -get-stream@^5.0.0: +get-stream@^5.0.0, get-stream@^5.1.0: version "5.2.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== @@ -6506,7 +6593,7 @@ glob@^5.0.13: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@~7.1.1: +glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.4, glob@~7.1.1: version "7.1.4" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255" integrity sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A== @@ -6518,6 +6605,18 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, gl once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.1.3: + version "7.1.7" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" + integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + global-dirs@^2.0.1: version "2.1.0" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.1.0.tgz#e9046a49c806ff04d6c1825e196c8f0091e8df4d" @@ -6525,6 +6624,13 @@ global-dirs@^2.0.1: dependencies: ini "1.3.7" +global-dirs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" + integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== + dependencies: + ini "2.0.0" + global-modules@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -6599,15 +6705,15 @@ globule@^1.0.0: lodash "~4.17.10" minimatch "~3.0.2" -graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6: +graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2: version "4.2.0" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b" integrity sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg== -graceful-fs@^4.2.0: - version "4.2.6" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" - integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.8" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" + integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== graceful-fs@^4.2.4: version "4.2.4" @@ -7150,6 +7256,11 @@ ini@1.3.7: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.7.tgz#a09363e1911972ea16d7a8851005d84cf09a9a84" integrity sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ== +ini@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" @@ -7285,6 +7396,13 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" +is-ci@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.0.tgz#c7e7be3c9d8eef7d0fa144390bd1e4b88dc4c994" + integrity sha512-kDXyttuLeslKAHYL/K28F2YkM3x5jvFPEw3yXbRptXydjD9rpLEz+C5K5iutY9ZiUu6AP41JdvRQwF4Iqs4ZCQ== + dependencies: + ci-info "^3.1.1" + is-core-module@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946" @@ -7453,6 +7571,14 @@ is-installed-globally@^0.3.2: global-dirs "^2.0.1" is-path-inside "^3.0.1" +is-installed-globally@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" + integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== + dependencies: + global-dirs "^3.0.0" + is-path-inside "^3.0.2" + is-negative-zero@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" @@ -7499,10 +7625,10 @@ is-observable@^1.1.0: dependencies: symbol-observable "^1.1.0" -is-path-inside@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" - integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== +is-path-inside@^3.0.1, is-path-inside@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== is-plain-obj@^1.0.0: version "1.1.0" @@ -7564,9 +7690,9 @@ is-stream@^1.0.1, is-stream@^1.1.0: integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= is-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" - integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-string@^1.0.4: version "1.0.4" @@ -7609,6 +7735,11 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= +is-unicode-supported@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== + is-utf8@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" @@ -8477,6 +8608,19 @@ listr-verbose-renderer@^0.5.0: date-fns "^1.27.2" figures "^2.0.0" +listr2@^3.8.3: + version "3.12.1" + resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.12.1.tgz#75e515b86c66b60baf253542cc0dced6b60fedaf" + integrity sha512-oB1DlXlCzGPbvWhqYBZUQEPJKqsmebQWofXG6Mpbe3uIvoNl8mctBEojyF13ZyqwQ91clCWXpwsWp+t98K4FOQ== + dependencies: + cli-truncate "^2.1.0" + colorette "^1.4.0" + log-update "^4.0.0" + p-map "^4.0.0" + rxjs "^6.6.7" + through "^2.3.8" + wrap-ansi "^7.0.0" + listr@^0.14.3: version "0.14.3" resolved "https://registry.yarnpkg.com/listr/-/listr-0.14.3.tgz#2fea909604e434be464c50bddba0d496928fa586" @@ -8639,7 +8783,7 @@ lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw== -lodash@^4.15.0, lodash@^4.17.20, lodash@^4.7.0: +lodash@^4.15.0, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -8649,11 +8793,6 @@ lodash@^4.17.15: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@^4.17.19: - version "4.17.20" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" - integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== - log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -8662,11 +8801,12 @@ log-symbols@^1.0.2: chalk "^1.0.0" log-symbols@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.0.0.tgz#69b3cc46d20f448eccdb75ea1fa733d9e821c920" - integrity sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA== + version "4.1.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== dependencies: - chalk "^4.0.0" + chalk "^4.1.0" + is-unicode-supported "^0.1.0" log-update@^2.3.0: version "2.3.0" @@ -8677,6 +8817,16 @@ log-update@^2.3.0: cli-cursor "^2.0.0" wrap-ansi "^3.0.1" +log-update@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" + integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== + dependencies: + ansi-escapes "^4.3.0" + cli-cursor "^3.1.0" + slice-ansi "^4.0.0" + wrap-ansi "^6.2.0" + lolex@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/lolex/-/lolex-5.1.2.tgz#953694d098ce7c07bc5ed6d0e42bc6c0c6d5a367" @@ -8958,7 +9108,19 @@ mime-db@1.40.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== -mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: +mime-db@1.49.0: + version "1.49.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed" + integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.32" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5" + integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A== + dependencies: + mime-db "1.49.0" + +mime-types@~2.1.24: version "2.1.24" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== @@ -9125,11 +9287,16 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@2.1.2, ms@^2.1.1: +ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + nan@^2.12.1, nan@^2.13.2: version "2.14.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" @@ -9827,6 +9994,13 @@ p-map@^2.0.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== +p-map@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" + integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== + dependencies: + aggregate-error "^3.0.0" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -10463,7 +10637,7 @@ prettier@2.2.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== -pretty-bytes@^5.4.1: +pretty-bytes@^5.4.1, pretty-bytes@^5.6.0: version "5.6.0" resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== @@ -10572,12 +10746,12 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= -psl@^1.1.24, psl@^1.1.28: +psl@^1.1.24: version "1.2.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.2.0.tgz#df12b5b1b3a30f51c329eacbdef98f3a6e136dc6" integrity sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -11001,7 +11175,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== @@ -11014,6 +11188,19 @@ read-pkg@^5.2.0: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^2.2.2: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^3.1.1: version "3.4.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" @@ -11457,6 +11644,14 @@ restore-cursor@^2.0.0: onetime "^2.0.0" signal-exit "^3.0.2" +restore-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== + dependencies: + onetime "^5.1.0" + signal-exit "^3.0.2" + ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -11523,10 +11718,10 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.3.3: - version "6.6.6" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.6.tgz#14d8417aa5a07c5e633995b525e1e3c0dec03b70" - integrity sha512-/oTwee4N4iWzAMAL9xdGKjkEHmIwupR3oXbQjCKywF1BeFohswF3vZdogbmEF6pZkOsXTzWkrZszrWpQTByYVg== +rxjs@^6.3.3, rxjs@^6.6.7: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== dependencies: tslib "^1.9.0" @@ -11535,11 +11730,16 @@ safe-buffer@5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" integrity sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg== -safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2: +safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@^5.1.1: version "5.2.0" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== +safe-buffer@^5.0.1, safe-buffer@^5.1.2: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -11821,9 +12021,9 @@ signal-exit@^3.0.0: integrity sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0= signal-exit@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + version "3.0.4" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.4.tgz#366a4684d175b9cab2081e3681fda3747b6c51d7" + integrity sha512-rqYhcAnZ6d/vTPGghdrw7iumdcbXpsk1b8IG/rz+VWV51DM0p7XCtMoJ3qhPLIbp3tvyt3pKRbaaEMZYpHto8Q== sisteransi@^1.0.5: version "1.0.5" @@ -11850,6 +12050,15 @@ slice-ansi@0.0.4: resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" integrity sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU= +slice-ansi@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" + integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== + dependencies: + ansi-styles "^4.0.0" + astral-regex "^2.0.0" + is-fullwidth-code-point "^3.0.0" + slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -12197,7 +12406,7 @@ string-width@^3.0.0, string-width@^3.1.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" -string-width@^4.1.0, string-width@^4.2.0: +string-width@^4.1.0: version "4.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== @@ -12206,6 +12415,15 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" +string-width@^4.2.0: + version "4.2.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" + integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.0" + string.prototype.matchall@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz#48bb510326fb9fdeb6a33ceaa81a6ea04ef7648e" @@ -12393,6 +12611,13 @@ supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" +supports-color@^8.1.1: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + supports-hyperlinks@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47" @@ -12542,6 +12767,11 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" +through@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= + timers-browserify@^2.0.4: version "2.0.10" resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-2.0.10.tgz#1d28e3d2aadf1d5a5996c4e9f95601cd053480ae" @@ -12737,11 +12967,16 @@ tslib@1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" integrity sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ== -tslib@^1.8.0, tslib@^1.8.1, tslib@^1.9.0: +tslib@^1.8.0, tslib@^1.8.1: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== +tslib@^1.9.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" + integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== + tslint-eslint-rules@^5.4.0: version "5.4.0" resolved "https://registry.yarnpkg.com/tslint-eslint-rules/-/tslint-eslint-rules-5.4.0.tgz#e488cc9181bf193fe5cd7bfca213a7695f1737b5" @@ -12838,6 +13073,11 @@ type-fest@^0.11.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== +type-fest@^0.21.3: + version "0.21.3" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== + type-fest@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" @@ -13005,9 +13245,9 @@ upper-case@^1.1.1: integrity sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg= uri-js@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" - integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" @@ -13089,9 +13329,9 @@ utils-merge@1.0.1: integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= uuid@^3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== uuid@^7.0.3: version "7.0.3" @@ -13103,6 +13343,11 @@ uuid@^8.3.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31" integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg== +uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + v8-compile-cache@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" @@ -13470,6 +13715,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"