diff --git a/.github/ISSUE_TEMPLATE/smoke-tests.md b/.github/ISSUE_TEMPLATE/smoke-tests.md deleted file mode 100644 index fb9d454adf..0000000000 --- a/.github/ISSUE_TEMPLATE/smoke-tests.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -name: Release QA -about: Checklist of required tests prior to release -title: '' -labels: '' -assignees: '' - ---- - -# Goal: easy-to-follow test steps for sanity checking a release manually - -**Fleet version** (Head to the "My account" page in the Fleet UI or run `fleetctl version`): - -**Web browser** _(e.g. Chrome 88.0.4324)_: - -# Important reference data - -1. [fleetctl preview setup](https://fleetdm.com/get-started) -2. [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) - -# Smoke Tests -Smoke tests are limited to core functionality and serve as a sanity test. If smoke tests are failing, a release cannot proceed. - -## Prerequisites - -1. `fleetctrl preview` is set up and running the desired test version using `--tag` parameters. -2. Unless you are explicitly testing older browser versions, browser is up to date. -3. Certificate & flagfile are in place to create new host. - -## Instructions - - - - - - - - -
Test nameStep instructionsExpected resultpass/fail
$Name{what a tester should do}{what a tester should see when they do that}pass/fail
Update flow - -1. Prereq: previous version is set up with data. -2. Update fleet to testing version -No data should be lost in updatepass/fail
Login flow - -1. navigate to the login page and attempt to login with both valid and invalid credentials to verify some combination of expected results. -2. Login with SSO - - -1. text fields prompt when blank -2. correct error message is "authentication failed" -3. forget password link prompts for email -4. valid credentials result in a successful login. pass/fail
Query flowCreate, edit, run, and delete queries. - -1. permissions regarding creating/editing/deleting queries are up to date with documentation -2. syntax errors result in error messaging -3. queries can be run manually -pass/fail
Host FlowVerify a new host can be added and removed following modal instructions using your own device. - -1. Host is added via command line -2. Host serial number and date added are accurate -3. Host is not visible after it is deleted -4. Warning and informational modals show when expected and make sense -pass/fail
- -# Notes - -* {any notes} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d8297eaafa..f38da8fefa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,8 @@ # Checklist for submitter -If some of the following don't apply, delete the relevant line. +If some of the following don't apply, please write a short explanation why. -- [ ] Changes file added (for user-visible changes) +- [ ] Changes file added (if needed) - [ ] Documented any API changes -- [ ] Documented any permissions changes -- [ ] Added/updated tests -- [ ] Manual QA for all new/changed functionality +- [ ] Added tests for all functionality +- [ ] Manual QA for all functionality diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 313c7e37d2..ed33f31b81 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,6 +9,7 @@ jobs: - uses: actions/checkout@master - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: + check-modified-files-only: 'yes' use-quiet-mode: 'yes' config-file: .github/workflows/markdown-link-check-config.json base-branch: ${{ github.base_ref }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 49be7f82d4..d1072848fc 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -17,4 +17,4 @@ jobs: # specified without patch version: we always use the latest patch # version. version: v1.42 - args: --timeout 10m + args: --timeout 3m diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml deleted file mode 100644 index 95ca8232b1..0000000000 --- a/.github/workflows/goreleaser-snapshot-fleet.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: Docker publish - -on: - push: - branches: - - main - - patch-* - pull_request: - -jobs: - # This check-secrets job is used to gate execution of the main job. External PRs will not have - # access to the necessary secrets for pushing an image, so we want to skip that job if the secret - # is unavailable. - check-secrets: - environment: Docker Hub - runs-on: ubuntu-latest - outputs: - available: ${{ steps.check-secrets.outputs.available }} - steps: - - name: Check Secrets availability - id: check-secrets - run: | - if [ ! -z "${{ secrets.DOCKERHUB_USERNAME }}" ]; then - echo "::set-output name=available::true" - fi - - # If the secrets are available, build and publish a Docker image. - publish: - needs: - - check-secrets - if: ${{ needs.check-secrets.outputs.available == 'true' }} - runs-on: ubuntu-latest - environment: Docker Hub - steps: - - name: Checkout (PR) - if: ${{ github.event.pull_request.head.sha }} - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Checkout (main) - if: ${{ !github.event.pull_request.head.sha }} - uses: actions/checkout@v2 - - - name: Login to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # v1.10.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.16.5 - - - name: Install JS Dependencies - run: make deps-js - - - name: Install Go Dependencies - run: make deps-go - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@ac067437f516133269923265894e77920c3dce18 # v2.6.1 - with: - distribution: goreleaser-pro - version: latest - args: release --snapshot --rm-dist -f .goreleaser-snapshot.yml - env: - GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - - - name: Tag image with branch name - if: ${{ !github.event.pull_request.head.sha }} - run: docker tag fleetdm/fleet:$(git rev-parse --short HEAD) fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) - - # Explicitly push the docker images as GoReleaser will not do so in snapshot mode - - name: Publish Docker images - run: docker push fleetdm/fleet --all-tags diff --git a/.github/workflows/markdown-link-check-config.json b/.github/workflows/markdown-link-check-config.json index d8da76dcb6..79ac5c0107 100644 --- a/.github/workflows/markdown-link-check-config.json +++ b/.github/workflows/markdown-link-check-config.json @@ -12,9 +12,6 @@ { "pattern": "fleet.corp.example.com" }, - { - "pattern": "fleet.example.com" - }, { "pattern": "/server/datastore/mysql/migrations/" }, diff --git a/.github/workflows/push-osquery-perf-to-ecr.yml b/.github/workflows/push-osquery-perf-to-ecr.yml deleted file mode 100644 index 197ec32864..0000000000 --- a/.github/workflows/push-osquery-perf-to-ecr.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Build docker image and publish to ECR - -on: - workflow_dispatch: - inputs: - enroll_secret: - description: 'Enroll Secret' - required: true - url: - description: 'Fleet server URL' - required: true - host_count: - description: 'Amount of hosts to emulate' - required: true - default: 20 - tag: - description: 'docker image tag' - required: true - default: latest - -jobs: - build-docker: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LOADTEST_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.LOADTEST_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: osquery-perf - IMAGE_TAG: ${{ github.event.inputs.tag }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg ENROLL_SECRET=${{ github.event.inputs.enroll_secret }} --build-arg HOST_COUNT=${{ github.event.inputs.host_count }} --build-arg SERVER_URL=${{ github.event.inputs.url }} -f Dockerfile.osquery-perf . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index d9a1e3d3fe..6488fc6dc1 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -41,7 +41,7 @@ jobs: - name: Run Go Tests run: | - NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 make test-go + MYSQL_TEST=1 make test-go - name: Upload to Codecov uses: codecov/codecov-action@v2 diff --git a/.gitignore b/.gitignore index b6049ee72b..c1aa8fb7ba 100644 --- a/.gitignore +++ b/.gitignore @@ -50,11 +50,5 @@ backup.sql.gz # committing a package-lock.json. Fleet app uses Yarn with yarn.lock. package-lock.json -# infra -.terraform -.terraform.tfstate* -.terraform.lock* -terraform.tfstate* - # generated installers -orbit-osquery* \ No newline at end of file +orbit-osquery* diff --git a/.goreleaser-snapshot.yml b/.goreleaser-snapshot.yml deleted file mode 100644 index fa57b30145..0000000000 --- a/.goreleaser-snapshot.yml +++ /dev/null @@ -1,74 +0,0 @@ -project_name: fleet - -monorepo: - tag_prefix: fleet- - dir: . - -before: - hooks: - - make deps - - make generate - -gomod: - proxy: true - -builds: - - id: fleet - dir: ./cmd/fleet/ - binary: fleet - env: - - CGO_ENABLED=1 - goos: - - linux - goarch: - - amd64 - flags: - - -tags=full,fts5,netgo - - -trimpath - ldflags: - - -extldflags "-static" - - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} - - -X github.com/kolide/kit/version.version={{ .Version }} - - -X github.com/kolide/kit/version.branch={{ .Branch }} - - -X github.com/kolide/kit/version.revision={{ .FullCommit }} - - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} - - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} - - - id: fleetctl - dir: ./cmd/fleetctl/ - binary: fleetctl - env: - - CGO_ENABLED=0 - goos: - - linux - goarch: - - amd64 - flags: - - -trimpath - ldflags: - - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} - - -X github.com/kolide/kit/version.version={{ .Version }} - - -X github.com/kolide/kit/version.branch={{ .Branch }} - - -X github.com/kolide/kit/version.revision={{ .FullCommit }} - - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} - - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} - - -dockers: - - goos: linux - goarch: amd64 - ids: - - fleet - - fleetctl - dockerfile: tools/docker/fleet.Dockerfile - image_templates: - - 'fleetdm/fleet:{{ .ShortCommit }}' - - - goos: linux - goarch: amd64 - ids: - - fleetctl - dockerfile: tools/docker/fleetctl.Dockerfile - image_templates: - - 'fleetdm/fleetctl:{{ .ShortCommit }}' - diff --git a/CHANGELOG.md b/CHANGELOG.md index 31bb2f8748..010e4736e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,33 +1,3 @@ -## Fleet 4.3.1 (Sept 21, 2021) - -* Add `fleetctl get software` command to list all software and the detected vulnerabilities. The Vulnerable software feature is currently in Beta. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://fleetdm.com/docs/using-fleet/vulnerability-processing). - -* Add `fleetctl vulnerability-data-stream` command to sync the vulnerabilities processing data streams by hand. - -* Add `disable_data_sync` vulnerabilities configuration option to avoid downloading the data streams. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#disable-data-sync). - -* Only show observers the queries they have permissions to run on the **Queries** page. In, Fleet 4.0.0, the Admin, Maintainer, and Observer user roles were introduced. Documentation for the permissions associated with each role can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/using-fleet/permissions). - -* Add `connect_retry_attempts` Redis configuration option to retry failed connections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-connect-retry-attempts). - -* Add `cluster_follow_redirections` Redis configuration option to follow cluster redirections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-cluster-follow-redirections). - -* Add `max_jitter_percent` osquery configuration option to prevent all hosts from returning data at roughly the same time. Note that this improves the Fleet server performance, but it will now take longer for new labels to populate. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#osquery-max-jitter-percent). - -* Improve the performance of database migrations. - -* Reduce database load for label membership recording. - -* Fail early if the process does not have permissions to write to the logging file. - -* Completely skip trying to save a host's users and software inventory if it's disabled to reduce database load. - -* Fix a bug in which team maintainers were unable to run live queries against the hosts assigned to their team(s). - -* Fix a bug in which a blank screen would intermittently appear on the **Hosts** page. - -* Fix a bug detecting disk space for hosts. - ## Fleet 4.3.0 (Sept 13, 2021) * Add Policies feature for detecting device compliance with organizational policies. @@ -168,7 +138,7 @@ * Add ability to create a Team schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). *Available for Fleet Basic customers*. -* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). +* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). * Add ability to see which logging destination is configured for Fleet in the Fleet UI. To see this information, head to the **Schedule** page and then select "Schedule a query." Configured logging destination information is also available in the `GET api/v1/fleet/config` API route. @@ -178,9 +148,9 @@ * Add ability to modify scheduled queries in your Schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). -* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/01-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. +* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/1-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. -* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#feature-flags). +* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/2-Configuration.md#feature-flags). * Improve performance of inserting `pack_stats` in the database. The `pack_stats` information is used to display "Frequency" and "Last run" information for a specific host's scheduled queries. You can find this information on the **Host details** page. @@ -359,9 +329,9 @@ There are currently no known issues in this release. However, we recommend only The primary additions in Fleet 4.0.0 are the new Role-based access control (RBAC) and Teams features. -RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/09-Permissions.md) for a breakdown of the new user roles and their respective capabilities. +RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/9-Permissions.md) for a breakdown of the new user roles and their respective capabilities. -Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md). +Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/10-Teams.md). There are several known issues that will be fixed for the stable release of Fleet 4.0.0. Therefore, we recommend only upgrading to Fleet 4.0.0 RC1 for testing purposes. Please file a GitHub issue for any issues discovered when testing Fleet 4.0.0! @@ -429,7 +399,7 @@ Fleet 4.0.0 is a major release and introduces several breaking changes and datab * Improve Fleet performance by batch updating host seen time instead of updating synchronously. This improvement reduces MySQL CPU usage by ~33% with 4,000 simulated hosts and MySQL running in Docker. -* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/02-Deploying/02-Configuration.md#software-inventory). +* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/2-Deploying/2-Configuration.md#software-inventory). * Add Windows support for `fleetctl` agent autoupdates. The `fleetctl updates` command provides the ability to self-manage an agent update server. Available for Fleet Basic customers. @@ -897,7 +867,7 @@ to 2.0.0. ## Kolide Fleet 2.0.0 (currently preparing for release) -The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md). +The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md). ### New Features diff --git a/Dockerfile.osquery-perf b/Dockerfile.osquery-perf deleted file mode 100644 index e78d967ef5..0000000000 --- a/Dockerfile.osquery-perf +++ /dev/null @@ -1,16 +0,0 @@ -FROM golang:1.17.1-alpine - -ARG ENROLL_SECRET -ARG HOST_COUNT -ARG SERVER_URL - -ENV ENROLL_SECRET ${ENROLL_SECRET} -ENV HOST_COUNT ${HOST_COUNT} -ENV SERVER_URL ${SERVER_URL} - -COPY ./cmd/osquery-perf/agent.go ./go.mod ./go.sum ./cmd/osquery-perf/mac10.14.6.tmpl /osquery-perf/ -WORKDIR /osquery-perf/ -RUN go mod download -RUN go build -o osquery-perf - -CMD ./osquery-perf -enroll_secret $ENROLL_SECRET -host_count $HOST_COUNT -server_url $SERVER_URL \ No newline at end of file diff --git a/README.md b/README.md index c3f8ce93e4..6b622672a9 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The Fleet UI is now available at http://localhost:1337. #### Now what? -Check out the [Ask questions about your devices tutorial](./docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md#how-to-ask-questions-about-your-devices) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. +Check out the [Ask questions about your devices tutorial](./docs/1-Using-Fleet/tutorials/Ask-questions-about-your-devices.md) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. ## Team Fleet is [independently backed](https://linkedin.com/company/fleetdm) and actively maintained with the help of many amazing [contributors](https://github.com/fleetdm/fleet/graphs/contributors). diff --git a/changes/add-policy-updated-at b/changes/add-policy-updated-at deleted file mode 100644 index 5a3d257a7e..0000000000 --- a/changes/add-policy-updated-at +++ /dev/null @@ -1 +0,0 @@ -* Add interval to policies to prevent running them every time the host checks in. diff --git a/changes/ensure-only-one-row-disk-space b/changes/ensure-only-one-row-disk-space new file mode 100644 index 0000000000..1e23bec84e --- /dev/null +++ b/changes/ensure-only-one-row-disk-space @@ -0,0 +1 @@ +* Ensure only one row is returned when checking for disk space in hosts. diff --git a/changes/issue-1894-surface-inherited-queries b/changes/issue-1894-surface-inherited-queries deleted file mode 100644 index 7faae4910e..0000000000 --- a/changes/issue-1894-surface-inherited-queries +++ /dev/null @@ -1 +0,0 @@ -* Fleet premium teams schedules surfaces inherited queries from All teams (global) schedule \ No newline at end of file diff --git a/changes/issue-1963-vulnerabilities-no-sync b/changes/issue-1963-vulnerabilities-no-sync new file mode 100644 index 0000000000..c99dd2d4a7 --- /dev/null +++ b/changes/issue-1963-vulnerabilities-no-sync @@ -0,0 +1,2 @@ +* Add fleetctl vulnerability-data-stream command to sync the vulnerabilities processing data streams by hand. +* Add vulnerabilities.disable_data_sync config to fleet serve to avoid downloading the data streams. diff --git a/changes/issue-1964-list-software b/changes/issue-1964-list-software new file mode 100644 index 0000000000..c6607ea255 --- /dev/null +++ b/changes/issue-1964-list-software @@ -0,0 +1 @@ +* Add `fleetctl get software` to list all software and the detected vulnerabilities. diff --git a/changes/issue-1984-label-count-on-edit b/changes/issue-1984-label-count-on-edit deleted file mode 100644 index 44e8a302d8..0000000000 --- a/changes/issue-1984-label-count-on-edit +++ /dev/null @@ -1 +0,0 @@ -* Do not reset label counts when modifying them. diff --git a/changes/issue-2078-enter-submits-create-team b/changes/issue-2078-enter-submits-create-team deleted file mode 100644 index 48aba2bfe5..0000000000 --- a/changes/issue-2078-enter-submits-create-team +++ /dev/null @@ -1 +0,0 @@ -- Fix create team modal to submit with Enter key \ No newline at end of file diff --git a/changes/issue-2160-straighten-permissions-around-policies b/changes/issue-2160-straighten-permissions-around-policies deleted file mode 100644 index 370c57ad9b..0000000000 --- a/changes/issue-2160-straighten-permissions-around-policies +++ /dev/null @@ -1 +0,0 @@ -* Expand and improve how roles interact with policies. diff --git a/changes/issue-2201-global-maintainer-create-query b/changes/issue-2201-global-maintainer-create-query deleted file mode 100644 index 4510cd6c41..0000000000 --- a/changes/issue-2201-global-maintainer-create-query +++ /dev/null @@ -1 +0,0 @@ -- Bug fix: Global maintainer can create and save a new query \ No newline at end of file diff --git a/changes/reduce-sql-queries-per-host b/changes/reduce-sql-queries-per-host deleted file mode 100644 index 833ebadfca..0000000000 --- a/changes/reduce-sql-queries-per-host +++ /dev/null @@ -1 +0,0 @@ -* Reduce the amount of queries to the database when a host checks in. diff --git a/charts/fleet/Chart.yaml b/charts/fleet/Chart.yaml index 66d0884e07..8f238ae4dd 100644 --- a/charts/fleet/Chart.yaml +++ b/charts/fleet/Chart.yaml @@ -4,8 +4,8 @@ name: fleet keywords: - fleet - osquery -version: v4.3.1 +version: v4.3.0 home: https://github.com/fleetdm/fleet sources: - https://github.com/fleetdm/fleet.git -appVersion: v4.3.1 +appVersion: v4.3.0 diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 3e9d3fdee8..7dc1636ab2 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -2,7 +2,7 @@ # All settings related to how Fleet is deployed in Kubernetes hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy -imageTag: v4.3.1 # Version of Fleet to deploy +imageTag: v4.3.0 # Version of Fleet to deploy createIngress: true # Whether or not to automatically create an Ingress ingressAnnotations: {} # Additional annotation to add to the Ingress podAnnotations: {} # Additional annotations to add to the Fleet pod diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index ab90d2bba6..13dbf29b6c 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -200,21 +200,21 @@ the way that the Fleet server works. } } - redisPool, err := redis.NewRedisPool(redis.PoolConfig{ - Server: config.Redis.Address, - Password: config.Redis.Password, - Database: config.Redis.Database, - UseTLS: config.Redis.UseTLS, - ConnTimeout: config.Redis.ConnectTimeout, - KeepAlive: config.Redis.KeepAlive, - ConnectRetryAttempts: config.Redis.ConnectRetryAttempts, - ClusterFollowRedirections: config.Redis.ClusterFollowRedirections, - }) + redisPool, err := redis.NewRedisPool( + config.Redis.Address, + config.Redis.Password, + config.Redis.Database, + config.Redis.UseTLS, + config.Redis.ConnectTimeout, + config.Redis.KeepAlive, + ) if err != nil { initFatal(err, "initialize Redis") } resultStore := pubsub.NewRedisQueryResults(redisPool, config.Redis.DuplicateResults) liveQueryStore := live_query.NewRedisLiveQuery(redisPool) + // TODO: should that only be done when a certain "migrate" flag is set, + // to prevent affecting every startup? if err := liveQueryStore.MigrateKeys(); err != nil { level.Info(logger).Log( "err", err, @@ -509,10 +509,6 @@ func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, if err != nil { level.Error(logger).Log("err", "cleaning scheduled query stats", "details", err) } - err = ds.CleanupOrphanLabelMembership(ctx) - if err != nil { - level.Error(logger).Log("err", "cleaning label_membership", "details", err) - } err = trySendStatistics(ctx, ds, fleet.StatisticsFrequency, "https://fleetdm.com/api/v1/webhooks/receive-usage-analytics") if err != nil { diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 84ec2fa996..64cd4bfc21 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -41,7 +41,8 @@ var userRoleSpecList = []*fleet.User{ } func TestApplyUserRoles(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil @@ -101,10 +102,11 @@ spec: func TestApplyTeamSpecs(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} - _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) + server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) + defer server.Close() teamsByName := map[string]*fleet.Team{ - "team1": { + "team1": &fleet.Team{ ID: 42, Name: "team1", Description: "team1 description", @@ -183,7 +185,8 @@ func writeTmpYml(t *testing.T, contents string) string { } func TestApplyAppConfig(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil diff --git a/cmd/fleetctl/convert_test.go b/cmd/fleetctl/convert_test.go index 3b838aae1e..1d0724494d 100644 --- a/cmd/fleetctl/convert_test.go +++ b/cmd/fleetctl/convert_test.go @@ -11,6 +11,7 @@ import ( ) func TestConvertFileOutput(t *testing.T) { + t.Parallel() // setup the cli and the convert command app := cli.NewApp() app.Commands = []*cli.Command{convertCommand()} @@ -40,6 +41,7 @@ func TestConvertFileOutput(t *testing.T) { } func TestConvertFileStdout(t *testing.T) { + t.Parallel() r, w, _ := os.Pipe() os.Stdout = w diff --git a/cmd/fleetctl/debug_test.go b/cmd/fleetctl/debug_test.go index d71e72196e..f8a30a99b8 100644 --- a/cmd/fleetctl/debug_test.go +++ b/cmd/fleetctl/debug_test.go @@ -44,7 +44,8 @@ oug6edBNpdhp8r2/4t6n3AouK0/zG2naAlmXV0JoFuEvy2bX0BbbbPg+v4WNZIsC func TestDebugConnectionCommand(t *testing.T) { t.Run("without certificate", func(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.VerifyEnrollSecretFunc = func(ctx context.Context, secret string) (*fleet.EnrollSecret, error) { return nil, errors.New("invalid") diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 2e96b18ca6..8212b268fe 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -55,7 +55,8 @@ var userRoleList = []*fleet.User{ } func TestGetUserRoles(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleList, nil @@ -113,7 +114,8 @@ func TestGetTeams(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { license := tt.license - _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) + server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) + defer server.Close() agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { @@ -194,7 +196,8 @@ spec: } func TestGetHosts(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() // this func is called when no host is specified i.e. `fleetctl get hosts --json` ds.ListHostsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) { @@ -340,7 +343,8 @@ func TestGetHosts(t *testing.T) { } func TestGetConfig(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{ @@ -408,7 +412,8 @@ spec: } func TestGetSoftawre(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() foo001 := fleet.Software{ Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "somecpe", diff --git a/cmd/fleetctl/hosts_test.go b/cmd/fleetctl/hosts_test.go index 936f0cae63..7e8f080605 100644 --- a/cmd/fleetctl/hosts_test.go +++ b/cmd/fleetctl/hosts_test.go @@ -10,7 +10,8 @@ import ( ) func TestHostTransferFlagChecks(t *testing.T) { - runServerWithMockedDS(t) + server, _ := runServerWithMockedDS(t) + defer server.Close() runAppCheckErr(t, []string{"hosts", "transfer", "--team", "team1", "--hosts", "host1", "--label", "AAA"}, @@ -23,7 +24,8 @@ func TestHostTransferFlagChecks(t *testing.T) { } func TestHostsTransferByHosts(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -46,7 +48,8 @@ func TestHostsTransferByHosts(t *testing.T) { } func TestHostsTransferByLabel(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -80,7 +83,8 @@ func TestHostsTransferByLabel(t *testing.T) { } func TestHostsTransferByStatus(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -114,7 +118,8 @@ func TestHostsTransferByStatus(t *testing.T) { } func TestHostsTransferByStatusAndSearchQuery(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) diff --git a/cmd/fleetctl/package_test.go b/cmd/fleetctl/package_test.go index aa4a9c468b..72b3db3561 100644 --- a/cmd/fleetctl/package_test.go +++ b/cmd/fleetctl/package_test.go @@ -1,16 +1,12 @@ package main import ( + "github.com/stretchr/testify/require" "os" "testing" - - "github.com/stretchr/testify/require" ) func TestPackage(t *testing.T) { - if os.Getenv("NETWORK_TEST") == "" { - t.Skip("set environment variable NETWORK_TEST=1 to run") - } // --type is required runAppCheckErr(t, []string{"package", "deb"}, "Required flag \"type\" not set") diff --git a/cmd/fleetctl/preview.go b/cmd/fleetctl/preview.go index 7afc3d9bf2..4ca9d3c1dd 100644 --- a/cmd/fleetctl/preview.go +++ b/cmd/fleetctl/preview.go @@ -22,9 +22,8 @@ import ( const ( downloadUrl = "https://github.com/fleetdm/osquery-in-a-box/archive/master.zip" - standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml" + standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml" licenseKeyFlagName = "license-key" - tagFlagName = "tag" ) func previewCommand() *cli.Command { @@ -46,11 +45,6 @@ Use the stop and reset subcommands to manage the server and dependencies once st Name: licenseKeyFlagName, Usage: "License key to enable Fleet Premium (optional)", }, - &cli.StringFlag{ - Name: tagFlagName, - Usage: "Run a specific version of Fleet", - Value: "latest", - }, }, Action: func(c *cli.Context) error { if err := checkDocker(); err != nil { @@ -78,10 +72,6 @@ Use the stop and reset subcommands to manage the server and dependencies once st return errors.Wrap(err, "make logs writable") } - if err := os.Setenv("FLEET_VERSION", c.String(tagFlagName)); err != nil { - return errors.Wrap(err, "failed to set Fleet version") - } - fmt.Println("Pulling Docker dependencies...") out, err := exec.Command("docker-compose", "pull").CombinedOutput() if err != nil { diff --git a/cmd/fleetctl/query_test.go b/cmd/fleetctl/query_test.go index 145d5d0f6e..306ecf1734 100644 --- a/cmd/fleetctl/query_test.go +++ b/cmd/fleetctl/query_test.go @@ -16,7 +16,8 @@ import ( func TestLiveQuery(t *testing.T) { rs := pubsub.NewInmemQueryResults() lq := new(live_query.MockLiveQuery) - _, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) + server, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) + defer server.Close() ds.HostIDsByNameFunc = func(ctx context.Context, filter fleet.TeamFilter, hostnames []string) ([]uint, error) { return []uint{1234}, nil diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index aed67bf5e4..8f4906c363 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -1 +1 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","policy_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index 2642bf768c..caf6b03972 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -34,7 +34,6 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" - policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/testdata/expectedListHostsJson.json b/cmd/fleetctl/testdata/expectedListHostsJson.json index 1945360d46..9d90c3914f 100644 --- a/cmd/fleetctl/testdata/expectedListHostsJson.json +++ b/cmd/fleetctl/testdata/expectedListHostsJson.json @@ -1,2 +1,2 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedListHostsYaml.yml b/cmd/fleetctl/testdata/expectedListHostsYaml.yml index b667063c25..2b75527a0f 100644 --- a/cmd/fleetctl/testdata/expectedListHostsYaml.yml +++ b/cmd/fleetctl/testdata/expectedListHostsYaml.yml @@ -32,7 +32,6 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" - policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false @@ -77,7 +76,6 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" - policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/users_test.go b/cmd/fleetctl/users_test.go index 482a1d4e0f..663c6514bc 100644 --- a/cmd/fleetctl/users_test.go +++ b/cmd/fleetctl/users_test.go @@ -9,7 +9,8 @@ import ( ) func TestUserDelete(t *testing.T) { - _, ds := runServerWithMockedDS(t) + server, ds := runServerWithMockedDS(t) + defer server.Close() ds.UserByEmailFunc = func(ctx context.Context, email string) (*fleet.User, error) { return &fleet.User{ diff --git a/cmd/fleetctl/vulnerability_data_stream_test.go b/cmd/fleetctl/vulnerability_data_stream_test.go index 8fb865447a..d3c7469eec 100644 --- a/cmd/fleetctl/vulnerability_data_stream_test.go +++ b/cmd/fleetctl/vulnerability_data_stream_test.go @@ -12,10 +12,6 @@ import ( ) func TestVulnerabilityDataStream(t *testing.T) { - if os.Getenv("NETWORK_TEST") == "" { - t.Skip("set environment variable NETWORK_TEST=1 to run") - } - runAppCheckErr(t, []string{"vulnerability-data-stream"}, "No directory provided") vulnPath := t.TempDir() diff --git a/cmd/osquery-perf/README.md b/cmd/osquery-perf/README.md deleted file mode 100644 index 9a809d34ae..0000000000 --- a/cmd/osquery-perf/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Osquery Server Performance Tester - -> **TODO: Archive this repo and move its contents inline into https://github.com/fleetdm/fleet** - -This repository provides a tool to generate realistic traffic to an osquery -management server (primarily, [Fleet](https://github.com/fleetdm/fleet)). With -this tool, many thousands of hosts can be simulated from a single host. - -## Requirements - -The only requirement for running this tool is a working installation of -[Go](https://golang.org/doc/install). - -## Usage - -Typically `go run` is used: - -``` -go run agent.go --help -Usage of agent.go: - -config_interval duration - Interval for config requests (default 1m0s) - -enroll_secret string - Enroll secret to authenticate enrollment - -host_count int - Number of hosts to start (default 10) (default 10) - -query_interval duration - Interval for live query requests (default 10s) - -seed int - Seed for random generator (default current time) (default 1586310930917739000) - -server_url string - URL (with protocol and port of osquery server) (default "https://localhost:8080") - -start_period duration - Duration to spread start of hosts over (default 10s) -exit status 2 -``` - -The tool should be invoked with the appropriate enroll secret. A typical -invocation looks like: - -``` -go run agent.go --enroll_secret hgh4hk3434l2jjf -``` - -When starting many hosts, it is a good idea to extend the intervals, and also -the period over which the hosts are started: - -``` -go run agent.go --enroll_secret hgh4hk3434l2jjf --host_count 5000 --start_period 5m --query_interval 60s --config_interval 5m -``` - -This will start 5,000 hosts over a period of 5 minutes. Each host will check in -for live queries at a 1 minute interval, and for configuration at a 5 minute -interval. Starting over a 5 minute period ensures that the configuration -requests are spread evenly over the 5 minute interval. - -It can be useful to start the "same" hosts. This can be achieved with the -`--seed` parameter: - -``` -go run agent.go --enroll_secret hgh4hk3434l2jjf --seed 0 -``` - -By using the same seed, along with other values, we usually get hosts that look -the same to the server. This is not guaranteed, but it is a useful technique. - -### Resource Limits - -On many systems, trying to simulate a large number of hosts will result in hitting system resource limits (such as number of open file descriptors). - -If you see errors such as `dial tcp: lookup localhost: no such host` or `read: connection reset by peer`, try increasing these limits. - -#### macOS - -Run the following command in the shell before running the Fleet server _and_ before running `agent.go` (run it once in each shell): - -``` sh -ulimit -n 64000 -``` - -## Bugs -To report a bug, [click here](https://github.com/fleetdm/fleet). diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go deleted file mode 100644 index a0339bd8fa..0000000000 --- a/cmd/osquery-perf/agent.go +++ /dev/null @@ -1,285 +0,0 @@ -package main - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "flag" - "fmt" - "log" - "math/rand" - "net/http" - "strings" - "text/template" - "time" - - "github.com/google/uuid" -) - -type Agent struct { - ServerAddress string - EnrollSecret string - NodeKey string - UUID string - Client http.Client - ConfigInterval time.Duration - QueryInterval time.Duration - Templates *template.Template - strings map[string]string -} - -func NewAgent(serverAddress, enrollSecret string, templates *template.Template, configInterval, queryInterval time.Duration) *Agent { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - transport.DisableCompression = true - return &Agent{ - ServerAddress: serverAddress, - EnrollSecret: enrollSecret, - Templates: templates, - ConfigInterval: configInterval, - QueryInterval: queryInterval, - UUID: uuid.New().String(), - Client: http.Client{Transport: transport}, - strings: make(map[string]string), - } -} - -type enrollResponse struct { - NodeKey string `json:"node_key"` -} - -type distributedReadResponse struct { - Queries map[string]string `json:"queries"` -} - -func (a *Agent) runLoop() { - a.Enroll() - - a.Config() - resp, err := a.DistributedRead() - if err != nil { - log.Println(err) - } else { - if len(resp.Queries) > 0 { - a.DistributedWrite(resp.Queries) - } - } - - configTicker := time.Tick(a.ConfigInterval) - liveQueryTicker := time.Tick(a.QueryInterval) - for { - select { - case <-configTicker: - a.Config() - case <-liveQueryTicker: - resp, err := a.DistributedRead() - if err != nil { - log.Println(err) - } else { - if len(resp.Queries) > 0 { - a.DistributedWrite(resp.Queries) - } - } - } - } -} - -const stringVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_." - -func (a *Agent) randomString(n int) string { - sb := strings.Builder{} - sb.Grow(n) - for i := 0; i < n; i++ { - sb.WriteByte(stringVals[rand.Int63()%int64(len(stringVals))]) - } - return sb.String() -} - -func (a *Agent) CachedString(key string) string { - if val, ok := a.strings[key]; ok { - return val - } - val := a.randomString(12) - a.strings[key] = val - return val -} - -func (a *Agent) Enroll() { - var body bytes.Buffer - if err := a.Templates.ExecuteTemplate(&body, "enroll", a); err != nil { - log.Println("execute template:", err) - return - } - - req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/enroll", &body) - if err != nil { - log.Println("create request:", err) - return - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - req.Header.Add("User-Agent", "osquery/4.6.0") - - resp, err := a.Client.Do(req) - if err != nil { - log.Println("do request:", err) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - log.Println("status:", resp.Status) - return - } - - var parsedResp enrollResponse - if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { - log.Println("json parse:", err) - return - } - - a.NodeKey = parsedResp.NodeKey -} - -func (a *Agent) Config() { - body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) - - req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/config", body) - if err != nil { - log.Println("create config request:", err) - return - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - req.Header.Add("User-Agent", "osquery/4.6.0") - - resp, err := a.Client.Do(req) - if err != nil { - log.Println("do config request:", err) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - log.Println("config status:", resp.Status) - return - } - - // No need to read the config body -} - -func (a *Agent) DistributedRead() (*distributedReadResponse, error) { - body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) - - req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/read", body) - if err != nil { - return nil, fmt.Errorf("create distributed read request: %s", err) - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - req.Header.Add("User-Agent", "osquery/4.6.0") - - resp, err := a.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("do distributed read request: %s", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("distributed read status: %s", resp.Status) - } - - var parsedResp distributedReadResponse - if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { - return nil, fmt.Errorf("json parse distributed read response: %s", err) - } - - return &parsedResp, nil -} - -type distributedWriteRequest struct { - Queries map[string]json.RawMessage `json:"queries"` - Statuses map[string]string `json:"statuses"` - NodeKey string `json:"node_key"` -} - -var defaultQueryResult = json.RawMessage(`[{"foo": "bar"}]`) - -const statusSuccess = "0" - -func (a *Agent) DistributedWrite(queries map[string]string) { - var body bytes.Buffer - - if _, ok := queries["fleet_detail_query_network_interface"]; ok { - // Respond to label/detail queries - a.Templates.ExecuteTemplate(&body, "distributed_write", a) - } else { - // Return a generic response for any other queries - req := distributedWriteRequest{ - Queries: make(map[string]json.RawMessage), - Statuses: make(map[string]string), - NodeKey: a.NodeKey, - } - - for name := range queries { - req.Queries[name] = defaultQueryResult - req.Statuses[name] = statusSuccess - } - json.NewEncoder(&body).Encode(req) - } - - req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/write", &body) - if err != nil { - log.Println("create distributed write request:", err) - return - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - req.Header.Add("User-Agent", "osquery/4.6.0") - - resp, err := a.Client.Do(req) - if err != nil { - log.Println("do distributed write request:", err) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - log.Println("distributed write status:", resp.Status) - return - } - - // No need to read the distributed write body -} - -func main() { - serverURL := flag.String("server_url", "https://localhost:8080", "URL (with protocol and port of osquery server)") - enrollSecret := flag.String("enroll_secret", "", "Enroll secret to authenticate enrollment") - hostCount := flag.Int("host_count", 10, "Number of hosts to start (default 10)") - randSeed := flag.Int64("seed", time.Now().UnixNano(), "Seed for random generator (default current time)") - startPeriod := flag.Duration("start_period", 10*time.Second, "Duration to spread start of hosts over") - configInterval := flag.Duration("config_interval", 1*time.Minute, "Interval for config requests") - queryInterval := flag.Duration("query_interval", 10*time.Second, "Interval for live query requests") - - flag.Parse() - - rand.Seed(*randSeed) - - tmpl, err := template.ParseGlob("*.tmpl") - if err != nil { - log.Fatal("parse templates: ", err) - } - - // Spread starts over the interval to prevent thunering herd - sleepTime := *startPeriod / time.Duration(*hostCount) - var agents []*Agent - for i := 0; i < *hostCount; i++ { - a := NewAgent(*serverURL, *enrollSecret, tmpl, *configInterval, *queryInterval) - agents = append(agents, a) - go a.runLoop() - time.Sleep(sleepTime) - } - - fmt.Println("Agents running. Kill with C-c.") - <-make(chan struct{}) -} diff --git a/cmd/osquery-perf/mac10.14.6.tmpl b/cmd/osquery-perf/mac10.14.6.tmpl deleted file mode 100644 index 88b6f713a9..0000000000 --- a/cmd/osquery-perf/mac10.14.6.tmpl +++ /dev/null @@ -1,309 +0,0 @@ -{{ define "enroll" -}} -{ - "enroll_secret": "{{ .EnrollSecret }}", - "host_details": { - "os_version": { - "build": "18G3020", - "major": "10", - "minor": "14", - "name": "Mac OS X", - "patch": "6", - "platform": "darwin", - "platform_like": "darwin", - "version": "10.14.6" - }, - "osquery_info": { - "build_distro": "10.12", - "build_platform": "darwin", - "config_hash": "", - "config_valid": "0", - "extensions": "inactive", - "instance_id": "{{ .UUID }}", - "pid": "12947", - "platform_mask": "21", - "start_time": "1580931224", - "uuid": "{{ .UUID }}", - "version": "4.6.0", - "watcher": "12946" - }, - "platform_info": { - "address": "0xff990000", - "date": "12/16/2019 ", - "extra": "MBP114; 196.0.0.0.0; root@xapp160; Mon Dec 16 15:55:18 PST 2019; 196 (B&I); F000_B00; Official Build, Release; Apple LLVM version 5.0 (clang-500.0.68) (based on LLVM 3.3svn)", - "revision": "196 (B&I)", - "size": "8388608", - "vendor": "Apple Inc. ", - "version": "196.0.0.0.0 ", - "volume_size": "1507328" - }, - "system_info": { - "computer_name": "{{ .CachedString "hostname" }}", - "cpu_brand": "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "cpu_logical_cores": "8", - "cpu_physical_cores": "4", - "cpu_subtype": "Intel x86-64h Haswell", - "cpu_type": "x86_64h", - "hardware_model": "MacBookPro11,4", - "hardware_serial": "D02R835DG8WK", - "hardware_vendor": "Apple Inc.", - "hardware_version": "1.0", - "hostname": "{{ .CachedString "hostname" }}", - "local_hostname": "{{ .CachedString "hostname" }}", - "physical_memory": "17179869184", - "uuid": "{{ .UUID }}" - } - }, - "host_identifier": "{{ .CachedString "hostname" }}", - "platform_type": "21" -} -{{- end }} - -{{ define "distributed_write" -}} -{ - "queries":{ - "fleet_detail_query_network_interface":[ - { - "point_to_point":"", - "address":"fe80::8cb:112d:ff51:1e5d%en0", - "mask":"ffff:ffff:ffff:ffff::", - "broadcast":"", - "interface":"en0", - "mac":"f8:2d:88:93:56:5c", - "type":"6", - "mtu":"1500", - "metric":"0", - "ipackets":"278493", - "opackets":"206238", - "ibytes":"275799040", - "obytes":"37720064", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582848084" - -}, - { - "point_to_point":"", - "address":"192.168.1.3", - "mask":"255.255.255.0", - "broadcast":"192.168.1.255", - "interface":"en0", - "mac":"f5:5a:80:92:52:5b", - "type":"6", - "mtu":"1500", - "metric":"0", - "ipackets":"278493", - "opackets":"206238", - "ibytes":"275799040", - "obytes":"37720064", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582848084" - -}, - { - "point_to_point":"127.0.0.1", - "address":"127.0.0.1", - "mask":"255.0.0.0", - "broadcast":"", - "interface":"lo0", - "mac":"00:00:00:00:00:00", - "type":"24", - "mtu":"16384", - "metric":"0", - "ipackets":"132952", - "opackets":"132952", - "ibytes":"67053568", - "obytes":"67053568", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582840871" - -}, - { - "point_to_point":"::1", - "address":"::1", - "mask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "broadcast":"", - "interface":"lo0", - "mac":"00:00:00:00:00:00", - "type":"24", - "mtu":"16384", - "metric":"0", - "ipackets":"132952", - "opackets":"132952", - "ibytes":"67053568", - "obytes":"67053568", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582840871" - -}, - { - "point_to_point":"", - "address":"fe80::1%lo0", - "mask":"ffff:ffff:ffff:ffff::", - "broadcast":"", - "interface":"lo0", - "mac":"00:00:00:00:00:00", - "type":"24", - "mtu":"16384", - "metric":"0", - "ipackets":"132952", - "opackets":"132952", - "ibytes":"67053568", - "obytes":"67053568", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582840871" - -}, - { - "point_to_point":"", - "address":"fe80::3a:84ff:fe6b:bf75%awdl0", - "mask":"ffff:ffff:ffff:ffff::", - "broadcast":"", - "interface":"awdl0", - "mac":"03:3b:94:5b:be:75", - "type":"6", - "mtu":"1484", - "metric":"0", - "ipackets":"0", - "opackets":"16", - "ibytes":"0", - "obytes":"3072", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582842892" - -}, - { - "point_to_point":"", - "address":"fe80::6eaf:9721:3476:b691%utun0", - "mask":"ffff:ffff:ffff:ffff::", - "broadcast":"", - "interface":"utun0", - "mac":"00:00:00:00:00:00", - "type":"1", - "mtu":"2000", - "metric":"0", - "ipackets":"0", - "opackets":"2", - "ibytes":"0", - "obytes":"0", - "ierrors":"0", - "oerrors":"0", - "idrops":"0", - "odrops":"0", - "last_change":"1582840897" - -} - -], - "fleet_detail_query_os_version":[ - { - "name":"Mac OS X", - "version":"10.14.6", - "major":"10", - "minor":"14", - "patch":"6", - "build":"18G3020", - "platform":"darwin", - "platform_like":"darwin", - "codename":"" - -} - -], - "fleet_detail_query_osquery_flags":[ - { - "name":"config_refresh", - "value":"{{ printf "%.0f" .ConfigInterval.Seconds }}" - -}, - { - "name":"distributed_interval", - "value":"{{ printf "%.0f" .QueryInterval.Seconds }}" - -}, - { - "name":"logger_tls_period", - "value":"99999" - -} - -], - "fleet_detail_query_osquery_info":[ - { - "pid":"11287", - "uuid":"{{ .UUID }}", - "instance_id":"{{ .UUID }}", - "version":"4.1.2", - "config_hash":"b01efbf375ac6767f259ae98751154fef727ce35", - "config_valid":"1", - "extensions":"inactive", - "build_platform":"darwin", - "build_distro":"10.12", - "start_time":"1582857555", - "watcher":"11286", - "platform_mask":"21" - -} - -], - "fleet_detail_query_system_info":[ - { - "hostname":"{{ .CachedString "hostname" }}", - "uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB", - "cpu_type":"x86_64h", - "cpu_subtype":"Intel x86-64h Haswell", - "cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", - "cpu_physical_cores":"4", - "cpu_logical_cores":"8", - "cpu_microcode":"", - "physical_memory":"17179869184", - "hardware_vendor":"Apple Inc.", - "hardware_model":"MacBookPro11,4", - "hardware_version":"1.0", - "hardware_serial":"C02R262BM8LN", - "computer_name":"{{ .CachedString "hostname" }}", - "local_hostname":"{{ .CachedString "hostname" }}" - -} - -], - "fleet_detail_query_uptime":[ - { - "days":"0", - "hours":"4", - "minutes":"38", - "seconds":"11", - "total_seconds":"16691" - -} - -] - -}, - "statuses":{ - "fleet_detail_query_network_interface":0, - "fleet_detail_query_os_version":0, - "fleet_detail_query_osquery_flags":0, - "fleet_detail_query_osquery_info":0, - "fleet_detail_query_system_info":0, - "fleet_detail_query_uptime":0 -}, - "node_key":"{{ .NodeKey }}" -} -{{- end }} diff --git a/cypress-free.json b/cypress-free.json index 91f97117f0..d2b6814b8c 100644 --- a/cypress-free.json +++ b/cypress-free.json @@ -1,9 +1,5 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,free}/**/*.spec.ts", - "retries": { - "runMode": 1, - "openMode": 0 - } + "testFiles": "{all,free}/**/*.spec.ts" } diff --git a/cypress-premium.json b/cypress-premium.json index 394960d126..86c1807f11 100644 --- a/cypress-premium.json +++ b/cypress-premium.json @@ -1,9 +1,5 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,premium}/**/*.spec.ts", - "retries": { - "runMode": 1, - "openMode": 0 - } + "testFiles": "{all,premium}/**/*.spec.ts" } diff --git a/cypress.json b/cypress.json index b15968f2da..73960449f2 100644 --- a/cypress.json +++ b/cypress.json @@ -2,9 +2,5 @@ "baseUrl": "https://localhost:8642", "fixturesFolder": false, "testFiles": "**/*.ts", - "defaultCommandTimeout": 8000, - "retries": { - "runMode": 1, - "openMode": 0 - } + "defaultCommandTimeout": 8000 } diff --git a/cypress/README.md b/cypress/README.md index ab80dd2069..c67e59da62 100644 --- a/cypress/README.md +++ b/cypress/README.md @@ -1,6 +1,6 @@ # Cypress Testing -Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) includes git instructions for test preparation and running tests. +Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) includes git instructions for test preparation and running tests. ## Fleet Cypress directories @@ -37,6 +37,6 @@ As much as possible, assert that the code is only selecting 1 item or that the f ## Resources -- [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) +- [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) - [Cypress documentation](https://docs.cypress.io/api/table-of-contents) - [React testing-library query documentation](https://testing-library.com/docs/queries/about) diff --git a/cypress/integration/all/app/hosts.spec.ts b/cypress/integration/all/app/hosts.spec.ts index 6131989072..39ae559ab3 100644 --- a/cypress/integration/all/app/hosts.spec.ts +++ b/cypress/integration/all/app/hosts.spec.ts @@ -10,7 +10,6 @@ describe( cy.setup(); cy.login(); cy.addDockerHost(); - cy.clearDownloads(); }); afterEach(() => { @@ -46,6 +45,9 @@ describe( cy.get("input[disabled]").should("have.value", contents); }); + // ensure load + cy.wait(5000); // eslint-disable-line cypress/no-unnecessary-waiting + // Wait until the host becomes available (usually immediate in local // testing, but may vary by environment). cy.waitUntil( diff --git a/cypress/integration/all/app/manageUsers.spec.ts b/cypress/integration/all/app/manageUsers.spec.ts index 5c6f25dc71..674ee386fa 100644 --- a/cypress/integration/all/app/manageUsers.spec.ts +++ b/cypress/integration/all/app/manageUsers.spec.ts @@ -6,7 +6,15 @@ describe("Manage Users", () => { }); it("Searching for a user", () => { + cy.intercept({ + method: "GET", + url: "/api/v1/fleet/users", + }).as("getUsers"); + cy.visit("/settings/users"); + cy.url().should("match", /\/settings\/users$/i); + + cy.wait("@getUsers"); cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("exist"); @@ -15,6 +23,8 @@ describe("Manage Users", () => { cy.findByPlaceholderText("Search").type("admin"); + cy.wait("@getUsers"); + cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("not.exist"); cy.findByText("observer@example.com").should("not.exist"); diff --git a/cypress/integration/all/app/queryflow.spec.ts b/cypress/integration/all/app/queryflow.spec.ts index bcc8804123..c91dc70aea 100644 --- a/cypress/integration/all/app/queryflow.spec.ts +++ b/cypress/integration/all/app/queryflow.spec.ts @@ -9,43 +9,50 @@ describe( cy.login(); }); + // TODO - Fix tests according to improved query experience - MP it("Create, check, edit, and delete a query successfully and create, edit, and delete a global scheduled query successfully", () => { cy.visit("/queries/manage"); - cy.findByRole("button", { name: /create new query/i }).click(); + cy.findByRole("button", { name: /create new query/i }).should("exist"); + // cy.findByRole("button", { name: /create new query/i }).click(); - // Using class selector because third party element doesn't work with Cypress Testing Selector Library - cy.get(".ace_scroller") - .click({ force: true }) - .type("{selectall}SELECT * FROM windows_crashes;"); + // cy.findByLabelText(/query name/i) + // .click() + // .type("Query all window crashes"); - cy.findByRole("button", { name: /save/i }).click(); + // // Using class selector because third party element doesn't work with Cypress Testing Selector Library + // cy.get(".ace_scroller") + // .click({ force: true }) + // .type("{selectall}{backspace}SELECT * FROM windows_crashes;"); - cy.findByLabelText(/name/i).click().type("Query all window crashes"); + // cy.findByLabelText(/description/i) + // .click() + // .type("See all window crashes"); - cy.findByLabelText(/description/i) - .click() - .type("See all window crashes"); + // cy.findByRole("button", { name: /save/i }).click(); - cy.findByRole("button", { name: /save query/i }).click(); + // cy.findByRole("button", { name: /save as new/i }).click(); - cy.findByText(/query created/i).should("exist"); - cy.findByText(/back to queries/i).should("exist"); - cy.visit("/queries/manage"); + // Just refreshes to create new query, needs success alert to user that they created a query + // cy.visit("/queries/manage"); - cy.findByText(/query all/i).click(); + // cy.findByText(/query all/i).click(); - cy.findByText(/run query/i).should("exist"); + // cy.findByText(/edit & run query/i).should("exist"); - cy.get(".ace_scroller") - .click({ force: true }) - .type("{selectall}SELECT datetime, username FROM windows_crashes;"); + // cy.get(".ace_scroller") + // .click({ force: true }) + // .type( + // "{selectall}{backspace}SELECT datetime, username FROM windows_crashes;" + // ); - cy.findByRole("button", { name: /^Save$/ }).click(); + // cy.findByRole("button", { name: /save/i }).click(); - cy.findByText(/query updated/i).should("be.visible"); + // cy.findByRole("button", { name: /save changes/i }).click(); - // // Start e2e test for schedules + // cy.findByText(/query updated/i).should("be.visible"); + + // // Test Schedules // cy.visit("/schedule/manage"); // cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting @@ -78,67 +85,64 @@ describe( // .contains("button", /schedule/i) // .click(); - // cy.visit("/schedule/manage"); + // cy.visit("/schedule/manage"); - // cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting - // cy.findByText(/query all window crashes/i).should("exist"); + // cy.wait(3000); // eslint-disable-line cypress/no-unnecessary-waiting + // cy.findByText(/query all window crashes/i).should("exist"); - // cy.findByText(/actions/i).click(); - // cy.findByText(/edit/i).click(); + // cy.findByText(/actions/i).click(); + // cy.findByText(/edit/i).click(); - // cy.get( - // ".schedule-editor-modal__form-field--frequency > .dropdown__select" - // ).click(); + // cy.get( + // ".schedule-editor-modal__form-field--frequency > .dropdown__select" + // ).click(); - // cy.findByText(/every 6 hours/i).click(); + // cy.findByText(/every 6 hours/i).click(); - // cy.findByText(/show advanced options/i).click(); + // cy.findByText(/show advanced options/i).click(); - // cy.findByText(/ignore removals/i).click(); - // cy.findByText(/snapshot/i).click(); + // cy.findByText(/ignore removals/i).click(); + // cy.findByText(/snapshot/i).click(); - // cy.get(".schedule-editor-modal__form-field--shard > .input-field") - // .click() - // .type("{selectall}{backspace}10"); + // cy.get(".schedule-editor-modal__form-field--shard > .input-field") + // .click() + // .type("{selectall}{backspace}10"); - // cy.get(".schedule-editor-modal__btn-wrap") - // .contains("button", /schedule/i) - // .click(); + // cy.get(".schedule-editor-modal__btn-wrap") + // .contains("button", /schedule/i) + // .click(); - // cy.visit("/schedule/manage"); + // cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + // cy.findByText(/actions/i).click(); + // cy.findByText(/remove/i).click(); - // cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting - // cy.findByText(/actions/i).click(); - // cy.findByText(/remove/i).click(); + // cy.get(".remove-scheduled-query-modal__btn-wrap") + // .contains("button", /remove/i) + // .click(); - // cy.get(".remove-scheduled-query-modal__btn-wrap") - // .contains("button", /remove/i) - // .click(); + // cy.findByText(/query all window crashes/i).should("not.exist"); + // // End Test Schedules - // cy.findByText(/query all window crashes/i).should("not.exist"); + // cy.visit("/queries/manage"); - // // End e2e test for schedules + // cy.findByText(/query all window crashes/i) + // .parent() + // .parent() + // .within(() => { + // cy.get(".fleet-checkbox__input").check({ force: true }); + // }); - cy.visit("/queries/manage"); + // cy.findByRole("button", { name: /delete/i }).click(); - cy.findByText(/query all window crashes/i) - .parent() - .parent() - .within(() => { - cy.get(".fleet-checkbox__input").check({ force: true }); - }); + // // Can't figure out how attach findByRole onto modal button + // // Can't use findByText because delete button under modal + // cy.get(".remove-query-modal") + // .contains("button", /delete/i) + // .click(); - cy.findByRole("button", { name: /delete/i }).click(); + // cy.findByText(/successfully removed query/i).should("be.visible"); - // Can't figure out how attach findByRole onto modal button - // Can't use findByText because delete button under modal - cy.get(".remove-query-modal") - .contains("button", /delete/i) - .click(); - - cy.findByText(/successfully removed query/i).should("be.visible"); - - cy.findByText(/query all/i).should("not.exist"); + // cy.findByText(/query all/i).should("not.exist"); }); } ); diff --git a/cypress/integration/all/app/resetsessions.spec.ts b/cypress/integration/all/app/resetsessions.spec.ts index 0a9e1e9100..c9bba80710 100644 --- a/cypress/integration/all/app/resetsessions.spec.ts +++ b/cypress/integration/all/app/resetsessions.spec.ts @@ -20,11 +20,14 @@ describe("Reset user sessions flow", () => { // first select the table cell with the user's email address then go back up to the containing row // so we can select reset sessions from actions dropdown - cy.contains("div.Select-placeholder", /actions/i); - // eslint-disable-next-line cypress/no-unnecessary-waiting - cy.wait(500); - cy.contains("div.Select-placeholder", /actions/i).click(); - cy.contains(/reset sessions/i).click(); + cy.get("tbody>tr>td") + .contains(/admin@example.com/i) + .parent() + .parent() + .within(() => { + cy.findByText(/actions/i).click(); + cy.findByText(/reset sessions/i).click(); + }); cy.get(".modal__modal_container").within(() => { cy.findByText(/reset sessions/i).should("exist"); diff --git a/cypress/integration/free/observer.spec.ts b/cypress/integration/free/observer.spec.ts index 253ceb2cf1..ab7e05af38 100644 --- a/cypress/integration/free/observer.spec.ts +++ b/cypress/integration/free/observer.spec.ts @@ -65,11 +65,20 @@ describe("Free tier - Observer user", () => { cy.findByText(/show sql/i).click(); cy.findByRole("button", { name: /run query/i }).should("exist"); + cy.visit("/queries/manage"); + + cy.findByText(/get authorized/i).click(); + cy.findByText(/packs/i).should("not.exist"); + cy.findByLabelText(/query name/i).should("not.exist"); + cy.findByLabelText(/sql/i).should("not.exist"); + cy.findByLabelText(/description/i).should("not.exist"); + cy.findByLabelText(/observer can run/i).should("not.exist"); + cy.findByText(/show sql/i).click(); + cy.findByRole("button", { name: /run query/i }).should("not.exist"); + // On the Profile page, they should… // See Observer in Role section, and no Team section cy.visit("/profile"); - - cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByText(/teams/i).should("not.exist"); cy.findByText("Role") .next() diff --git a/cypress/integration/premium/teamflow.spec.ts b/cypress/integration/premium/teamflow.spec.ts index ba60bb2589..2d26bcfdcb 100644 --- a/cypress/integration/premium/teamflow.spec.ts +++ b/cypress/integration/premium/teamflow.spec.ts @@ -5,13 +5,12 @@ describe("Teams flow", () => { cy.viewport(1200, 660); }); - /* TODO fix and reenable - This test is causing major flake issues due to the dropdown menu */ - it("Create, edit, and delete a team successfully", () => { cy.visit("/settings/teams"); - cy.findByRole("button", { name: /create team/i }).click({ force: true }); + cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting + + cy.findByRole("button", { name: /create team/i }).click(); cy.findByLabelText(/team name/i) .click() @@ -29,92 +28,24 @@ describe("Teams flow", () => { cy.findByText(/agent options/i).click(); - cy.contains(".ace_content", "config:"); - cy.get(".ace_text-input") - .first() - .focus() - .type("{selectall}{backspace}config:\n options:"); + cy.get(".ace_content") + .click() + .type("{selectall}{backspace}apiVersion: v1{enter}kind: options"); cy.findByRole("button", { name: /save options/i }).click(); - cy.contains("span", /successfully saved/i); - cy.visit("/settings/teams/1/options"); - cy.contains(/config:/i).should("be.visible"); - cy.contains(/options:/i).should("be.visible"); + cy.contains(/apiVersion: v1/i).should("be.visible"); + cy.contains(/kind: options/i).should("be.visible"); - // Check team in schedules - cy.visit("/queries/manage"); - - cy.findByRole("button", { name: /create new query/i }).click(); - - // Using class selector because third party element doesn't work with Cypress Testing Selector Library - cy.get(".ace_scroller") - .click({ force: true }) - .type("{selectall}SELECT * FROM windows_crashes;"); - - cy.findByRole("button", { name: /save/i }).click(); - - cy.findByLabelText(/name/i).click().type("Query all window crashes"); - - cy.findByLabelText(/description/i) - .click() - .type("See all window crashes"); - - cy.findByRole("button", { name: /save query/i }).click(); - - cy.visit("/schedule/manage"); - - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting - - cy.findByRole("button", { name: /schedule a query/i }).click(); - - cy.findByText(/select query/i).click(); - - cy.findByText(/query all window crashes/i).click(); - - cy.get( - ".schedule-editor-modal__form-field--frequency > .dropdown__select" - ).click(); - - cy.findByText(/every week/i).click(); - - cy.findByText(/show advanced options/i).click(); - - cy.get( - ".schedule-editor-modal__form-field--logging > .dropdown__select" - ).click(); - - cy.findByText(/ignore removals/i).click(); - - cy.get(".schedule-editor-modal__form-field--shard > .input-field") - .click() - .type("50"); - - cy.get(".schedule-editor-modal__btn-wrap") - .contains("button", /schedule/i) - .click(); - - cy.visit("/schedule/manage"); - - cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting - cy.findByText(/all teams/i).click(); - cy.findByText(/valor/i).click(); - - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting - cy.findByText(/query all window crashes/i).should("not.exist"); - cy.findByText(/inherited query/i).click(); - cy.findByText(/query all window crashes/i).should("exist"); - - // Edit Team cy.visit("/settings/teams"); - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting - cy.findByText(/actions/i).click({ force: true }); - cy.findByText(/edit/i).click({ force: true }); // need force:true for dropdown + cy.contains("Valor").get(".Select-arrow-zone").click(); + + // need force:true for dropdown + cy.findByText(/edit/i).click({ force: true }); - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByLabelText(/team name/i) .click() .type("{selectall}{backspace}Mystic"); @@ -130,7 +61,6 @@ describe("Teams flow", () => { cy.findByText(/delete/i).click({ force: true }); - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.findByRole("button", { name: /delete/i }).click(); cy.findByText(/successfully deleted/i).should("be.visible"); diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index b73c3ccae9..9af750deac 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -285,7 +285,3 @@ Cypress.Commands.add("stopDockerHost", () => { }, }); }); - -Cypress.Commands.add("clearDownloads", () => { - cy.exec(`rm -rf ${Cypress.config("downloadsFolder")}`); -}); diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml deleted file mode 100644 index a85702f19e..0000000000 --- a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: team -spec: - team: - name: Client Platform Engineerin - agent_options: - config: - decorators: - load: - - SELECT uuid AS host_uuid FROM system_info; - - SELECT hostname AS hostname FROM system_info; - options: - disable_distributed: false - distributed_interval: 10 - distributed_plugin: tls - distributed_tls_max_attempts: 3 - logger_plugin: tls - logger_tls_endpoint: /api/v1/osquery/log - logger_tls_period: 10 - pack_delimiter: / - overrides: {} - secrets: - - secret: RzTlxPvugG4o4O5IKS/HqEDJUmI1hwBoffff diff --git a/docs/03-Contributing/06-Seeding-Data.md b/docs/03-Contributing/06-Seeding-Data.md deleted file mode 100644 index 4c18effe4a..0000000000 --- a/docs/03-Contributing/06-Seeding-Data.md +++ /dev/null @@ -1,44 +0,0 @@ -# Seeding Data - -When developing Fleet, it may be useful to create seed data that includes users and teams. - -Check out this Loom demo video that walks through creating teams seed data: -https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 - -For a text-based walkthrough, check out the following steps: - -First, create a `env` file with the following contents: - -``` -export SERVER_URL=https://localhost:8080 # your fleet server url and port -export CURL_FLAGS='-k -s' # set insecure flag -export TOKEN=eyJhbGciOi... # your login token -``` - -Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. - -``` -export FLEET_ENV_PATH=/Users/victor/fleet_env -``` - -Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. - -The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. - -``` -./tools/api/fleet/teams/create_free -``` - -The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. - -``` -./tools/api/fleet/teams/create_premium -``` - -The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. - -``` -./tools/api/fleet/teams/create_figma -``` - -Each user generated by the script has their password set to `user123#`. \ No newline at end of file diff --git a/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md b/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md similarity index 89% rename from docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md rename to docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md index c10538fcc9..aa7a894d83 100644 --- a/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md +++ b/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md @@ -28,14 +28,19 @@ So, let's start by asking the following questions about Fleet's 7 simulated Linu 2. Do these devices have a high severity vulnerable version of OpenSSL installed? -These questions can easily be answered, by running this simple query: "Get OpenSSL versions." +These questions can easily be answered, by running the following query: "Detect Linux hosts with high severity vulnerable versions of OpenSSL." -On the **Queries** page, enter the query name, "Get OpenSSL versions," in the search box, and select it to enter the **query console**. Then from the **query console**, hit "Run query", and from the "Select targets" page, select "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. +On the **Queries** page, enter the query name, "Detect Linux hosts with high severity vulnerable versions of OpenSSL," in the search box, select the query from the results table, and navigate to the **Edit or run query** page. -Fleet select targets +Fleet query search +On the **Edit or run query** page, open the "Select targets" dropdown, and press the purple "+" icon to the right of "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. + + +Fleet select targets + The query may take several seconds to complete, because Fleet has to wait for the osquery agents to respond with results. > Fleet's query response time is inherently variable because of osquery's heartbeat response time. This helps prevent performance issues on hosts. @@ -43,7 +48,7 @@ The query may take several seconds to complete, because Fleet has to wait for th When the query has finished, you should see 4 columns and several rows in the "Results" table: -Fleet query results +Fleet query results - The "hostname" column answers: which device responded for a given row of results? diff --git a/docs/01-Using-Fleet/01-Fleet-UI.md b/docs/1-Using-Fleet/1-Fleet-UI.md similarity index 97% rename from docs/01-Using-Fleet/01-Fleet-UI.md rename to docs/1-Using-Fleet/1-Fleet-UI.md index b90b2b2e89..e6e0c68954 100644 --- a/docs/01-Using-Fleet/01-Fleet-UI.md +++ b/docs/1-Using-Fleet/1-Fleet-UI.md @@ -27,7 +27,7 @@ To add queries to a pack, use the right-hand sidebar. You can take an existing s ![Schedule Query Sidebar](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/schedule-query-sidebar.png) -Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../01-Using-Fleet/05-Osquery-logs.md). +Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../1-Using-Fleet/5-Osquery-logs.md). ## Configuring agent options diff --git a/docs/01-Using-Fleet/10-Teams.md b/docs/1-Using-Fleet/10-Teams.md similarity index 94% rename from docs/01-Using-Fleet/10-Teams.md rename to docs/1-Using-Fleet/10-Teams.md index 4904e900db..86591f1055 100644 --- a/docs/01-Using-Fleet/10-Teams.md +++ b/docs/1-Using-Fleet/10-Teams.md @@ -94,7 +94,7 @@ To add users to a team: 4. Select one or more users by searching for their full name and confirm the action. -Users will be given the [Observer role](./09-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. +Users will be given the [Observer role](./9-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. ## Edit a member's role diff --git a/docs/01-Using-Fleet/11-Usage-statistics.md b/docs/1-Using-Fleet/11-Usage-statistics.md similarity index 100% rename from docs/01-Using-Fleet/11-Usage-statistics.md rename to docs/1-Using-Fleet/11-Usage-statistics.md diff --git a/docs/01-Using-Fleet/12-Supported-browsers.md b/docs/1-Using-Fleet/12-Supported-browsers.md similarity index 68% rename from docs/01-Using-Fleet/12-Supported-browsers.md rename to docs/1-Using-Fleet/12-Supported-browsers.md index 5fae7fb5ea..644545005c 100644 --- a/docs/01-Using-Fleet/12-Supported-browsers.md +++ b/docs/1-Using-Fleet/12-Supported-browsers.md @@ -19,7 +19,4 @@ We test each browser on Windows whenever possible, because our engineering team - Mobile Safari on iOS 10 - Mobile Chrome on Android 6 -### Note -> - Mobile web is not yet supported in the Fleet product. -> - The Fleet user interface [may not be fully supported](https://github.com/fleetdm/fleet/issues/969) in Google Chrome when the browser is running on Chrome OS - +**Note:** Mobile web is not yet supported in the Fleet Product. diff --git a/docs/01-Using-Fleet/13-Vulnerability-Processing.md b/docs/1-Using-Fleet/13-Vulnerability-Processing.md similarity index 96% rename from docs/01-Using-Fleet/13-Vulnerability-Processing.md rename to docs/1-Using-Fleet/13-Vulnerability-Processing.md index acb02e5a87..70092798c5 100644 --- a/docs/01-Using-Fleet/13-Vulnerability-Processing.md +++ b/docs/1-Using-Fleet/13-Vulnerability-Processing.md @@ -69,6 +69,6 @@ FLEET_VULNERABILITIES_DATABASES_PATH=/some/path The path specified needs to exist and Fleet needs to be able to read and write to and from it. This is the only mandatory configuration needed for vulnerability processing to work. Additional options, like vulnerability check frequency, can be -found in the [configuration documentation](../02-Deploying/02-Configuration.md#vulnerabilities). +found in the [configuration documentation](../2-Deploying/2-Configuration.md#vulnerabilities). You'll need to restart the Fleet instances after changing these settings. \ No newline at end of file diff --git a/docs/01-Using-Fleet/02-fleetctl-CLI.md b/docs/1-Using-Fleet/2-fleetctl-CLI.md similarity index 96% rename from docs/01-Using-Fleet/02-fleetctl-CLI.md rename to docs/1-Using-Fleet/2-fleetctl-CLI.md index 9e7ef885ae..396280f346 100644 --- a/docs/01-Using-Fleet/02-fleetctl-CLI.md +++ b/docs/1-Using-Fleet/2-fleetctl-CLI.md @@ -24,7 +24,7 @@ This guide illustrates: ### Running Fleet -For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../03-Contributing/01-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): +For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../3-Contributing/1-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): ``` docker-compose up -d @@ -186,7 +186,7 @@ spec: Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allows the Fleet server to request files (and sets of files) from osquery agents, returning the full contents to Fleet. -File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../02-Deploying/02-Configuration.md#s3-file-carving-backend). +File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../2-Deploying/2-Configuration.md#s3-file-carving-backend). ### Configuration diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/1-Using-Fleet/3-REST-API.md similarity index 95% rename from docs/01-Using-Fleet/03-REST-API.md rename to docs/1-Using-Fleet/3-REST-API.md index e2adf152c7..b055201859 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/1-Using-Fleet/3-REST-API.md @@ -10,7 +10,6 @@ - [Schedule](#schedule) - [Packs](#packs) - [Policies](#policies) -- [Team Policies](#team-policies) - [Activities](#activities) - [Targets](#targets) - [Fleet configuration](#fleet-configuration) @@ -31,7 +30,7 @@ Fleet is powered by a Go API server which serves three types of endpoints: ### fleetctl -Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./02-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. +Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./2-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. ### Current API @@ -101,7 +100,7 @@ Authenticates the user with the specified credentials. Use the token returned fr ##### Request body -```json +``` { "email": "janedoe@example.com", "password": "VArCjNW7CfsxGp67" @@ -167,7 +166,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi ##### Request body -```json +``` { "email": "janedoe@example.com" } @@ -181,7 +180,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi `Status: 500` -```json +``` { "message": "Unknown Error", "errors": [ @@ -214,7 +213,7 @@ Changes the password for the authenticated user. ##### Request body -```json +``` { "old_password": "VArCjNW7CfsxGp67", "new_password": "zGq7mCLA6z4PzArC" @@ -229,7 +228,7 @@ Changes the password for the authenticated user. `Status: 422 Unprocessable entity` -```json +``` { "message": "Validation Failed", "errors": [ @@ -261,10 +260,10 @@ Resets a user's password. Which user is determined by the password reset token u ##### Request body -```json +``` { - "new_password": "abc123", - "new_password_confirmation": "abc123", + "new_password": "abc123" + "new_password_confirmation": "abc123" "password_reset_token": "UU5EK0JhcVpsRkY3NTdsaVliMEZDbHJ6TWdhK3oxQ1Q=" } ``` @@ -273,6 +272,9 @@ Resets a user's password. Which user is determined by the password reset token u `Status: 200` +``` +{} +``` --- @@ -290,7 +292,7 @@ Retrieves the user data for the authenticated user. `Status: 200` -```json +``` { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -322,7 +324,7 @@ Resets the password of the authenticated user. Requires that `force_password_res ##### Request body -```json +``` { "new_password": "sdPz8CV5YhzH47nK" } @@ -332,7 +334,7 @@ Resets the password of the authenticated user. Requires that `force_password_res `Status: 200` -```json +``` { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -366,7 +368,7 @@ Gets the current SSO configuration. `Status: 200` -```json +``` { "settings": { "idp_name": "IDP Vendor 1", @@ -394,7 +396,7 @@ Gets the current SSO configuration. ##### Request body -```json +``` { "relay_url": "/hosts/manage" } @@ -408,7 +410,7 @@ Gets the current SSO configuration. `Status: 500` -```json +``` { "message": "Unknown Error", "errors": [ @@ -438,7 +440,7 @@ This is the callback endpoint that the identity provider will use to send securi ##### Request body -```json +``` { "SAMLResponse": "" } @@ -448,6 +450,9 @@ This is the callback endpoint that the identity provider will use to send securi `Status: 200` +``` +{} +``` --- @@ -476,7 +481,7 @@ This is the callback endpoint that the identity provider will use to send securi | order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. | | status | string | query | Indicates the status of the hosts to return. Can either be `new`, `online`, `offline`, or `mia`. | | query | string | query | Search query keywords. Searchable fields include `hostname`, `machine_serial`, `uuid`, and `ipv4`. | -| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../01-Using-Fleet/02-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | +| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | | team_id | integer | query | _Available in Fleet Premium_ Filters the users to only include users in the specified team. | | policy_id | integer | query | The ID of the policy to filter hosts by. `policy_response` must also be specified with `policy_id`. | | policy_response | string | query | Valid options are `passing` or `failing`. `policy_id` must also be specified with `policy_response`. | @@ -489,7 +494,7 @@ If `additional_info_filters` is not specified, no `additional` information will ##### Request query parameters -```json +``` { "page": 0, "per_page": 100, @@ -501,7 +506,7 @@ If `additional_info_filters` is not specified, no `additional` information will `Status: 200` -```json +``` { "hosts": [ { @@ -565,7 +570,7 @@ None. `Status: 200` -```json +``` { "online_count": 2267, "offline_count": 141, @@ -578,7 +583,7 @@ None. Returns the information of the specified host. -The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../02-Deploying/02-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. +The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../2-Deploying/2-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. `GET /api/v1/fleet/hosts/{id}` @@ -596,7 +601,7 @@ The endpoint returns the host's installed `software` if the software inventory f `Status: 200` -```json +``` { "host": { "created_at": "2021-08-19T02:02:22Z", @@ -733,7 +738,7 @@ Returns the information of the host specified using the `uuid`, `osquery_host_id `Status: 200` -```json +``` { "host": { "created_at": "2020-11-05T05:09:44Z", @@ -799,6 +804,9 @@ Deletes the specified host from Fleet. Note that a deleted host will fail authen `Status: 200` +``` +{} +``` ### Refetch host @@ -820,6 +828,9 @@ Flags the host details to be refetched the next time the host checks in for live `Status: 200` +``` +{} +``` ### Transfer hosts to a team @@ -840,7 +851,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "team_id": 1, "hosts": [3, 2, 4, 6, 1, 5, 7] @@ -851,6 +862,9 @@ _Available in Fleet Premium_ `Status: 200` +``` +{} +``` ### Transfer hosts to a team by filter @@ -871,7 +885,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "team_id": 1, "filters": { @@ -884,6 +898,9 @@ _Available in Fleet Premium_ `Status: 200` +``` +{} +``` --- @@ -921,7 +938,7 @@ Creates a dynamic label. ##### Request body -```json +``` { "name": "Ubuntu hosts", "description": "Filters ubuntu hosts", @@ -934,7 +951,7 @@ Creates a dynamic label. `Status: 200` -```json +``` { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -972,7 +989,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T ##### Request body -```json +``` { "name": "macOS label", "description": "Now this label only includes macOS machines", @@ -984,7 +1001,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T `Status: 200` -```json +``` { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -1023,7 +1040,7 @@ Returns the specified label. `Status: 200` -```json +``` { "label": { "created_at": "2021-02-09T22:09:43Z", @@ -1063,7 +1080,7 @@ Returns a list of all the labels in Fleet. `Status: 200` -```json +``` { "labels": [ { @@ -1167,7 +1184,7 @@ Returns a list of the hosts that belong to the specified label. `Status: 200` -```json +``` { "hosts": [ { @@ -1234,6 +1251,9 @@ Deletes the label specified by name. `Status: 200` +``` +{} +``` ### Delete label by ID @@ -1255,6 +1275,9 @@ Deletes the label specified by ID. `Status: 200` +``` +{} +``` ### Apply labels specs @@ -1278,7 +1301,7 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als ##### Request body -```json +``` { "specs": [ { @@ -1303,6 +1326,9 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als `Status: 200` +``` +{} +``` ### Get labels specs @@ -1320,7 +1346,7 @@ None. `Status: 200` -```json +``` { "specs": [ { @@ -1395,7 +1421,7 @@ None. `Status: 200` -```json +``` { "specs": { "id": 12, @@ -1454,7 +1480,7 @@ None. `Status: 200` -```json +``` { "users": [ { @@ -1486,7 +1512,7 @@ None. `Status: 401 Authentication Failed` -```json +``` { "message": "Authentication Failed", "errors": [ @@ -1522,7 +1548,7 @@ Creates a user account after an invited user provides registration information a ##### Request query parameters -```json +``` { "email": "janedoe@example.com", "invite_token": "SjdReDNuZW5jd3dCbTJtQTQ5WjJTc2txWWlEcGpiM3c=", @@ -1546,7 +1572,7 @@ Creates a user account after an invited user provides registration information a `Status: 200` -```json +``` { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1568,7 +1594,7 @@ Creates a user account after an invited user provides registration information a `Status: 401 Authentication Failed` -```json +``` { "message": "Authentication Failed", "errors": [ @@ -1584,7 +1610,7 @@ Creates a user account after an invited user provides registration information a `Status: 404 Resource Not Found` -```json +``` { "message": "Resource Not Found", "errors": [ @@ -1602,7 +1628,7 @@ Creates a user account after an invited user provides registration information a The same error will be returned whenever one of the required parameters fails the validation. -```json +``` { "message": "Validation Failed", "errors": [ @@ -1638,7 +1664,7 @@ Creates a user account without requiring an invitation, the user is enabled imme ##### Request body -```json +``` { "name": "Jane Doe", "email": "janedoe@example.com", @@ -1660,7 +1686,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 200` -```json +``` { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1677,11 +1703,11 @@ Creates a user account without requiring an invitation, the user is enabled imme "teams": [ { "id": 2, - "role": "observer" + "role: "observer" }, { "id": 3, - "role": "maintainer" + "role: "maintainer" }, ] } @@ -1692,7 +1718,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 404 Resource Not Found` -```json +``` { "message": "Resource Not Found", "errors": [ @@ -1722,7 +1748,7 @@ Returns all information about a specific user. ##### Request query parameters -```json +``` { "id": 1 } @@ -1732,7 +1758,7 @@ Returns all information about a specific user. `Status: 200` -```json +``` { "user": { "created_at": "2020-12-10T05:20:25Z", @@ -1754,7 +1780,7 @@ Returns all information about a specific user. `Status: 404 Resource Not Found` -```json +``` { "message": "Resource Not Found", "errors": [ @@ -1789,7 +1815,7 @@ Returns all information about a specific user. ##### Request body -```json +``` { "name": "Jane Doe", "global_role": "admin" @@ -1800,7 +1826,7 @@ Returns all information about a specific user. `Status: 200` -```json +``` { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1824,15 +1850,15 @@ Returns all information about a specific user. ##### Request body -```json +``` { "teams": [ { "id": 1, - "role": "observer" + "role: "observer" }, { - "id": 2, + "id": 2 "role": "maintainer" } ] @@ -1843,7 +1869,7 @@ Returns all information about a specific user. `Status: 200` -```json +``` { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1855,15 +1881,15 @@ Returns all information about a specific user. "force_password_reset": false, "gravatar_url": "", "sso_enabled": false, - "global_role": "admin", + "global_role": "admin" "teams": [ { "id": 2, - "role": "observer" + "role: "observer" }, { "id": 3, - "role": "maintainer" + "role: "maintainer" }, ] } @@ -1890,6 +1916,9 @@ Delete the specified user from Fleet. `Status: 200` +``` +{} +``` ### Require password reset @@ -1910,7 +1939,7 @@ The selected user is logged out of Fleet and required to reset their password du ##### Request body -```json +``` { "require": true } @@ -1920,7 +1949,7 @@ The selected user is logged out of Fleet and required to reset their password du `Status: 200` -```json +``` { "user": { "created_at": "2021-02-23T22:23:34Z", @@ -1955,7 +1984,7 @@ None. `Status: 200` -```json +``` { "sessions": [ { @@ -1997,6 +2026,9 @@ Deletes the selected user's sessions in Fleet. Also deletes the user's API token `Status: 200` +``` +{} +``` --- @@ -2025,7 +2057,7 @@ Returns the session information for the session specified by ID. `Status: 200` -```json +``` { "session_id": 1, "user_id": 1, @@ -2053,6 +2085,9 @@ Deletes the session specified by ID. When the user associated with the session n `Status: 200` +``` +{} +``` --- @@ -2095,7 +2130,7 @@ Returns the query specified by ID. `Status: 200` -```json +``` { "query": { "created_at": "2021-01-19T17:08:24Z", @@ -2144,7 +2179,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -```json +``` { "queries": [ { @@ -2215,7 +2250,7 @@ Returns a list of all queries in the Fleet instance. ##### Request body -```json +``` { "description": "This is a new query.", "name": "new_query", @@ -2227,7 +2262,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -```json +``` { "query": { "created_at": "0001-01-01T00:00:00Z", @@ -2267,7 +2302,7 @@ Returns the query specified by ID. ##### Request body -```json +``` { "name": "new_title_for_my_query" } @@ -2277,7 +2312,7 @@ Returns the query specified by ID. `Status: 200` -```json +``` { "query": { "created_at": "2021-01-22T17:23:27Z", @@ -2315,6 +2350,9 @@ Deletes the query specified by name. `Status: 200` +``` +{} +``` ### Delete query by ID @@ -2336,6 +2374,9 @@ Deletes the query specified by ID. `Status: 200` +``` +{} +``` ### Delete queries @@ -2355,7 +2396,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d ##### Request body -```json +``` { "ids": [ 2, 24, 25 @@ -2367,7 +2408,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d `Status: 200` -```json +``` { "deleted": 3 } @@ -2391,7 +2432,7 @@ None. `Status: 200` -```json +``` { "specs": [ { @@ -2428,7 +2469,7 @@ Returns the name, description, and SQL of the query specified by name. `Status: 200` -```json +``` { "specs": { "name": "query1", @@ -2456,7 +2497,7 @@ Creates and/or modifies the queries included in the specs list. To modify an exi ##### Request body -```json +``` { "specs": [ { @@ -2477,6 +2518,9 @@ Creates and/or modifies the queries included in the specs list. To modify an exi `Status: 200` +``` +{} +``` ### Live query health check @@ -2496,6 +2540,9 @@ None. `Status: 200` +``` +{} +``` ### live query result store health check @@ -2515,6 +2562,9 @@ None. `Status: 200` +``` +{} +``` ### Run live query @@ -2540,7 +2590,7 @@ One of `query` and `query_id` must be specified. ##### Request body -```json +``` { "query": "select instance_id from system_info", "selected": { @@ -2553,7 +2603,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -```json +``` { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2579,7 +2629,7 @@ One of `query` and `query_id` must be specified. ##### Request body -```json +``` { "query": "select instance_id from system_info;", "selected": { @@ -2592,7 +2642,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -```json +``` { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2636,7 +2686,7 @@ One of `query` and `query_id` must be specified. ##### Request body -```json +``` { "query_id": 1, "selected": { @@ -2651,7 +2701,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -```json +``` { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2677,7 +2727,7 @@ One of `query` and `query_id` must be specified. ##### Request body -```json +``` { "query": "select instance_id from system_info", "selected": { @@ -2692,7 +2742,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -```json +``` { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2756,7 +2806,7 @@ socket.onmessage = ({ data }) => { ###### Response data -```json +``` o ``` @@ -2764,7 +2814,7 @@ o ###### Request data -```json +``` [ { "type": "auth", @@ -2773,7 +2823,7 @@ o ] ``` -```json +``` [ { "type": "select_campaign", @@ -2786,7 +2836,7 @@ o ###### Response data -```json +``` // Sends the total number of hosts targeted and segments them by status [ @@ -2803,7 +2853,7 @@ o ] ``` -```json +``` // Sends the expected results, actual results so far, and the status of the live query [ @@ -2819,7 +2869,7 @@ o ] ``` -```json +``` // Sends the result for a given host [ @@ -2839,7 +2889,7 @@ o ] ``` -```json +``` // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -2898,7 +2948,7 @@ socket.onmessage = ({ data }) => { ###### Response data -```json +``` o ``` @@ -2906,7 +2956,7 @@ o ###### Request data -```json +``` [ { "type": "auth", @@ -2915,7 +2965,7 @@ o ] ``` -```json +``` [ { "type": "select_campaign", @@ -2924,11 +2974,11 @@ o ] ``` -##### socket.onmessage() +##### `socket.onmessage()` ###### Response data -```json +``` // Sends the total number of hosts targeted and segments them by status [ @@ -2945,7 +2995,7 @@ o ] ``` -```json +``` // Sends the expected results, actual results so far, and the status of the live query [ @@ -2961,7 +3011,7 @@ o ] ``` -```json +``` // Sends the result for a given host [ @@ -2981,7 +3031,7 @@ o ] ``` -```json +``` // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -3028,7 +3078,7 @@ None. `Status: 200` -```json +``` { "global_schedule": [ { @@ -3091,7 +3141,7 @@ None. ##### Request body -```json +``` { "interval": 86400, "query_id": 2, @@ -3103,7 +3153,7 @@ None. `Status: 200` -```json +``` { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3149,7 +3199,7 @@ None. ##### Request body -```json +``` { "interval": 604800, } @@ -3159,7 +3209,7 @@ None. `Status: 200` -```json +``` { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3196,6 +3246,9 @@ None. `Status: 200` +``` +{} +``` --- @@ -3212,7 +3265,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Get team schedule -`GET /api/v1/fleet/teams/{id}/schedule` +`GET /api/v1/fleet/team/{id}/schedule` #### Parameters @@ -3226,13 +3279,13 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`GET /api/v1/fleet/teams/2/schedule` +`GET /api/v1/fleet/team/2/schedule` ##### Default response `Status: 200` -```json +``` { "scheduled": [ { @@ -3275,7 +3328,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Add query to team schedule -`POST /api/v1/fleet/teams/{id}/schedule` +`POST /api/v1/fleet/team/{id}/schedule` #### Parameters @@ -3292,11 +3345,11 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`POST /api/v1/fleet/teams/2/schedule` +`POST /api/v1/fleet/team/2/schedule` ##### Request body -```json +``` { "interval": 86400, "query_id": 2, @@ -3308,7 +3361,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3330,7 +3383,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Edit query in team schedule -`PATCH /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` +`PATCH /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3347,11 +3400,11 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`PATCH /api/v1/fleet/teams/2/schedule/5` +`PATCH /api/v1/fleet/team/2/schedule/5` ##### Request body -```json +``` { "interval": 604800, } @@ -3361,7 +3414,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3384,7 +3437,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Remove query from team schedule -`DELETE /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` +`DELETE /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3395,12 +3448,15 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`DELETE /api/v1/fleet/teams/2/schedule/5` +`DELETE /api/v1/fleet/team/2/schedule/5` ##### Default response `Status: 200` +``` +{} +``` --- @@ -3441,7 +3497,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -```json +``` { "description": "Collects osquery data.", "host_ids": [], @@ -3454,7 +3510,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "pack": { "created_at": "0001-01-01T00:00:00Z", @@ -3494,7 +3550,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -```json +``` { "description": "MacOS hosts are targeted", "host_ids": [], @@ -3506,7 +3562,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3543,7 +3599,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3581,7 +3637,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "packs": [ { @@ -3634,6 +3690,9 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` +``` +{} +``` ### Delete pack by ID @@ -3653,6 +3712,9 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` +``` +{} +``` ### Get scheduled queries in a pack @@ -3672,7 +3734,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": [ { @@ -3753,7 +3815,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -```json +``` { "interval": 120, "pack_id": 15, @@ -3770,7 +3832,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3809,7 +3871,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3853,7 +3915,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -```json +``` { "platform": "", } @@ -3863,7 +3925,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json +``` { "scheduled": { "created_at": "2021-01-28T19:40:04Z", @@ -3902,6 +3964,9 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` +``` +{} +``` ### Get packs specs @@ -3917,7 +3982,7 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -```json +``` { "specs": [ { @@ -4037,7 +4102,7 @@ Returns the specs for all packs in the Fleet instance. ##### Request body -```json +``` { "specs": [ { @@ -4128,6 +4193,9 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` +``` +{} +``` ### Get pack spec by name @@ -4149,7 +4217,7 @@ Returns the spec for the specified pack by pack name. `Status: 200` -```json +``` { "specs": { "id": 15, @@ -4248,7 +4316,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -```json +``` { "policies": [ { @@ -4287,7 +4355,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -```json +``` { "policy": { "id": 1, @@ -4315,7 +4383,7 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body -```json +``` { "query_id": 12 } @@ -4325,7 +4393,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -```json +``` { "policy": { "id": 2, @@ -4353,161 +4421,6 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body -```json -{ - "ids": [ 1 ] -} -``` - -##### Default response - -`Status: 200` - -```json -{ - "deleted": 1 -} -``` - ---- - -## Team Policies - -- [List team policies](#list-team-policies) -- [Get team policy by ID](#get-team-policy-by-id) -- [Add team policy](#add-team-policy) -- [Remove team policies](#remove-team-policies) - -_Available in Fleet Premium_ - -Team policies work the same as policies, but at the team level. - -### List team policies - -`GET /api/v1/fleet/teams/{team_id}/policies` - -#### Parameters - -| Name | Type | In | Description | -| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | -| team_id | integer | url | Defines what team id to operate on | - -#### Example - -`GET /api/v1/fleet/teams/1/policies` - -##### Default response - -`Status: 200` - -``` -{ - "policies": [ - { - "id": 1, - "query_id": 2, - "query_name": "Gatekeeper enabled", - "passing_host_count": 2000, - "failing_host_count": 300, - }, - { - "id": 2, - "query_id": 3, - "query_name": "Primary disk encrypted", - "passing_host_count": 2300, - "failing_host_count": 0, - } - ] -} -``` - -### Get team policy by ID - -`GET /api/v1/fleet/teams/{team_id}/policies/{id}` - -#### Parameters - -| Name | Type | In | Description | -| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | -| team_id | integer | url | Defines what team id to operate on | -| id | integer | path | **Required.** The policy's ID. | - -#### Example - -`GET /api/v1/fleet/teams/1/policies/1` - -##### Default response - -`Status: 200` - -``` -{ - "policy": { - "id": 1, - "query_id": 2, - "query_name": "Gatekeeper enabled", - "passing_host_count": 2000, - "failing_host_count": 300, - } -} -``` - -### Add team policy - -`POST /api/v1/fleet/teams/{team_id}/policies` - -#### Parameters - -| Name | Type | In | Description | -| -------- | ------- | ---- | ----------------------------------- | -| team_id | integer | url | Defines what team id to operate on | -| query_id | integer | body | **Required.** The query's ID. | - -#### Example - -`POST /api/v1/fleet/teams/1/policies` - -#### Request body - -``` -{ - "query_id": 12 -} -``` - -##### Default response - -`Status: 200` - -``` -{ - "policy": { - "id": 2, - "query_id": 2, - "query_name": "Primary disk encrypted", - "passing_host_count": 0, - "failing_host_count": 0, - }, -} -``` - -### Remove team policies - -`POST /api/v1/fleet/teams/{team_id}/policies/delete` - -#### Parameters - -| Name | Type | In | Description | -| -------- | ------- | ---- | ------------------------------------------------- | -| team_id | integer | url | Defines what team id to operate on | -| ids | list | body | **Required.** The IDs of the policies to delete. | - -#### Example - -`POST /api/v1/fleet/teams/1/policies/delete` - -#### Request body - ``` { "ids": [ 1 ] @@ -4561,7 +4474,7 @@ Returns a list of the activities that have been performed in Fleet. The followin ##### Default response -```json +``` { "activities": [ { @@ -4724,7 +4637,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Request body -```json +``` { "query": "172", "selected": { @@ -4737,7 +4650,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Default response -```json +``` { "targets": { "hosts": [ @@ -4887,7 +4800,7 @@ None. `Status: 200` -```json +``` { "certificate_chain": } @@ -4911,7 +4824,7 @@ None. `Status: 200` -```json +``` { "org_info": { "org_name": "fleet", @@ -5002,7 +4915,7 @@ None. "enable_log_compression": false } } - }, + } "license": { "tier": "free", "organization": "fleet", @@ -5042,11 +4955,10 @@ None. "enable_log_compression": false } } - }, + } "update_interval": { - "osquery_detail": 3600000000000, - "osquery_policy": 3600000000000 - }, + "osquery_detail": 3600000000000 + } } ``` @@ -5093,13 +5005,13 @@ Modifies the Fleet's configuration with the supplied information. ##### Request body -```json +``` { "org_info": { "org_name": "Fleet Device Management", "org_logo_url": "https://fleetdm.com/logo.png" }, - "smtp_settings": { + "smtp_settings: { "enable_smtp": true, "server": "localhost", "port": "1025" @@ -5111,7 +5023,7 @@ Modifies the Fleet's configuration with the supplied information. `Status: 200` -```json +``` { "org_info": { "org_name": "Fleet Device Management", @@ -5236,7 +5148,7 @@ None. `Status: 200` -```json +``` { "spec": { "secrets": [ @@ -5269,7 +5181,7 @@ Replaces the active global enroll secrets with the secrets specified. ##### Request body -```json +``` { "spec": { "secrets": [ @@ -5287,6 +5199,9 @@ Replaces the active global enroll secrets with the secrets specified. `Status: 200` +``` +{} +``` ### Get enroll secret for a team @@ -5306,7 +5221,7 @@ None. `Status: 200` -```json +``` { "secrets": [ { @@ -5336,20 +5251,20 @@ None. ##### Request body -```json +``` { "email": "john_appleseed@example.com", - "name": "John", + "name": John, "sso_enabled": false, - "global_role": "admin", + "global_role": "admin" "teams": [ { "id": 2, - "role": "observer" + "role: "observer" }, { "id": 3, - "role": "maintainer" + "role: "maintainer" }, ] } @@ -5361,13 +5276,13 @@ None. `Status: 200` -```json +``` { "invite": { "created_at": "0001-01-01T00:00:00Z", "updated_at": "0001-01-01T00:00:00Z", "id": 3, - "invited_by": 1, + "invited_by": 1 "email": "john_appleseed@example.com", "name": "John", "sso_enabled": false, @@ -5419,7 +5334,7 @@ Returns a list of the active invitations in Fleet. `Status: 200` -```json +``` { "invites": [ { @@ -5466,6 +5381,9 @@ Delete the specified invite from Fleet. `Status: 200` +``` +{} +``` ### Verify invite @@ -5487,7 +5405,7 @@ Verify the specified invite. `Status: 200` -```json +``` { "invite": { "created_at": "2021-01-15T00:58:33Z", @@ -5506,7 +5424,7 @@ Verify the specified invite. `Status: 404` -```json +``` { "message": "Resource Not Found", "errors": [ @@ -5536,7 +5454,7 @@ None. `Status: 200` -```json +``` { "version": "3.9.0-93-g1b67826f-dirty", "branch": "version", @@ -5559,7 +5477,7 @@ Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allo To initiate a file carve using the Fleet API, you can use the [live query](#run-live-query) or [scheduled query](#add-scheduled-query-to-a-pack) endpoints to run a query against the `carves` table. -For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../01-Using-Fleet/02-fleetctl-CLI.md#file-carving-with-fleet). +For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../1-Using-Fleet/2-fleetctl-CLI.md#file-carving-with-fleet). ### List carves @@ -5579,7 +5497,7 @@ None. `Status: 200` -```json +``` { "carves": [ { @@ -5634,7 +5552,7 @@ Retrieves the specified carve. `Status: 200` -```json +``` { "carve": { "id": 1, @@ -5674,7 +5592,7 @@ Retrieves the specified carve block. This endpoint retrieves the data that was c `Status: 200` -```json +``` { "data": "aG9zdHMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA..." } @@ -5708,9 +5626,9 @@ _Available in Fleet Premium_ `Status: 200` -```json +``` { - "teams": [ + "teams: [ { "id": 1. "created_at": "2021-07-28T15:58:21Z", @@ -5806,7 +5724,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "name": "workstations" } @@ -5816,12 +5734,12 @@ _Available in Fleet Premium_ `Status: 200` -```json +``` { - "teams": [ + "teams: [ { "name": "workstations", - "id": 1, + "id": 1 "user_ids": [], "host_ids": [], "user_count": 0, @@ -5875,7 +5793,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "user_ids": [1, 17, 22, 32], } @@ -5885,11 +5803,11 @@ _Available in Fleet Premium_ `Status: 200` -```json +``` { "team": { "name": "Workstations", - "id": 1, + "id": 1 "user_ids": [1, 17, 22, 32], "host_ids": [], "user_count": 4, @@ -5927,7 +5845,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], } @@ -5937,11 +5855,11 @@ _Available in Fleet Premium_ `Status: 200` -```json +``` { "team": { "name": "Workstations", - "id": 1, + "id": 1 "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -5979,7 +5897,7 @@ _Available in Fleet Premium_ ##### Request body -```json +``` { "agent_options": { "spec": { @@ -6011,11 +5929,11 @@ _Available in Fleet Premium_ `Status: 200` -```json +``` { "team": { "name": "Workstations", - "id": 1, + "id": 1 "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -6067,74 +5985,9 @@ _Available in Fleet Premium_ `Status: 200` -### Apply team spec - -_Available in Fleet Premium_ - -If the `name` specified is associated with an existing team, this API route, completely replaces this team's existing `agent_options` and `secrets` with those that are specified. - -If the `name` is not already associated with an existing team, this API route creates a new team with the specified `name`, `agent_options`, and `secrets`. - -`POST /api/v1/fleet/spec/teams` - -#### Parameters - -| Name | Type | In | Description | -| ---- | ------ | ---- | ------------------------------ | -| name | string | body | **Required.** The team's name. | -| agent_options | string | body | **Required.** The agent options spec that is applied to the hosts assigned to the specified to team. These agent agent options completely override the global agent options specified in the [`GET /api/v1/fleet/config API route`](#get-configuration)| -| secrets | list | body | **Required.** A list of plain text strings used as the enroll secrets. | - -#### Example - -`POST /api/v1/fleet/spec/teams` - -##### Request body - -```json -{ - "specs": [ - { - "name": "Client Platform Engineering", - "agent_options": { - "spec": { - "config": { - "options": { - "logger_plugin": "tls", - "pack_delimiter": "/", - "logger_tls_period": 10, - "distributed_plugin": "tls", - "disable_distributed": false, - "logger_tls_endpoint": "/api/v1/osquery/log", - "distributed_interval": 10, - "distributed_tls_max_attempts": 3 - }, - "decorators": { - "load": [ - "SELECT uuid AS host_uuid FROM system_info;", - "SELECT hostname AS hostname FROM system_info;" - ] - } - }, - "overrides": {} - } - }, - "secrets": [ - { - "secret": "fTp52/twaxBU6gIi0J6PHp8o5Sm1k1kn", - }, - { - "secret": "bhD5kiX2J+KBgZSk118qO61ZIdX/v8On", - } - ] - } - ] -} ``` - -#### Default response - -`Status: 200` +{} +``` --- @@ -6156,7 +6009,7 @@ If the `name` is not already associated with an existing team, this API route cr ##### Request body -```json +``` { "list": [ { @@ -6191,7 +6044,7 @@ If the `name` is not already associated with an existing team, this API route cr `Status: 200` -```json +``` { "list": [ { @@ -6251,7 +6104,7 @@ If the `name` is not already associated with an existing team, this API route cr `Status: 200` -```json +``` { “software”: [ { diff --git a/docs/01-Using-Fleet/04-Adding-hosts.md b/docs/1-Using-Fleet/4-Adding-hosts.md similarity index 99% rename from docs/01-Using-Fleet/04-Adding-hosts.md rename to docs/1-Using-Fleet/4-Adding-hosts.md index 9ea6e02e39..4872b7a204 100644 --- a/docs/01-Using-Fleet/04-Adding-hosts.md +++ b/docs/1-Using-Fleet/4-Adding-hosts.md @@ -135,4 +135,4 @@ Multiple enroll secrets can be set to allow different groups of hosts to authenticate with Fleet. When a host enrolls, the corresponding enroll secret is recorded and can be used to segment hosts. -To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../01-Using-Fleet/02-fleetctl-CLI.md#enroll-secrets) +To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../1-Using-Fleet/2-fleetctl-CLI.md#enroll-secrets) diff --git a/docs/01-Using-Fleet/05-Osquery-logs.md b/docs/1-Using-Fleet/5-Osquery-logs.md similarity index 90% rename from docs/01-Using-Fleet/05-Osquery-logs.md rename to docs/1-Using-Fleet/5-Osquery-logs.md index 48223c68c6..96109f95c2 100644 --- a/docs/01-Using-Fleet/05-Osquery-logs.md +++ b/docs/1-Using-Fleet/5-Osquery-logs.md @@ -22,21 +22,21 @@ Fleet supports the following logging plugins for osquery logs: - [PubSub](#pubsub) - Logs are written to Google Cloud PubSub topics. - [Stdout](#stdout) - Logs are written to stdout. -To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../02-Deploying/02-Configuration.md#options)). +To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../2-Deploying/2-Configuration.md#options)). ### Filesystem The default logging plugin. - Plugin name: `filesystem` -- Flag namespace: [filesystem](../02-Deploying/02-Configuration.md#filesystem) +- Flag namespace: [filesystem](../2-Deploying/2-Configuration.md#filesystem) With the filesystem plugin, osquery result and/or status logs are written to the local filesystem on the Fleet server. This is typically used with a log forwarding agent on the Fleet server that will push the logs into a logging pipeline. Note that if multiple load-balanced Fleet servers are used, the logs will be load-balanced across those servers (not duplicated). ### Firehose - Plugin name: `firehose` -- Flag namespace: [firehose](../02-Deploying/02-Configuration.md#firehose) +- Flag namespace: [firehose](../2-Deploying/2-Configuration.md#firehose) With the Firehose plugin, osquery result and/or status logs are written to [AWS Firehose](https://aws.amazon.com/kinesis/data-firehose/) streams. This is a very good method for aggregating osquery logs into AWS S3 storage. @@ -45,7 +45,7 @@ Note that Firehose logging has limits [discussed in the documentation](https://d ### Kinesis - Plugin name: `kinesis` -- Flag namespace: [kinesis](../02-Deploying/02-Configuration.md#kinesis) +- Flag namespace: [kinesis](../2-Deploying/2-Configuration.md#kinesis) With the Kinesis plugin, osquery result and/or status logs are written to [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams) streams. @@ -58,7 +58,7 @@ output in the Fleet logs and those logs _will not_ be sent to Kinesis. ### Lambda - Plugin name: `lambda` -- Flag namespace: [lambda](../02-Deploying/02-Configuration.md#lambda) +- Flag namespace: [lambda](../2-Deploying/2-Configuration.md#lambda) With the Lambda plugin, osquery result and/or status logs are written to [AWS Lambda](https://aws.amazon.com/lambda/) functions. @@ -79,7 +79,7 @@ Keep this in mind when using Lambda, as you're charged based on the number of re ### PubSub - Plugin name: `pubsub` -- Flag namespace: [pubsub](../02-Deploying/02-Configuration.md#pubsub) +- Flag namespace: [pubsub](../2-Deploying/2-Configuration.md#pubsub) With the PubSub plugin, osquery result and/or status logs are written to [PubSub](https://cloud.google.com/pubsub/) topics. @@ -88,7 +88,7 @@ Note that messages over 10MB will be dropped, with a notification sent to the fl ### Stdout - Plugin name: `stdout` -- Flag namespace: [stdout](../02-Deploying/02-Configuration.md#stdout) +- Flag namespace: [stdout](../2-Deploying/2-Configuration.md#stdout) With the stdout plugin, osquery result and/or status logs are written to stdout on the Fleet server. This is typically used for debugging or with a log diff --git a/docs/01-Using-Fleet/06-Monitoring-Fleet.md b/docs/1-Using-Fleet/6-Monitoring-Fleet.md similarity index 85% rename from docs/01-Using-Fleet/06-Monitoring-Fleet.md rename to docs/1-Using-Fleet/6-Monitoring-Fleet.md index f7534f6a01..99d1ac1ef2 100644 --- a/docs/01-Using-Fleet/06-Monitoring-Fleet.md +++ b/docs/1-Using-Fleet/6-Monitoring-Fleet.md @@ -54,7 +54,7 @@ Scaling Fleet horizontally is as simple as running more Fleet server processes c The Fleet/osquery system is resilient to loss of availability. Osquery agents will continue executing the existing configuration and buffering result logs during downtime due to lack of network connectivity, server maintenance, or any other reason. Buffering in osquery can be configured with the `--buffered_log_max` flag. -Note that short downtimes are expected during [Fleet server upgrades](./08-Updating-Fleet.md)-fleet.md) that require database migrations. +Note that short downtimes are expected during [Fleet server upgrades](./8-Updating-Fleet.md)-fleet.md) that require database migrations. ### Debugging performance issues @@ -68,7 +68,7 @@ For performance issues in the Fleet server process, please [file an issue](https ##### Generate debug archive (Fleet 3.4.0+) -Use the `fleetctl debug archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./02-fleetctl-CLI.md)) for details on configuring `fleetctl`. +Use the `fleetctl archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./2-fleetctl-CLI.md)) for details on configuring `fleetctl`. The generated `.tar.gz` archive will be available in the current directory. @@ -86,4 +86,4 @@ fleetctl debug archive --context server-a ###### Confidential information -The `fleetctl debug archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. +The `fleetctl archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. diff --git a/docs/01-Using-Fleet/07-Security-best-practices.md b/docs/1-Using-Fleet/7-Security-best-practices.md similarity index 97% rename from docs/01-Using-Fleet/07-Security-best-practices.md rename to docs/1-Using-Fleet/7-Security-best-practices.md index caa13d02b2..4311907ce2 100644 --- a/docs/01-Using-Fleet/07-Security-best-practices.md +++ b/docs/1-Using-Fleet/7-Security-best-practices.md @@ -33,7 +33,7 @@ Passwords are never stored in plaintext in the database. We store a `bcrypt`ed h ### Authentication tokens -The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../02-Deploying/02-Configuration.md#session_duration). +The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../2-Deploying/2-Configuration.md#session_duration). It is possible to revoke all session tokens for a user by forcing a password reset. diff --git a/docs/01-Using-Fleet/08-Updating-Fleet.md b/docs/1-Using-Fleet/8-Updating-Fleet.md similarity index 92% rename from docs/01-Using-Fleet/08-Updating-Fleet.md rename to docs/1-Using-Fleet/8-Updating-Fleet.md index 65d9c6e49c..af41724025 100644 --- a/docs/01-Using-Fleet/08-Updating-Fleet.md +++ b/docs/1-Using-Fleet/8-Updating-Fleet.md @@ -7,7 +7,7 @@ ## Overview -This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../02-Deploying/01-Installation.md). +This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../2-Deploying/1-Installation.md). There are two steps to perform a typical Fleet update. If any other steps are required, they will be noted in the release notes. @@ -18,7 +18,7 @@ As with any enterprise software update, it's a good idea to back up your MySQL d ## Updating the Fleet binary -To update to a new version of Fleet, follow the [same binary install instructions](../02-Deploying/01-Installation.md) from the original installation method you used to install Fleet. +To update to a new version of Fleet, follow the [same binary install instructions](../2-Deploying/1-Installation.md) from the original installation method you used to install Fleet. ### Raw binaries diff --git a/docs/01-Using-Fleet/09-Permissions.md b/docs/1-Using-Fleet/9-Permissions.md similarity index 86% rename from docs/01-Using-Fleet/09-Permissions.md rename to docs/1-Using-Fleet/9-Permissions.md index 17e2acba77..af3bc7c678 100644 --- a/docs/01-Using-Fleet/09-Permissions.md +++ b/docs/1-Using-Fleet/9-Permissions.md @@ -16,8 +16,6 @@ The following table depicts various permissions levels for each role. | ---------------------------------------------------- | -------- | ---------- | ----- | | Browse all hosts | ✅ | ✅ | ✅ | | Filter hosts using labels | ✅ | ✅ | ✅ | -| Browse all policies | ✅ | ✅ | ✅ | -| Filter hosts using policies | ✅ | ✅ | ✅ | | Target hosts using labels | ✅ | ✅ | ✅ | | Run saved queries as live queries against all hosts | ✅ | ✅ | ✅ | | Run custom queries as live queries against all hosts | | ✅ | ✅ | @@ -35,8 +33,6 @@ The following table depicts various permissions levels for each role. | Create labels | | ✅ | ✅ | | Edit labels | | ✅ | ✅ | | Delete labels | | ✅ | ✅ | -| Create new global policies | | ✅ | ✅ | -| Delete global policies | | ✅ | ✅ | | Create users | | | ✅ | | Edit users | | | ✅ | | Delete users | | | ✅ | @@ -70,13 +66,9 @@ The following table depicts various permissions levels in a team. | Action | Observer | Maintainer | | ------------------------------------------------------------ | -------- | ---------- | | Browse hosts assigned to team | ✅ | ✅ | -| Browse policies for hosts assigned to team | ✅ | ✅ | -| Filter hosts assigned to team using policies | ✅ | ✅ | | Filter hosts assigned to team using labels | ✅ | ✅ | | Target hosts assigned to team using labels | ✅ | ✅ | | Run saved queries as live queries on hosts assigned to team | ✅ | ✅ | -| Create new team policies | | ✅ | -| Delete team policies | | ✅ | | Run custom queries as live queries on hosts assigned to team | | ✅ | | Enroll hosts to member team | | ✅ | | Delete hosts belonging to member team | | ✅ | diff --git a/docs/01-Using-Fleet/FAQ.md b/docs/1-Using-Fleet/FAQ.md similarity index 79% rename from docs/01-Using-Fleet/FAQ.md rename to docs/1-Using-Fleet/FAQ.md index bc334c2733..99e0b69318 100644 --- a/docs/01-Using-Fleet/FAQ.md +++ b/docs/1-Using-Fleet/FAQ.md @@ -13,7 +13,6 @@ - [Why aren’t my live queries being logged?](#why-arent-my-live-queries-being-logged) - [Can I use the Fleet API to fetch results from a scheduled query pack?](#can-i-use-the-fleet-api-to-fetch-results-from-a-scheduled-query-pack) - [How do I automatically add hosts to packs when the hosts enroll to Fleet?](#how-do-i-automatically-add-hosts-to-packs-when-the-hosts-enroll-to-Fleet) -- [How do I automatically assign a host to a team when it enrolls with Fleet?](#how-do-i-automatically-assign-a-host-to-a-team-when-it-enrolls-with-fleet) - [How do I resolve an "unknown column" error when upgrading Fleet?](#how-do-i-resolve-an-unknown-column-error-when-upgrading-fleet) ## What do I need to do to switch from Kolide Fleet to FleetDM Fleet? @@ -22,7 +21,7 @@ The upgrade from kolide/fleet to fleetdm/fleet works the same as any minor versi Minor version upgrades in Kolide Fleet often included database migrations and the recommendation to back up the database before migrating. The same goes for FleetDM Fleet versions. -To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./08-Updating-Fleet.md) of the documentation. +To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./8-Updating-Fleet.md) of the documentation. ## Has anyone stress tested Fleet? How many clients can the Fleet server handle? @@ -34,13 +33,13 @@ It’s standard deployment practice to have multiple Fleet servers behind a load No, currently, there’s no way to retrieve the name of the enroll secret with a query. This means that there's no way to create a label using your hosts' enroll secrets and then use this label as a target for queries or query packs. -Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./02-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. +Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./2-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. There is, however, a way to accomplish this even though the answer to the question remains "no": Teams. As of Fleet v4.0.0, you can group hosts in Teams either by enrolling them with a team specific secret, or by transferring hosts to a team. One the hosts you want to target are part of a team, you can create a query and target the team in question. ## How often do labels refresh? Is the refresh frequency configurable? -The update frequency for labels is configurable with the [—osquery_label_update_interval](../02-Deploying/02-Configuration.md#osquery_label_update_interval) flag (default 1 hour). +The update frequency for labels is configurable with the [—osquery_label_update_interval](../2-Deploying/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). ## How do I revoke the authorization tokens for a user? @@ -52,7 +51,7 @@ Fleet can live query the `osquery_schedule` table. Performing this live query al ## How do I monitor a Fleet server? -Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./06-Monitoring-Fleet.md) documentation for details. +Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./6-Monitoring-Fleet.md) documentation for details. ## Why is the “Add User” button disabled? @@ -77,7 +76,7 @@ Live query results (executed in the web UI or `fleetctl query`) are pushed direc ### Scheduled Queries -Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../02-Deploying/02-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. +Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../2-Deploying/2-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. It is possible to configure osqueryd to log query results outside of Fleet. For results to go to Fleet, the `--logger_plugin` flag must be set to `tls`. @@ -85,7 +84,7 @@ It is possible to configure osqueryd to log query results outside of Fleet. For Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. -The [logger configuration options](../02-Deploying/02-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. +The [logger configuration options](../2-Deploying/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. ### Troubleshooting @@ -95,7 +94,7 @@ Expecting results, but not seeing anything in the logs? - Check whether the query is scheduled in differential mode. If so, new results will only be logged when the result set changes. - Ensure that the query is scheduled to run on the intended platforms, and that the tables queried are supported by those platforms. - Use live query to `SELECT * FROM osquery_schedule` to check whether the query has been scheduled on the host. -- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../02-Deploying/02-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. +- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../2-Deploying/2-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. ## Why aren’t my live queries being logged? @@ -105,17 +104,17 @@ Live query results are never logged to the filesystem of the Fleet server. See [ You cannot. Scheduled query results are logged to whatever logging plugin you have configured and are not stored in the Fleet DB. -However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./03-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./03-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). +However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./3-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./3-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). As an example, let's say you want to retrieve a host's OS version, installed software, and kernel version: -Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./03-REST-API.md#list-hosts). +Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./3-REST-API.md#list-hosts). -The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../02-Deploying/02-Configuration.md#feature-flags). +The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../2-Deploying/2-Configuration.md#feature-flags). -Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./03-REST-API.md#get-host). +Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./3-REST-API.md#get-host). -It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./02-fleetctl-CLI.md#fleet-configuration-options). +It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./2-fleetctl-CLI.md#fleet-configuration-options). ## How do I automatically add hosts to packs when the hosts enroll to Fleet? @@ -123,14 +122,10 @@ You can accomplish this by adding specific labels as targets of your pack. First When your hosts enroll to Fleet, they will become a member of the label and, because the label is a target of your pack, these hosts will automatically become targets of the pack. -You can also do this by setting the `targets` field in the [YAML configuration file](./02-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. - -## How do I automatically assign a host to a team when it enrolls with Fleet? - -[Team Enroll Secrets](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md#enroll-hosts-to-a-team) allow you to automatically assign a host to a team. +You can also do this by setting the `targets` field in the [YAML configuration file](./2-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. ## How do I resolve an "unknown column" error when upgrading Fleet? The `unknown column` error typically occurs when the database migrations haven't been run during the upgrade process. -Check out the [documentation on running database migrations](./08-Updating-Fleet.md#running-database-migrations) to resolve this issue. +Check out the [documentation on running database migrations](./8-Updating-Fleet.md#running-database-migrations) to resolve this issue. diff --git a/docs/01-Using-Fleet/README.md b/docs/1-Using-Fleet/README.md similarity index 68% rename from docs/01-Using-Fleet/README.md rename to docs/1-Using-Fleet/README.md index 1edfbc95e9..9dbd331a0b 100644 --- a/docs/01-Using-Fleet/README.md +++ b/docs/1-Using-Fleet/README.md @@ -1,27 +1,27 @@ # Using Fleet -### [Fleet UI](./01-Fleet-UI.md) +### [Fleet UI](./1-Fleet-UI.md) Provides documentation about running and scheduling queries from within the Fleet UI -### [fleetctl CLI](./02-fleetctl-CLI.md) +### [fleetctl CLI](./2-fleetctl-CLI.md) Includes resources for setting up and configuring Fleet via the fleetctl CLI -### [REST API](./03-REST-API.md) +### [REST API](./3-REST-API.md) Provides resources for working with Fleet's API and includes example code for endpoints -### [Adding hosts](./04-Adding-hosts.md) +### [Adding hosts](./4-Adding-hosts.md) Provides resources for enrolling your hosts to Fleet -### [Osquery logs](./05-Osquery-logs.md) +### [Osquery logs](./5-Osquery-logs.md) Includes documentation on the plugin options for working with osquery logs -### [Monitoring Fleet](./06-Monitoring-Fleet.md) +### [Monitoring Fleet](./6-Monitoring-Fleet.md) Provides documentation for load balancer health checks and working with Fleet server metrics and performance -### [Security best practices](./07-Security-best-practices.md) +### [Security best practices](./7-Security-best-practices.md) Includes resources for ways to mitigate against the OWASP top 10 issues -### [Updating Fleet](./08-Updating-Fleet.md) +### [Updating Fleet](./8-Updating-Fleet.md) Includes a guide for how to update and run new versions of Fleet ### [FAQ](./FAQ.md) diff --git a/docs/01-Using-Fleet/configuration-files/README.md b/docs/1-Using-Fleet/configuration-files/README.md similarity index 100% rename from docs/01-Using-Fleet/configuration-files/README.md rename to docs/1-Using-Fleet/configuration-files/README.md diff --git a/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml b/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml rename to docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml diff --git a/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml b/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml rename to docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml diff --git a/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml b/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml similarity index 94% rename from docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml rename to docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml index 098270f021..a26f40a92c 100644 --- a/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +++ b/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml @@ -1,19 +1,19 @@ -apiVersion: v1 -kind: Service -metadata: - name: fleet-loadbalancer - labels: - app: fleet-loadbalancer -spec: - type: LoadBalancer - ports: - - name: proxy-tls - port: 443 - targetPort: 443 - protocol: TCP - - name: proxy-http - port: 80 - targetPort: 80 - protocol: TCP - selector: - app: fleet-webserver +apiVersion: v1 +kind: Service +metadata: + name: fleet-loadbalancer + labels: + app: fleet-loadbalancer +spec: + type: LoadBalancer + ports: + - name: proxy-tls + port: 443 + targetPort: 443 + protocol: TCP + - name: proxy-http + port: 80 + targetPort: 80 + protocol: TCP + selector: + app: fleet-webserver diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml b/docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml rename to docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml b/docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml rename to docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml b/docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml rename to docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml b/docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml rename to docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml b/docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml rename to docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml diff --git a/docs/01-Using-Fleet/configuration-files/single-file-configuration.yml b/docs/1-Using-Fleet/configuration-files/single-file-configuration.yml similarity index 100% rename from docs/01-Using-Fleet/configuration-files/single-file-configuration.yml rename to docs/1-Using-Fleet/configuration-files/single-file-configuration.yml diff --git a/docs/01-Using-Fleet/standard-query-library/README.md b/docs/1-Using-Fleet/standard-query-library/README.md similarity index 100% rename from docs/01-Using-Fleet/standard-query-library/README.md rename to docs/1-Using-Fleet/standard-query-library/README.md diff --git a/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml b/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml similarity index 98% rename from docs/01-Using-Fleet/standard-query-library/standard-query-library.yml rename to docs/1-Using-Fleet/standard-query-library/standard-query-library.yml index a9f9017dce..41d3e8eee0 100644 --- a/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml +++ b/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml @@ -446,13 +446,13 @@ spec: query: SELECT name, path, pid FROM processes WHERE on_disk = 0; purpose: Incident response contributors: alphabrevity ---- + --- apiVersion: v1 kind: query spec: name: Get user files matching a specific hash platforms: macOS, Linux - description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) + Description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) query: SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE ""/Users/%/Documents/%%"") AND sha256 = ""16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75""; purpose: Informational contributors: alphabrevity @@ -466,13 +466,13 @@ spec: query: SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid; purpose: Informational contributors: alphabrevity ---- +—-- apiVersion: v1 kind: query spec: name: Get all listening ports, by process platforms: Linux, macOS, Windows - description: List ports that are listening on all interfaces, along with the process to which they are attached. + Description: List ports that are listening on all interfaces, along with the process to which they are attached. query: SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = "0.0.0.0"; purpose: Informational contributors: alphabrevity @@ -482,7 +482,7 @@ kind: query spec: name: Get whether TeamViewer is installed/running platforms: Windows - description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. + description: Description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. query: SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE "%teamviewer%"; purpose: Informational contributors: alphabrevity diff --git a/docs/02-Deploying/01-Installation.md b/docs/2-Deploying/1-Installation.md similarity index 91% rename from docs/02-Deploying/01-Installation.md rename to docs/2-Deploying/1-Installation.md index d59018f353..7a1f0f89c7 100644 --- a/docs/02-Deploying/01-Installation.md +++ b/docs/2-Deploying/1-Installation.md @@ -15,7 +15,7 @@ The Fleet application is distributed as a single static binary. This binary serv - The Fleet application API endpoints - The osquery TLS server API endpoints -All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./02-Configuration.md) documentation for information on how to use and configure the Fleet application. +All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./2-Configuration.md) documentation for information on how to use and configure the Fleet application. ## Installing the Fleet binary @@ -29,7 +29,7 @@ Pull the latest Fleet docker image: docker pull fleetdm/fleet ``` -For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. ### Raw binaries @@ -47,7 +47,7 @@ unzip fleet.zip 'linux/*' -d fleet ./fleet/linux/fleet_linux_amd64 --help ``` -For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. ## TLS configuration @@ -65,7 +65,7 @@ Fleet currently has two infrastructure dependencies in addition to the `fleet` w ### MySQL -Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./02-Configuration.md) document. +Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./2-Configuration.md) document. Fleet requires at least MySQL version 5.7. @@ -73,4 +73,4 @@ For host expiry configuration, the [event scheduler](https://dev.mysql.com/doc/r ### Redis -Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./02-Configuration.md) document. +Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./2-Configuration.md) document. diff --git a/docs/02-Deploying/02-Configuration.md b/docs/2-Deploying/2-Configuration.md similarity index 84% rename from docs/02-Deploying/02-Configuration.md rename to docs/2-Deploying/2-Configuration.md index 91ab7e63d1..1a41211d35 100644 --- a/docs/02-Deploying/02-Configuration.md +++ b/docs/2-Deploying/2-Configuration.md @@ -131,9 +131,7 @@ All duration-based settings accept valid time units of `s`, `m`, `h`. ##### MySQL -This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica, and it is recommended in that case to set a non-zero value for `mysql_read_replica_conn_max_lifetime` as in some environments, the replica's address may dynamically change to point -from the primary to an actual distinct replica based on auto-scaling options, so existing idle connections need to be recycled -periodically. +This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica. ###### mysql_address @@ -291,7 +289,7 @@ Maximum idle connections to database. This value should be equal to or less than max_idle_conns: 50 ``` -###### mysql_conn_max_lifetime +###### conn_max_lifetime Maximum amount of time, in seconds, a connection may be reused. @@ -360,7 +358,7 @@ Whether or not to duplicate Live Query results to another Redis channel named `L ###### redis_connect_timeout -Timeout for redis connection. +Timeout for redis connection. - Default value: 5s - Environment variable: `FLEET_REDIS_CONNECT_TIMEOUT` @@ -384,38 +382,6 @@ Interval between keep alive probes. keep_alive: 30s ``` -###### redis_connect_retry_attempts - -Maximum number of attempts to retry a failed connection to a redis node. Only -certain type of errors are retried, such as connection timeouts. - -- Default value: 0 (no retry) -- Environment variable: `FLEET_REDIS_CONNECT_RETRY_ATTEMPTS` -- Config file format: - - ``` - redis: - connect_retry_attempts: 2 - ``` - -###### redis_cluster_follow_redirections - -Whether or not to automatically follow redirection errors received from the -Redis server. Applies only to Redis Cluster setups, ignored in standalone -Redis. In Redis Cluster, keys can be moved around to different nodes when the -cluster is unstable and reorganizing the data. With this configuration option -set to true, those (typically short and transient) redirection errors can be -handled transparently instead of ending in an error. - -- Default value: false -- Environment variable: `FLEET_REDIS_CLUSTER_FOLLOW_REDIRECTIONS` -- Config file format: - - ``` - redis: - cluster_follow_redirections: true - ``` - ##### Server ###### server_address @@ -435,7 +401,7 @@ The address to serve the Fleet webserver. The TLS cert to use when terminating TLS. -See [TLS certificate considerations](./01-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. +See [TLS certificate considerations](./1-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. - Default value: `./tools/osquery/fleet.crt` - Environment variable: `FLEET_SERVER_CERT` @@ -681,23 +647,6 @@ Valid time units are `s`, `m`, `h`. osquery: label_update_interval: 30m ``` - -###### osquery_policy_update_interval - -The interval at which Fleet will ask osquery agents to update their results for policy queries. - -Setting this to a higher value can reduce baseline load on the Fleet server in larger deployments. - -Valid time units are `s`, `m`, `h`. - -- Default value: `1h` -- Environment variable: `FLEET_OSQUERY_POLICY_UPDATE_INTERVAL` -- Config file format: - - ``` - osquery: - policy_update_interval: 30m - ``` ###### osquery_detail_update_interval @@ -746,24 +695,6 @@ Options are `filesystem`, `firehose`, `kinesis`, `lambda`, `pubsub`, and `stdout result_log_plugin: firehose ``` -###### osquery_max_jitter_percent - -Given an update interval (label, or details), this will add up to the defined percentage in randomness to the interval. - -The goal of this is to prevent all hosts from checking in with data at the same time. - -So for example, if the label_update_interval is 1h, and this is set to 10. It'll add up a random number between 0 and 6 minutes -to the amount of time it takes for fleet to give the host the label queries. - -- Default value: `10` -- Environment variable: `FLEET_OSQUERY_MAX_JITTER_PERCENT` -- Config file format: - - ``` - osquery: - max_jitter_percent: 10 - ``` - ##### Logging (Fleet server logging) ###### logging_debug @@ -1365,13 +1296,13 @@ When `current_instance_checks` is set to `auto` (the default), Fleet instances w How often vulnerabilities are checked. -- Default value: `1h` +- Default value: `1hr` - Environment variable: `FLEET_VULNERABILITIES_PERIODICITY` - Config file format: ``` vulnerabilities: - periodicity: 1h + periodicity: 1hr ``` ###### cpe_database_url @@ -1481,7 +1412,7 @@ sudo systemctl daemon-reload sudo systemctl restart fleet.service ``` -## Configuring Single Sign On (SSO) +## Configuring Single Sign On Fleet supports SAML single sign on capability. @@ -1491,22 +1422,33 @@ Fleet supports the SAML Web Browser SSO Profile using the HTTP Redirect Binding. ### Identity Provider (IDP) Configuration -Setting up the service provider (Fleet) with an identity provider generally requires the following information: +Setting up the connected application (Fleet) with an identity provider generally requires the following information: - _Assertion Consumer Service_ - This is the call back URL that the identity provider - will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. On Google it is "ACS URL". The value that you supply will be a fully qualified URL consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, if your Fleet web address is https://fleet.example.com, then the value you would use in the identity provider configuration would be: + will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. The value that you supply will be a fully qualified URL + consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, + if your Fleet web address is https://fleet.acme.org, then the value you would + use in the identity provider configuration would be: ``` - https://fleet.example.com/api/v1/fleet/sso/callback + https://fleet.acme.org/api/v1/fleet/sso/callback ``` -- _Entity ID_ - This value is an identifier that you choose. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. +- _Entity ID_ - This value is a URI that you define. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. - _Name ID Format_ - The value should be `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`. This may be shortened in the IDP setup to something like `email` or `EmailAddress`. - _Subject Type (Application username in Okta)_ - `email`. -After supplying the above information, the IDP will generate an issuer URI and a metadata that will be used to configure Fleet as a service provider. +After supplying the above information, the IDP will generate an issuer URI and a metadata URL that will be used to configure Fleet as a service provider. + + #### Example Okta IDP Configuration + + ![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) + +> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. + +> Individual users must also be setup on the IDP before they can sign in to Fleet. ### Fleet SSO Configuration @@ -1516,10 +1458,12 @@ If your IDP supports dynamic configuration, like Okta, you only need to provide Otherwise, the following values are required: -- _Identity Provider Name_ - A human readable name of the IDP. This is rendered on the login page. +- _Identity Provider Name_ - A human friendly name of the IDP. - _Entity ID_ - A URI that identifies your Fleet instance as the issuer of authorization - requests (eg. `fleet.example.com`). This much match the _Entity ID_ configured with the IDP. + requests. Assuming your company name is Acme, an example might be `fleet.acme.org` although + the value could be anything as long as it is unique to Fleet as a service provider + and matches the entity provider value used in the IDP configuration. - _Issuer URI_ - This value is obtained from the IDP. @@ -1543,69 +1487,6 @@ It is strongly recommended that at least one admin user is set up to use the tra based log in so that there is a fallback method for logging into Fleet in the event of SSO configuration problems. -#### Okta IDP Configuration - -![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) - -> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. - -> Individual users must also be setup on the IDP before they can sign in to Fleet. - -### Google Workspace IDP Configuration - -Follow these steps to configure Fleet SSO with Google Workspace. This will require administrator permissions in Google Workspace. - -1. Navigate to the [Web and Mobile Apps](https://admin.google.com/ac/apps/unified) section of the Google Workspace dashboard. Click _Add App -> Add custom SAML app_. - -Screen Shot 2021-09-15 at 2 47 36 PM - -2. Enter `Fleet` for the _App name_ and click _Continue_. - -Screen Shot 2021-09-15 at 2 50 06 PM - -3. Click _Download Metadata_, saving the metadata to your computer. Copy the _SSO URL_. Click _Continue_. - -Screen Shot 2021-09-15 at 2 52 56 PM - -4. In Fleet, navigate to the _Organization Settings_ page. Configure the _SAML Single Sign On Options_ section. - -- Check the _Enable Single Sign On_ checkbox. -- For _Identity Provider Name_ use `Google`. -- For _Entity ID_, use a unique identifier such as `fleet.example.com`. Note that Google seems to error when the provided ID includes `https://`. -- For _Issuer URI_, paste the _SSO URL_ copied from step 3. -- For _Metadata_, paste the contents of the downloaded metadata XML from step 3. -- All other fields can be left blank. - -Click _Update settings_ at the bottom of the page. - -Screen Shot 2021-09-15 at 5 32 25 PM - -5. In Google Workspace, configure the _Service provider details_. - -- For _ACS URL_, use `https:///api/v1/fleet/sso/callback` (eg. `https://fleet.example.com/api/v1/fleet/sso/callback`). -- For Entity ID, use **the same unique identifier from step 4** (eg. `fleet.example.com`). -- For _Name ID format_ choose `EMAIL`. -- For _Name ID_ choose `Basic Information > Primary email`. -- All other fields can be left blank. - -Click _Continue_ at the bottom of the page. - -Screen Shot 2021-09-15 at 5 36 56 PM - -6. Click _Finish_. - -Screen Shot 2021-09-15 at 2 57 20 PM - -7. Click the down arrow on the _User access_ section of the app details page. - -Screen Shot 2021-09-15 at 2 57 51 PM - -8. Check _ON for everyone_. Click _Save_. - -Screen Shot 2021-09-15 at 2 58 09 PM - -9. Enable SSO for a test user and try logging in. Note that Google sometimes takes a long time to propagate the SSO configuration, and it can help to try logging in to Fleet with an Incognito/Private window in the browser. - ## Feature flags Fleet features are sometimes gated behind feature flags. This will usually be due to not-yet-stable APIs, or not-fully-tested performance characteristics. diff --git a/docs/02-Deploying/03-Example-deployment-scenarios.md b/docs/2-Deploying/3-Example-deployment-scenarios.md similarity index 95% rename from docs/02-Deploying/03-Example-deployment-scenarios.md rename to docs/2-Deploying/3-Example-deployment-scenarios.md index 7dde2f46d7..b1b004859e 100644 --- a/docs/02-Deploying/03-Example-deployment-scenarios.md +++ b/docs/2-Deploying/3-Example-deployment-scenarios.md @@ -48,7 +48,7 @@ vagrant ssh ### Installing Fleet -To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/01-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. +To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/1-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. For example, after downloading: ```sh @@ -190,11 +190,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on CentOS, you can run the following: @@ -357,11 +357,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on Ubuntu, you can run the following: @@ -453,14 +453,14 @@ We will use this address when we configure the Kubernetes deployment and databas The last step is to run the Fleet database migrations on your new MySQL server. To do this, run the following: ``` -kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` In Kubernetes, you can only run a job once. If you'd like to run it again (i.e.: you'd like to run the migrations again using the same file), you must delete the job before re-creating it. To delete the job and re-run it, you can run the following commands: ``` -kubectl delete -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml -kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl delete -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` #### Redis @@ -536,7 +536,7 @@ ts=2017-11-16T02:48:38.441148166Z transport=https address=0.0.0.0:443 msg=listen Now that the Fleet server is running on our cluster, we have to expose the Fleet webservers to the internet via a load balancer. To create a Kubernetes `Service` of type `LoadBalancer`, run the following: ``` -kubectl apply -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +kubectl apply -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml ``` #### Configure DNS diff --git a/docs/02-Deploying/04-fleetctl-agent-updates.md b/docs/2-Deploying/4-fleetctl-agent-updates.md similarity index 100% rename from docs/02-Deploying/04-fleetctl-agent-updates.md rename to docs/2-Deploying/4-fleetctl-agent-updates.md diff --git a/docs/02-Deploying/FAQ.md b/docs/2-Deploying/FAQ.md similarity index 84% rename from docs/02-Deploying/FAQ.md rename to docs/2-Deploying/FAQ.md index 5ed8391279..4514a88d9b 100644 --- a/docs/02-Deploying/FAQ.md +++ b/docs/2-Deploying/FAQ.md @@ -12,8 +12,6 @@ - [Why am I receiving a database connection error when attempting to "prepare" the database?](#why-am-i-receiving-a-database-connection-error-when-attempting-to-prepare-the-database) - [Is Fleet available as a SaaS product?](#is-fleet-available-as-a-saas-product) - [Is Fleet compatible with X flavor of MySQL?](#is-fleet-compatible-with-x-flavor-of-mysql) -- [What are the MySQL user access requirements?](#what-are-the-mysql-user-requirements) -- [What is duplicate enrollment and how do I fix it?](#what-is-duplicate-enrollment-and-how-do-i-fix-it) ## How do I get support for working with Fleet? @@ -27,7 +25,7 @@ Yes. Fleet scales horizontally out of the box as long as all of the Fleet server Note that osquery logs will be distributed across the Fleet servers. -Read the [performance documentation](../01-Using-Fleet/06-Monitoring-Fleet.md#fleet-server-performance) for more. +Read the [performance documentation](../1-Using-Fleet/6-Monitoring-Fleet.md#fleet-server-performance) for more. ## Why aren't my osquery agents connecting to Fleet? @@ -73,15 +71,15 @@ These configurations cannot be managed centrally from Fleet. ## What do I do about "too many open files" errors? -This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./02-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. +This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./2-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. -Some deployments may benefit by setting the [`--server_keepalive`](./02-Configuration.md#server_keepalive) flag to false. +Some deployments may benefit by setting the [`--server_keepalive`](./2-Configuration.md#server_keepalive) flag to false. This was also seen as a symptom of a different issue: if you're deploying on AWS on T type instances, there are different scenarios where the activity can increase and the instances will burst. If they run out of credits, then they'll stop processing leaving the file descriptors open. ## I upgraded my database, but Fleet is still running slowly. What could be going on? -This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./02-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./02-Configuration.md#mysql_max_idle_conns) flags appropriately. +This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./2-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./2-Configuration.md#mysql_max_idle_conns) flags appropriately. ## Why am I receiving a database connection error when attempting to "prepare" the database? @@ -106,13 +104,3 @@ No. Currently, Fleet is only available for self-hosting on premises or in the cl ## Is Fleet compatible with X flavor of MySQL? Fleet is built to run on MySQL 5.7 or above. However, particularly with AWS Aurora, we recommend 2.10.0 and above, as we've seen issues with anything below that. - -## What are the MySQL user requirements? - -The user `fleet prepare db` (via environment variable `FLEET_MYSQL_USERNAME` or command line flag `--mysql_username=`) uses to interact with the database needs to be able to create, alter, and drop tables as well as the ability to create temporary tables. - -## What is duplicate enrollment and how do I fix it? - -Duplicate host enrollment is when more than one host enrolls in Fleet using the same identifier (hardware UUID or osquery generated UUID). This can be caused by cloning a VM Image with an already enrolled -osquery client. To resolve the issues, it's advised to configure `--osquery_host_identifier` to `uuid`, and then delete the single host record for that whole set of hosts in the Fleet UI. You can find more information about -[host identifiers here](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#osquery_host_identifier). \ No newline at end of file diff --git a/docs/02-Deploying/README.md b/docs/2-Deploying/README.md similarity index 67% rename from docs/02-Deploying/README.md rename to docs/2-Deploying/README.md index 5b62992f4b..19ad6389e1 100644 --- a/docs/02-Deploying/README.md +++ b/docs/2-Deploying/README.md @@ -1,15 +1,15 @@ # Deployment -### [Installation](./01-Installation.md) +### [Installation](./1-Installation.md) Provides documentation on installing the Fleet binary and Fleet’s infrastructure dependencies -### [Configuration](./02-Configuration.md) +### [Configuration](./2-Configuration.md) Includes resources for configuring the Fleet binary, managing osquery configurations, and running with systemd -### [Example deployment scenarios](./03-Example-deployment-scenarios.md) +### [Example deployment scenarios](./3-Example-deployment-scenarios.md) Includes deployment walkthroughs for Fleet on CentOS, Ubuntu, and Kubernetes. -### [Self-managed agent updates](./04-fleetctl-agent-updates.md) +### [Self-managed agent updates](./4-fleetctl-agent-updates.md) Information about running an update server with fleetctl. ### [FAQ](./FAQ.md) diff --git a/docs/03-Contributing/01-Building-Fleet.md b/docs/3-Contributing/1-Building-Fleet.md similarity index 100% rename from docs/03-Contributing/01-Building-Fleet.md rename to docs/3-Contributing/1-Building-Fleet.md diff --git a/docs/03-Contributing/02-Testing.md b/docs/3-Contributing/2-Testing.md similarity index 78% rename from docs/03-Contributing/02-Testing.md rename to docs/3-Contributing/2-Testing.md index 1aae17579d..73ef0a7d42 100644 --- a/docs/03-Contributing/02-Testing.md +++ b/docs/3-Contributing/2-Testing.md @@ -7,7 +7,7 @@ - [Test hosts](#test-hosts) - [Email](#email) - [Database backup/restore](#database-backuprestore) -- [Seeding Data](./06-Seeding-Data.md) +- [Teams seed data](#teams-seed-data) - [MySQL shell](#mysql-shell) - [Testing SSO](#testing-sso) @@ -40,7 +40,7 @@ go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.0 Make sure it is available in your PATH. To execute the basic unit and integration tests, run the following from the root of the repository: ``` -REDIS_TEST=1 MYSQL_TEST=1 make test +MYSQL_TEST=1 make test ``` It is a good idea to run `make test` before submitting a Pull Request. @@ -50,7 +50,7 @@ It is a good idea to run `make test` before submitting a Pull Request. To run all Go unit tests, run the following: ``` -REDIS_TEST=1 MYSQL_TEST=1 make test-go +make test-go ``` #### Go linters @@ -105,14 +105,6 @@ To run email related integration tests using MailHog set environment as follows: MAIL_TEST=1 make test-go ``` -#### Network tests - -A few tests require network access as they make requests to external hosts. Given that the network is unreliable, may not be available, and those hosts may also not be unavailable, those tests are skipped by default and are opt-in via the `NETWORK_TEST` environment variable. To run them: - -``` -NETWORK_TEST=1 make test-go -``` - ### Viewing test coverage When you run `make test` or `make test-go` from the root of the repository, test coverage reports are generated in every subpackage. For example, the `server` subpackage will have a coverage report generated in `./server/server.cover` @@ -137,7 +129,7 @@ E2E tests are run using Docker and Cypress. #### Preparation -Make sure dependencies are up to date and the [Fleet binaries are built locally](./01-Building-Fleet.md). +Make sure dependencies are up to date and the [Fleet binaries are built locally](./1-Building-Fleet.md). For Fleet Free tests: @@ -231,6 +223,51 @@ Restore: Note that a "restore" will replace the state of the development database with the state from the backup. +## Teams seed data + +When developing Fleet, it may be useful to create seed data that includes users and teams. + +Check out this Loom demo video that walks through creating teams seed data: +https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 + +For a text-based walkthrough, check out the following steps: + +First, create a `env` file with the following contents: + +``` +export SERVER_URL=https://localhost:8080 # your fleet server url and port +export CURL_FLAGS='-k -s' # set insecure flag +export TOKEN=eyJhbGciOi... # your login token +``` + +Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. + +``` +export FLEET_ENV_PATH=/Users/victor/fleet_env +``` + +Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. + +The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. + +``` +./tools/api/fleet/teams/create_free +``` + +The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. + +``` +./tools/api/fleet/teams/create_premium +``` + +The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. + +``` +./tools/api/fleet/teams/create_figma +``` + +Each user generated by the script has their password set to `user123#`. + ## MySQL shell Connect to the MySQL shell to view and interact directly with the contents of the development database. diff --git a/docs/03-Contributing/03-Migrations.md b/docs/3-Contributing/3-Migrations.md similarity index 100% rename from docs/03-Contributing/03-Migrations.md rename to docs/3-Contributing/3-Migrations.md diff --git a/docs/03-Contributing/04-Committing-Changes.md b/docs/3-Contributing/4-Committing-Changes.md similarity index 96% rename from docs/03-Contributing/04-Committing-Changes.md rename to docs/3-Contributing/4-Committing-Changes.md index 73ed37ff02..ac99857ae9 100644 --- a/docs/03-Contributing/04-Committing-Changes.md +++ b/docs/3-Contributing/4-Committing-Changes.md @@ -12,14 +12,11 @@ For significant changes, it is a good idea to discuss the proposal with the Flee Please keep in mind that any code merged to the Fleet repository becomes the responsibility of the Fleet team to maintain. Because of this, we are careful to ensure any contributions fit Fleet's vision, are well-tested, and high quality. We will work with contributors to ensure the appropriate standards are met. -## Fleet Device Management team members -Fleet Device Management team members may not copy queries from external sources except when that content has an explicit license allowing such use, or permission has been granted by the creator. - ## Pull Requests Each developer (internal or external) creates a fork of the Fleet repository, committing changes to a branch within their fork. Changes are submitted by PR to be merged into Fleet. -GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [02-Testing.md](./02-Testing.md). +GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [2-Testing.md](./2-Testing.md). For features that are still in-progress, the Pull Request can be marked as a "Draft". This helps make it clear which PRs are ready for review and merge. diff --git a/docs/03-Contributing/05-Releasing-Fleet.md b/docs/3-Contributing/5-Releasing-Fleet.md similarity index 100% rename from docs/03-Contributing/05-Releasing-Fleet.md rename to docs/3-Contributing/5-Releasing-Fleet.md diff --git a/docs/03-Contributing/FAQ.md b/docs/3-Contributing/FAQ.md similarity index 93% rename from docs/03-Contributing/FAQ.md rename to docs/3-Contributing/FAQ.md index 68c8d53b1b..4020cd0854 100644 --- a/docs/03-Contributing/FAQ.md +++ b/docs/3-Contributing/FAQ.md @@ -25,7 +25,7 @@ server/fleet/emails.go:90:23: undefined: Asset make: *** [fleet] Error 2 ``` -If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./01-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. +If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./1-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. ## Adding hosts for testing diff --git a/docs/03-Contributing/README.md b/docs/3-Contributing/README.md similarity index 69% rename from docs/03-Contributing/README.md rename to docs/3-Contributing/README.md index 98a9b38856..159d3045b4 100644 --- a/docs/03-Contributing/README.md +++ b/docs/3-Contributing/README.md @@ -1,18 +1,18 @@ # Contribution -### [Building Fleet](./01-Building-Fleet.md) +### [Building Fleet](./1-Building-Fleet.md) Provides documentation about building the code, development infrastructure, and database migrations -### [Testing](./02-Testing.md) +### [Testing](./2-Testing.md) Includes documentation about Fleet's full test suite and integration tests -### [Migrations](./03-Migrations.md) +### [Migrations](./3-Migrations.md) Information about creating and updating database migrations -### [Committing Changes](./04-Committing-Changes.md) +### [Committing Changes](./4-Committing-Changes.md) Contains information about how to merge changes into the codebase -### [Releasing Fleet](./05-Releasing-Fleet.md) +### [Releasing Fleet](./5-Releasing-Fleet.md) Provides a guide for Fleet's release process ### [FAQ](./FAQ.md) diff --git a/docs/README.md b/docs/README.md index 5e29c0c967..3da7649ef8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,13 +2,13 @@ Welcome to the documentation for the Fleet osquery fleet manager. -### [Using Fleet](./01-Using-Fleet/README.md) +### [Using Fleet](./1-Using-Fleet/README.md) Resources for using the Fleet UI, fleetctl CLI, and Fleet REST API. -### [Deploying](./02-Deploying/README.md) +### [Deploying](./2-Deploying/README.md) Resources for installing Fleet's infrastructure dependencies, configuring Fleet, deploying osquery to hosts, and viewing example deployment scenarios. -### [Contributing](./03-Contributing/README.md) +### [Contributing](./3-Contributing/README.md) If you're interested in interacting with the Fleet source code, you'll find information on modifying and building the code here. --- diff --git a/frontend/README.md b/frontend/README.md index 147b24000f..6331e7ee9f 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,40 +1,34 @@ # Fleet Front-End -The Fleet front-end is a Single Page Application using React with Typescript and Hooks. - -## Table of Contents -- [Running the Fleet web app](#running-the-fleet-web-app) -- [Directory Structure](#directory-structure) -- [Deprecated](#deprecated) -- [Patterns](#patterns) - - [Typing](#typing) - - [React Hooks (Functional Components)](#react-hooks-functional-components) - - [React Context](#react-context) - - [Fleet API Calls](#fleet-api-calls) - - [Page Routing](#page-routing) - - [Other](#other) +The Fleet front-end is a Single Page Application using React and Redux. ## Running the Fleet web app For details instruction on building and serving the Fleet web application -consult the [Contributing documentation](../docs/03-Contributing/README.md). +consult the [Contributing documentation](../docs/3-Contributing/README.md). ## Directory Structure Component directories in the Fleet front-end application encapsulate the entire -component, including files for the component and its styles. The +component, including files for the component, helper functions, styles, and tests. The typical directory structure for a component is as follows: ``` -└── ComponentName - ├── _styles.scss - ├── ComponentName.tsx - ├── index.ts +|-- ComponentName +| |-- _styles.scss +| |-- ComponentName.jsx +| |-- ComponentName.tests.jsx +| |-- helpers.js +| |-- helpers.tests.js +| |-- index.js ``` - `_styles.scss`: The component css styles -- `ComponentName.tsx`: The React component -- `index.ts`: Exports the React component +- `ComponentName.jsx`: The React component +- `ComponentName.tests.jsx`: The React component tests +- `helpers.js`: Helper functions used by the component +- `helpers.tests.js`: Tests for the component's helper functions +- `index.js`: Exports the React component - This file is helpful as it allows other components to import the component by it's directory name. Without this file the component name would have to be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). @@ -52,34 +46,40 @@ The component directory contains the React components rendered by pages. They are typically not connected to the redux state but receive props from their parent components to render data and handle user interactions. -### [context](./context) - -The context directory contains the React Context API pattern for various entities. -Only entities that are needed across the app has a global context. For example, -the [logged in user](./context/app.tsx) (`currentUser`) has multiple pages and components -where its information is pulled. - ### [interfaces](./interfaces) -Files in the interfaces directory are used to specify the Typescript interface for a reusable Fleet +Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet entity. This is designed to DRY up the code and increase re-usability. These -interfaces are imported in to component files and implemented when defining the -component's props. +interfaces are imported into component files and implemented when defining the +component's PropTypes. -**Additionally, local interfaces are used for props of local components.** +### [fleet](./fleet) + +The default export of the `fleet` directory is the API client. More info can be +found at the [API client documentation page](./fleet/README.md). ### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). -The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general -app UI (header, sidebar) and render child components. +The Layout is rendered from the [router](./router/index.jsx) and are used to set up the general app UI (header, sidebar) and render child components. The child components rendered by the layout are typically page components. ### [pages](./pages) Page components are React components typically rendered from the [router](./router). -React Router passed props to these pages in case they are needed. Examples include -the `router`, `location`, and `params` objects. +These components are connected to redux state and are used to gather data from +redux and pass that data to child components (located in the [components +directory](./components). As +connected components, Pages are also used to dispatch actions. Actions +dispatched from Pages are intended to update redux state and oftentimes include +making a call to the Fleet API. + +### [redux](./redux) + +The redux directory holds all of the application's redux middleware, actions, +and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. +More information about the redux configuration can be found at the [Redux +Documentation page](./redux/README.md) ### [router](./router) @@ -90,10 +90,6 @@ file which holds the application paths as string constants for reference throughout the app. These paths are typically referenced from the [App Constants](./app_constants) object. -### [services](./services) - -CRUD functions for all Fleet entities (e.g. `query`) that link directly to the Fleet API. - ### [styles](./styles) The styles directory contains the general app style setup and variables. It @@ -104,221 +100,22 @@ includes variables for the app color hex codes, fonts (families, weights and siz The templates directory contains the HTML file that renders the React application via including the `bundle.js` and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. +### [test](./test) + +The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. +More on test helpers, stubs, and request mocks [here](./test/README.md). + ### [utilities](./utilities) -The utilities directory contains re-usable functions and constants for use throughout the +The utilities directory contains re-usable functions for use throughout the application. The functions include helpers to convert an array of objects to CSV, debounce functions to prevent multiple form submissions, format API errors, etc. -## Deprecated +## Forms -These directories and files are still used (as of 9/14/21) but are being replaced by newer code: +For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). -- [fleet](./fleet), now using [services](./services) -- [redux](./redux), now using [services](./services), local states, and various entities directly (e.g. React Router) -- [Form.jsx Higher Order Component](./components/forms/README.md), now creating forms with local states with React Hooks (i.e. `useState`) +## API Client -To view the deprecated documentation, [click here](./README_deprecated.md). - -## Patterns - -### Typing -All Javascript and React files use Typescript, meaning the extensions are `.ts` and `.tsx`. Here are the guidelines on how we type at Fleet: - -- Use *[global entity interfaces](#interfaces)* when interfaces are used multiple times across the app -- Use *local interfaces* when typing entities limited to the specific page or component -- Local interfaces for page and component props - - ```typescript - // page - interface IPageProps { - prop1: string; - prop2: number; - ... - } - - // Note: Destructure props in page/component signature - const PageOrComponent = ({ - prop1, - prop2, - }: IPageProps) => { - - return ( - // ... - ); - }; - ``` - -- Local states -```typescript -const [item, setItem] = useState(""); -``` - -- Fetch function signatures (i.e. `react-query`) -```typescript -useQuery(params) -``` - -- Custom functions, including callbacks -```typescript -const functionWithTableName = (tableName: string): boolean => { - // do something -}; -``` - -### React Hooks (Functional Components) - -[Hooks](https://reactjs.org/docs/hooks-intro.html) are used to track state and use other features -of React. Hooks are only allowed in functional components, which are created like so: - -```typescript -import React, { useState, useEffect } from "React"; - -const PageOrComponent = (props) => { - const [item, setItem] = useState(""); - - // runs only on first mount (replaces componentDidMount) - useEffect(() => { - // do something - }, []); - - // runs only when `item` changes (replaces componentDidUpdate) - useEffect(() => { - // do something - }, [item]); - - return ( - // ... - ); -}; -``` - -**Note: Other hooks are available per [React's documentation](https://reactjs.org/docs/hooks-intro.html).** - -### React Context - -[React Context](https://reactjs.org/docs/context.html) is a store similar to Redux. It stores -data that is desired and allows for retrieval of that data in whatever component is in need. -View currently working contexts in the [context directory](./context). - -### Fleet API Calls - -**Deprecated:** - -Redux was used to make API calls, along with the [fleet](./fleet) directory. - -**Current:** - -The [services](./services) directory stores all API calls and is to be used in two ways: -- A direct `async/await` assignment -- Using `react-query` if requirements call for loading data right away or based on dependencies. - -Examples below: - -*Direct assignment* -```typescript -// page -import ... -import queryAPI from "services/entities/queries"; - -const PageOrComponent = (props) => { - const doSomething = async () => { - const response = await queryAPI.load(param); - // do something - }; - - return ( - // ... - ); -}; -``` - -*React Query* - -`react-query` ([docs here](https://react-query.tanstack.com/overview)) is a data-fetching library that -gives us the ability to fetch, cache, sync and update data with a myriad of options and properties. - -```typescript -import ... -import { useQuery, useMutation } from "react-query"; -import queryAPI from "services/entities/queries"; - -const PageOrComponent = (props) => { - // retrieve the query based on page/component load - // and dependencies for when to refetch - const { - isLoading, - data, - error, - ...otherProps, - } = useQuery( - "query", - () => queryAPI.load(param), - { - ...options - } - ); - - // `props` is a bucket of properties that can be used when - // updating data. for example, if you need to know whether - // a mutation is loading, there is a prop for that. - const { ...props } = useMutation((formData: IForm) => - queryAPI.create(formData) - ); - - return ( - // ... - ); -}; -``` - -### Page Routing - -**Deprecated:** - -Redux was used to manage redirecting to different pages of the app. - -**Current:** - -We use React Router directly to navigate between pages. For page components, -React Router (v3) supplies a `router` prop that can be easily accessed. -When needed, the `router` object contains a `push` function that redirects -a user to whatever page desired. For example: - -```typescript -// page -import PATHS from "router/paths"; - -interface IPageProps { - router: any; // no typing in react-router v3 -} - -const PageOrComponent = ({ - router, -}: IPageProps) => { - const doSomething = async () => { - router.push(PATHS.HOME); - }; - - return ( - // ... - ); -}; -``` - -### Other - -**Local states** - -Our first line of defense for state management is local states (i.e. `useState`). We -use local states to keep pages/components separate from one another and easy to -maintain. If states need to be passed to direct children, then prop-drilling should -suffice as long as we do not go more than two levels deep. Otherwise, if states need -to be used across multiple unrelated components or 3+ levels from a parent, -then the [app's context](#react-context) should be used. - -**File size** - -The recommend line limit per page/component is 500 lines. This is only a recommendation. -Larger files are to be split into multiple files if possible. \ No newline at end of file +For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). diff --git a/frontend/README_deprecated.md b/frontend/README_deprecated.md deleted file mode 100644 index 79ac72db76..0000000000 --- a/frontend/README_deprecated.md +++ /dev/null @@ -1,128 +0,0 @@ -**This documentation has many deprecated patterns. Please follow the current [README](./README.md).** - -# Fleet Front-End - -**Note: Redux is deprecated.** - -The Fleet front-end is a Single Page Application using React and Redux. - -## Running the Fleet web app - -For details instruction on building and serving the Fleet web application -consult the [Contributing documentation](../docs/03-Contributing/README.md). - -## Directory Structure - -Component directories in the Fleet front-end application encapsulate the entire -component, including files for the component, helper functions, styles, and tests. The -typical directory structure for a component is as follows: - -**Note: `.jsx` and `.js` is deprecated.** -``` -|-- ComponentName -| |-- _styles.scss -| |-- ComponentName.jsx -| |-- ComponentName.tests.jsx // deprecated -| |-- helpers.js -| |-- helpers.tests.js -| |-- index.js -``` - -- `_styles.scss`: The component css styles -- `ComponentName.jsx`: The React component -- `ComponentName.tests.jsx`: The React component tests `Deprecated` -- `helpers.js`: Helper functions used by the component -- `helpers.tests.js`: Tests for the component's helper functions -- `index.js`: Exports the React component - - This file is helpful as it allows other components to import the component - by it's directory name. Without this file the component name would have to - be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). - -### [app_constants](./app_constants) - -The app_constants directory exports the constants used in the app. Examples -include the app's URL paths, settings, and http statuses. When building features -that require constants, the constants should be added here for accessibility -throughout the application. - -### [components](./components) - -The component directory contains the React components rendered by pages. They -are typically not connected to the redux state but receive props from their -parent components to render data and handle user interactions. - -### [interfaces](./interfaces) - -**Note: PropTypes is deprecated.** - -Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet -entity. This is designed to DRY up the code and increase re-usability. These -interfaces are imported into component files and implemented when defining the -component's PropTypes. - -### [fleet](./fleet) `Deprecated` - -The default export of the `fleet` directory is the API client. More info can be -found at the [API client documentation page](./fleet/README.md). - -### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) - -The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). -The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general app UI (header, sidebar) and render child components. -The child components rendered by the layout are typically page components. - -### [pages](./pages) - -Page components are React components typically rendered from the [router](./router). -These components are connected to redux state and are used to gather data from -redux and pass that data to child components (located in the [components -directory](./components). As -connected components, Pages are also used to dispatch actions. Actions -dispatched from Pages are intended to update redux state and oftentimes include -making a call to the Fleet API. - -### [redux](./redux) `Deprecated` - -The redux directory holds all of the application's redux middleware, actions, -and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. -More information about the redux configuration can be found at the [Redux -Documentation page](./redux/README.md) - -### [router](./router) - -The router directory is where the react router lives. The router decides which -component will render at a given URL. Components rendered from the router are -typically located in the [pages directory](./pages). The router directory also holds a `paths` -file which holds the application paths as string constants for reference -throughout the app. These paths are typically referenced from the [App -Constants](./app_constants) object. - -### [styles](./styles) - -The styles directory contains the general app style setup and variables. It -includes variables for the app color hex codes, fonts (families, weights and sizes), and padding. - -### [templates](./templates) - -The templates directory contains the HTML file that renders the React application via including the `bundle.js` -and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. - -### [test](./test) `Deprecated` - -The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. -More on test helpers, stubs, and request mocks [here](./test/README.md). - -### [utilities](./utilities) - -The utilities directory contains re-usable functions for use throughout the -application. The functions include helpers to convert an array of objects to -CSV, debounce functions to prevent multiple form submissions, format API errors, -etc. - -## Forms - -For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). - -## API Client - -For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). diff --git a/frontend/components/App/App.tsx b/frontend/components/App/App.tsx index 0f5bfd600c..6ba08768d8 100644 --- a/frontend/components/App/App.tsx +++ b/frontend/components/App/App.tsx @@ -26,15 +26,9 @@ interface IRootState { const App = ({ children }: IAppProps) => { const dispatch = useDispatch(); + const { setCurrentUser, setConfig } = useContext(AppContext); const user = useSelector((state: IRootState) => state.auth.user); const queryClient = new QueryClient(); - const { - setCurrentUser, - setConfig, - currentUser, - isGlobalObserver, - isOnlyObserver, - } = useContext(AppContext); useDeepEffect(() => { // on page refresh @@ -47,21 +41,9 @@ const App = ({ children }: IAppProps) => { dispatch(getConfig()) .then((config: IConfig) => setConfig(config)) .catch(() => false); - } - }, [user]); - - useDeepEffect(() => { - const canGetEnrollSecret = - currentUser && - typeof isGlobalObserver !== "undefined" && - !isGlobalObserver && - typeof isOnlyObserver !== "undefined" && - !isOnlyObserver; - - if (canGetEnrollSecret) { dispatch(getEnrollSecret()).catch(() => false); } - }, [currentUser, isGlobalObserver, isOnlyObserver]); + }, [user]); const wrapperStyles = classnames("wrapper"); return ( diff --git a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx index caca7075c4..e037aca8f3 100644 --- a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx +++ b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx @@ -11,6 +11,7 @@ import enrollSecretInterface from "interfaces/enroll_secret"; import EnrollSecretTable from "components/config/EnrollSecretTable"; import InputField from "components/forms/fields/InputField"; import OrgLogoIcon from "components/icons/OrgLogoIcon"; +import Slider from "components/forms/fields/Slider"; import validate from "components/forms/admin/AppConfigForm/validate"; import IconToolTip from "components/IconToolTip"; import InfoBanner from "components/InfoBanner/InfoBanner"; diff --git a/frontend/components/forms/queries/QueryForm/QueryForm.tsx b/frontend/components/forms/queries/QueryForm/QueryForm.tsx index 7bf58d4528..03c8959dde 100644 --- a/frontend/components/forms/queries/QueryForm/QueryForm.tsx +++ b/frontend/components/forms/queries/QueryForm/QueryForm.tsx @@ -40,9 +40,9 @@ interface IQueryFormProps { } interface IRenderProps { + nameText: string; + descText: string; queryValue: string; - nameText?: string; - descText?: string; queryError?: any; queryOnChange?: any; name?: IFormField; @@ -251,42 +251,6 @@ const QueryForm = ({ ); - const renderCreateForTeamMaintainer = ({ - queryValue, - queryOnChange, - queryError, - }: IRenderProps) => ( - <> -
-

New query

- {baseError &&
{baseError}
} - - {renderLiveQueryWarning()} -
- -
- - - ); - const renderForGlobalAdminOrMaintainer = ({ nameText, descText, @@ -304,14 +268,14 @@ const QueryForm = ({ name?.onChange(evt.target.value) } /> ) : ( -

New query

+

New query

)} {isEditMode && ( void; setConfig: (config: IConfig) => void; }; @@ -26,14 +26,14 @@ type InitialStateType = { const initialState = { currentUser: null, config: null, - isFreeTier: undefined, - isPremiumTier: undefined, - isGlobalAdmin: undefined, - isGlobalMaintainer: undefined, - isGlobalObserver: undefined, - isOnGlobalTeam: undefined, - isAnyTeamMaintainer: undefined, - isOnlyObserver: undefined, + isFreeTier: false, + isPremiumTier: false, + isGlobalAdmin: false, + isGlobalMaintainer: false, + isGlobalObserver: false, + isOnGlobalTeam: false, + isAnyTeamMaintainer: false, + isOnlyObserver: false, setCurrentUser: () => null, setConfig: () => null, }; diff --git a/frontend/fleet/endpoints.ts b/frontend/fleet/endpoints.ts index 6b58583227..13267fbd1c 100644 --- a/frontend/fleet/endpoints.ts +++ b/frontend/fleet/endpoints.ts @@ -41,7 +41,7 @@ export default { STATUS_LABEL_COUNTS: "/v1/fleet/host_summary", TARGETS: "/v1/fleet/targets", TEAM_SCHEDULE: (id: number): string => { - return `/v1/fleet/teams/${id}/schedule`; + return `/v1/fleet/team/${id}/schedule`; }, USERS: "/v1/fleet/users", USERS_ADMIN: "/v1/fleet/users/admin", diff --git a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx index 499d103055..25c15dc3b8 100644 --- a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx +++ b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx @@ -41,7 +41,7 @@ const CreateTeamModal = (props: ICreateTeamModalProps): JSX.Element => { return ( -
+ {

- - ); -}; +}: ITargetPillSelectorProps): JSX.Element => ( + +); const SelectTargets = ({ baseClass, @@ -241,7 +228,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsLoading) { return (
-

Select targets

+

Select Targets

@@ -252,7 +239,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsError) { return (
-

Select targets

+

Select Targets

@@ -277,7 +264,7 @@ const SelectTargets = ({ return (
-

Select targets

+

Select Targets

{allHostsLabels && allHostsLabels.length > 0 && diff --git a/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx b/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx index 84dc6cbb20..ff881bc279 100644 --- a/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx +++ b/frontend/pages/schedule/ManageSchedulePage/ManageSchedulePage.tsx @@ -25,7 +25,6 @@ import paths from "router/paths"; import Button from "components/buttons/Button"; // @ts-ignore import Dropdown from "components/forms/fields/Dropdown"; -import IconToolTip from "components/IconToolTip"; import TableDataError from "components/TableDataError"; import ScheduleListWrapper from "./components/ScheduleListWrapper"; import ScheduleEditorModal from "./components/ScheduleEditorModal"; @@ -34,10 +33,8 @@ import RemoveScheduledQueryModal from "./components/RemoveScheduledQueryModal"; const baseClass = "manage-schedule-page"; const renderTable = ( - onRemoveScheduledQueryClick: (selectIds: number[]) => void, - onEditScheduledQueryClick: ( - selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery - ) => void, + onRemoveScheduledQueryClick: React.MouseEventHandler, + onEditScheduledQueryClick: React.MouseEventHandler, allScheduledQueriesList: IGlobalScheduledQuery[] | ITeamScheduledQuery[], allScheduledQueriesError: { name: string; reason: string }[], toggleScheduleEditorModal: () => void, @@ -58,26 +55,6 @@ const renderTable = ( ); }; -const renderAllTeamsTable = ( - teamId: number, - allTeamsScheduledQueriesList: IGlobalScheduledQuery[], - allTeamsScheduledQueriesError: { name: string; reason: string }[] -): JSX.Element => { - if (Object.keys(allTeamsScheduledQueriesError).length > 0) { - return ; - } - - return ( -
- -
- ); -}; - interface ITeamSchedulesPageProps { params: { team_id: string; @@ -162,18 +139,6 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { const allScheduledQueriesList = Object.values(allScheduledQueries.data); const allScheduledQueriesError = allScheduledQueries.errors; - const allTeamsScheduledQueries = useSelector((state: IRootState) => { - return state.entities.global_scheduled_queries; - }); - - const allTeamsScheduledQueriesList = Object.values( - allTeamsScheduledQueries.data - ); - const allTeamsScheduledQueriesError = allTeamsScheduledQueries.errors; - - const inheritedQueryOrQueries = - allTeamsScheduledQueriesList.length === 1 ? "query" : "queries"; - const allTeams = useSelector((state: IRootState) => state.entities.teams); const allTeamsList = Object.values(allTeams.data); @@ -183,7 +148,7 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { const teamOptions: ITeamOptions[] = [ { disabled: false, - label: "All teams", + label: "Global", value: "global", }, ]; @@ -198,9 +163,6 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { return teamOptions; }; - const [showInheritedQueries, setShowInheritedQueries] = useState( - false - ); const [showScheduleEditorModal, setShowScheduleEditorModal] = useState(false); const [ showRemoveScheduledQueryModal, @@ -213,10 +175,6 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { IGlobalScheduledQuery | ITeamScheduledQuery >(); - const toggleInheritedQueries = () => { - setShowInheritedQueries(!showInheritedQueries); - }; - const toggleScheduleEditorModal = useCallback(() => { setSelectedScheduledQuery(undefined); // create modal renders setShowScheduleEditorModal(!showScheduleEditorModal); @@ -226,16 +184,12 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { setShowRemoveScheduledQueryModal(!showRemoveScheduledQueryModal); }, [showRemoveScheduledQueryModal, setShowRemoveScheduledQueryModal]); - const onRemoveScheduledQueryClick = ( - selectedTableQueryIds: number[] - ): void => { + const onRemoveScheduledQueryClick = (selectedTableQueryIds: any): void => { toggleRemoveScheduledQueryModal(); setSelectedQueryIds(selectedTableQueryIds); }; - const onEditScheduledQueryClick = ( - selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery - ): void => { + const onEditScheduledQueryClick = (selectedQuery: any): void => { toggleScheduleEditorModal(); setSelectedScheduledQuery(selectedQuery); // edit modal renders }; @@ -425,39 +379,6 @@ const ManageSchedulePage = (props: ITeamSchedulesPageProps): JSX.Element => { teamId )}
- {/* must use ternary for NaN */} - {teamId && allTeamsScheduledQueriesList.length > 0 ? ( - <> - - - -
-

Queries from the “All teams”
schedule run on this team’s hosts.

\ - " - } - /> -
- - ) : null} - {showInheritedQueries && - renderAllTeamsTable( - teamId, - allTeamsScheduledQueriesList, - allTeamsScheduledQueriesError - )} {showScheduleEditorModal && ( void; - onEditScheduledQueryClick?: ( - selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery - ) => void; + onRemoveScheduledQueryClick: any; + onEditScheduledQueryClick: any; allScheduledQueriesList: IGlobalScheduledQuery[] | ITeamScheduledQuery[]; - toggleScheduleEditorModal?: () => void; + toggleScheduleEditorModal: () => void; teamId: number; - inheritedQueries?: boolean; } interface IRootState { entities: { @@ -54,7 +47,6 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { toggleScheduleEditorModal, onEditScheduledQueryClick, teamId, - inheritedQueries, } = props; const dispatch = useDispatch(); const { MANAGE_PACKS } = paths; @@ -100,14 +92,10 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { ): void => { switch (action) { case "edit": - if (onEditScheduledQueryClick) { - onEditScheduledQueryClick(global_scheduled_query); - } + onEditScheduledQueryClick(global_scheduled_query); break; default: - if (onRemoveScheduledQueryClick) { - onRemoveScheduledQueryClick([global_scheduled_query.id]); - } + onRemoveScheduledQueryClick([global_scheduled_query.id]); break; } }; @@ -135,33 +123,6 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { [dispatch] ); - const loadingInheritedQueriesTableData = useSelector((state: IRootState) => { - return state.entities.global_scheduled_queries.isLoading; - }); - - if (inheritedQueries) { - const inheritedQueriesTableHeaders = generateInheritedQueriesTableHeaders(); - - return ( -
- -
- ); - } - return (
{ - return [ - { - title: "Query", - Header: "Query", - disableSortBy: true, - accessor: "query_name", - Cell: (cellProps: ICellProps): JSX.Element => ( - - ), - }, - { - title: "Frequency", - Header: "Frequency", - disableSortBy: true, - accessor: "interval", - Cell: (cellProps: ICellProps): JSX.Element => ( - - ), - }, - ]; -}; - const generateActionDropdownOptions = (): IDropdownOption[] => { const dropdownOptions = [ { @@ -185,8 +162,4 @@ const generateDataSet = ( return [...enhanceAllScheduledQueryData(all_scheduled_queries, teamId)]; }; -export { - generateInheritedQueriesTableHeaders, - generateTableHeaders, - generateDataSet, -}; +export { generateTableHeaders, generateDataSet }; diff --git a/frontend/styles/global/_global.scss b/frontend/styles/global/_global.scss index 1e335648f7..586d4bb247 100644 --- a/frontend/styles/global/_global.scss +++ b/frontend/styles/global/_global.scss @@ -55,12 +55,11 @@ a { border-radius: 3px; background-color: $core-white; border: solid 1px $core-white; - min-width: 798px; + min-width: 910px; } .has-sidebar { display: flex; - flex-grow: 1; & > *:first-child { flex-grow: 1; @@ -122,4 +121,4 @@ hr { margin-bottom: $pad-xlarge; border: none; border-bottom: 1px solid $ui-fleet-blue-15; -} +} \ No newline at end of file diff --git a/frontend/test/mocks/team_scheduled_query_mocks.js b/frontend/test/mocks/team_scheduled_query_mocks.js index bfbe3e7d1a..30a5533a34 100644 --- a/frontend/test/mocks/team_scheduled_query_mocks.js +++ b/frontend/test/mocks/team_scheduled_query_mocks.js @@ -16,7 +16,7 @@ export default { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/teams/${params.team_id}/schedule`, + endpoint: `/api/v1/fleet/team/${params.team_id}/schedule`, method: "post", params, response: { scheduled: teamScheduledQueryStub }, @@ -28,7 +28,7 @@ export default { valid: (bearerToken, teamScheduledQuery) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "delete", response: {}, }); @@ -38,7 +38,7 @@ export default { valid: (bearerToken, teamID) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/teams/${teamID}/schedule`, + endpoint: `/api/v1/fleet/team/${teamID}/schedule`, method: "get", response: { scheduled: [teamScheduledQueryStub] }, }); @@ -48,7 +48,7 @@ export default { valid: (bearerToken, teamScheduledQuery, params) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "patch", params, response: { scheduled: { ...teamScheduledQuery, ...params } }, diff --git a/go.mod b/go.mod index 6b0d2d7ea9..b7b7271608 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/gomodule/redigo v1.8.5 github.com/google/go-cmp v0.5.6 github.com/google/go-github/v37 v37.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.1.2 github.com/goreleaser/nfpm/v2 v2.2.2 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index c045b59475..dd625a9fa6 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,6 @@ github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/handbook/README.md b/handbook/README.md index d33d6ed6f6..3ea51a385e 100644 --- a/handbook/README.md +++ b/handbook/README.md @@ -98,7 +98,7 @@ If the action fails, please complete the following steps: ##### Browser compatibility checking -A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/01-Using-Fleet/12-Supported-browsers.md). +A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/1-Using-Fleet/12-Supported-browsers.md). - We use [BrowserStack](https://www.browserstack.com/users/sign_in) (logins can be found in [1Password](https://start.1password.com/open/i?a=N3F7LHAKQ5G3JPFPX234EC4ZDQ&v=3ycqkai6naxhqsylmsos6vairu&i=nwnxrrbpcwkuzaazh3rywzoh6e&h=fleetdevicemanagement.1password.com)) for our cross-browser checks. - Check for issues against the latest version of Google Chrome (macOS). We use this as our baseline for quality assurance. diff --git a/handbook/growth.md b/handbook/growth.md deleted file mode 100644 index 694a6b5e5e..0000000000 --- a/handbook/growth.md +++ /dev/null @@ -1,8 +0,0 @@ -# Growth handbook - -Welcome to the Fleet growth handbook. - -TODO - - - diff --git a/handbook/manual-qa.md b/handbook/manual-qa.md index 3f84eb2412..6af722b196 100644 --- a/handbook/manual-qa.md +++ b/handbook/manual-qa.md @@ -2,31 +2,63 @@ This living document outlines the manual quality assurance process conducted to ensure each release of Fleet meets organization standards. -All steps should be conducted during each QA pass. All steps are possible with `fleetctrl preview`. +All steps should be conducted during each QA pass. As new features are added to Fleet, new steps and flows will be added. ## Collecting bugs -The goal of manual QA is to catch unexpected behavior prior to release. All Manual QA steps should be possible using `fleetctl preview`. Please refer to [docs/03-Contributing/02-Testing.md](https://github.com/fleetdm/fleet/blob/main/docs/03-Contributing/02-Testing.md) for flows that cannot be completed using `fleetctl preview`. +The goal of manual QA is to catch unexpected behavior prior to release. Please start the manual QA process by creating a blank GitHub issue. As you complete each of the flows, record a list of the bugs you encounter in this new issue. Each item in this list should contain one sentence describing the bug and a screenshot if the item is a frontend bug. ## Fleet UI -For all following flows, please refer to the [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) to ensure that actions are limited to the appropriate user type. Any users with access beyond what this document lists as availale should be considered a bug and reported for either documentation updates or investigation. +### Clear your local MySQL database + +Before you fire up your local Fleet server, wipe your local MySQL database by running the following command: + +``` +docker volume rm fleet_mysql-persistent-volume +``` + +If you receive an error that says "No such volume," double check that the MySQL volume doesn't have a different name by running this command: + +``` +docker volume ls +``` + +### Start your development server + +Next, fire up your local Fleet server. Check out [this Loom video](https://www.loom.com/share/e7439f058eb44c45af872abe8f8de4a1) for instructions on starting up your local development environment. ### Set up flow -Successfully set up `fleetctl preview` using the preview steps outlined [here](https://fleetdm.com/get-started) +Successfully set up Fleet. ### Login and logout flow Successfully logout and then login to your local Fleet. +### Enroll host flow + +Enroll your local machine to Fleet. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/orbit#orbit-osquery) for instructions on generating and installing an Orbit package. + +### Host page + +To populate the Fleet UI with more than just one host you'll need to use the [fleetdm/osquery-perf tool](https://github.com/fleetdm/osquery-perf/tree/629a7efb6097f9108f706ccd45828793ff73cf9c). + +First, clone the fleetdm/osquery perf repo and then run the following commands from the top level of the cloned directory: + +``` +go run agent.go --host_count 200 --enroll_secret +``` + +After about 10 seconds, the Fleet UI should be populated with 200 simulated hosts. + ### Host details page -Select a host from the "Hosts" table as a global user with the Maintainer role. You may create a user with a fake email for this purpose. +Select a host from the "Hosts" table as a global user with the Maintainer role. You should be able to see and select the "Delete" button on this host's **Host details** page. @@ -38,9 +70,7 @@ You should be able to see and select the "Query" button on this host's **Host de Create a new label by selecting "Add a new label" on the Hosts page. Make sure it correctly filters the host on the hosts page. -Edit this label. Confirm users can only edit the "Name" and "Description" fields for a label. Users cannot edit the "Query" field because label queries are immutable. - -Delete this label. +Edit this label and then delete this label. ### Query flow @@ -56,12 +86,88 @@ Edit this query and then delete this query. `Flow is covered by e2e testing` -Create a new pack (under Schedule/advanced). +Create a new pack. Add a query as a saved query to the pack. Remove this query. Delete the pack. +### Organization settings flow + +As an admin user, select the "Settings" tab in the top navigation and then select "Organization settings". + +Follow [the instructions outlined in the Testing documentation](../docs/3-Contributing/2-Testing.md#email) to set up a local SMTP server. + +Successfully edit your organization's name in Fleet. + +### Manage users flow + +Invite a new user. To be able to invite users, you must have your local SMTP server configured. Instructions for setting up a local SMTP server are outlined in [the Testing documentation](../docs/3-Contributing/2-Testing.md#email) + +Logout of your current admin user and accept the invitation for the newly invited user. With your local SMTP server configured, head to https://localhost:8025 to view and select the invitation link. + +### Agent options flow + +Head to the global agent options page and set the `distributed_iterval` field to `5`. + +Refresh the page to confirm that the agent options have been updated. ### My account flow -Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. Please do this with an extra user created for this purpose to maintain accessibility of `fleetctl preview` admin user. +Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. +## `fleetctl` CLI + +### Set up flow + +Successfully set up Fleet by running the `fleetctl setup` command. + +You may have to wipe your local MySQL database in order to successfully set up Fleet. Check out the [Clear your local MySQL database](#clear-your-local-mysql-database) section of this document for instructions. + +### Login and logout flow + +Successfully login by running the `fleetctl login` command. + +Successfully logout by running the `fleetctl logout` command. Then, log in again. + +### Hosts + +Run the `fleetctl get hosts` command. + +You should see your local machine returned. If your host isn't showing up, you may have to reenroll your local machine. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/fleet/blob/main/orbit/README.md) for instructions on generating and installing an Orbit package. + +### Query flow + +Apply the standard query library by running the following command: + +`fleetctl apply -f docs/1-Using-Fleet/standard-query-library/standard-query-library.yml` + +Make sure all queries were successfully added by running the following command: + +`fleetctl get queries` + +Run the "Get the version of the resident operating system" query against your local machine by running the following command: + +`fleetctl query --hosts --query-name "Get the version of the resident operating system"` + +### Pack flow + +Apply a pack by running the following commands: + +`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml` + +`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml` + +Make sure the pack was successfully added by running the following command: + +`fleetctl get packs` + +### Organization settings flow + +Apply organization settings by running the following command: + +`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml` + +### Manage users flow + +Create a new user by running the `fleetctl user create` command. + +Logout of your current user and log in with the newly created user. diff --git a/handbook/release-process.md b/handbook/release-process.md index 9e7d01f262..52b5c6aaec 100644 --- a/handbook/release-process.md +++ b/handbook/release-process.md @@ -28,7 +28,7 @@ Check out the [Fleet 4.1.0 blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa **More improvements** - Includes each additional feature's name, availability (Free v. Premium), and 1-2 sentences that answer the 'why should the user care?' questions. -**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/08-Updating-Fleet.md +**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/8-Updating-Fleet.md ### Manual QA @@ -38,4 +38,4 @@ Documentation on conducting the manual QA pass can be found [here](./manual-qa.m ## Release day -Documentation on completing the release process can be found [here](../docs/03-Contributing/05-Releasing-Fleet.md). +Documentation on completing the release process can be found [here](../docs/3-Contributing/5-Releasing-Fleet.md). diff --git a/handbook/support-process.md b/handbook/support-process.md index 58d09e964e..cbacf8587c 100644 --- a/handbook/support-process.md +++ b/handbook/support-process.md @@ -90,6 +90,6 @@ There are four sources that the individual on-call should monitor for activity: There are several locations in Fleet's public and internal documentation that can be helpful when answering questions raised by the community: -1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/01-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/02-Deploying/FAQ.md), and [Contributing FAQ](../docs/03-Contributing/FAQ.md). +1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/1-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/2-Deploying/FAQ.md), and [Contributing FAQ](../docs/3-Contributing/FAQ.md). 2. The [Internal FAQ](https://docs.google.com/document/d/1I6pJ3vz0EE-qE13VmpE2G3gd5zA1m3bb_u8Q2G3Gmp0/edit#heading=h.ltavvjy511qv) document. diff --git a/handbook/why-fleet.md b/handbook/why-fleet.md new file mode 100644 index 0000000000..54ad90368f --- /dev/null +++ b/handbook/why-fleet.md @@ -0,0 +1,43 @@ +## Why Fleet? + +We believe device management should be safe, easy, and transparent to end users. + +### The premier osquery fleet manager. +Fleet makes it easier to query and track your servers, containers, and laptops. It extends osquery to answer questions about multiple devices at the same time and provides log streams that enable automated threat detection. You can run it on your own hardware or deploy it in any cloud. + +#### Realtime visibility +- Share with any team (ops, security, helpdesk, etc) +- Unmetered access to user seats +- Intuitive user interface +- Verify remediations were actually successful +- Make compliance goals highly visible +- Track progress towards compliance goals together +- Auto-ticket any compliance regressions +- Reliable, efficient source of truth + +#### Get flexible +- Receive data from any device +- Send data anywhere it needs to go +- Express and track custom compliance goals +- Integrate existing business processes +- Play nice with internal tools and requirements +- Self-managed (cloud first, but can still be within your VPC) +- Complete control (no need to rely on 3rd party vendors, reduced risk of getting hacked) +- Deploy anywhere + +#### Developer productivity +- Programmable (REST API, ±gRPC) +- Command line tool (`fleetctl`) +- "Infrastructure as code"-ready (YAML) +- All UI functionality accessible via APIs +- Focus on value, not boilerplate +- Skip reinventing the wheel +- Ad hoc audits (live queries answer in seconds) +- 100% source code available + +#### Unlock efficiencies +- Designed for enterprise (from day 1) +- Built by osquery creators (Zach Wasserman + Mike Arpaia) +- Battle-tested in production (3+ years) +- Learn from peers' mistakes & best practices +- Widespread adoption in F1000 diff --git a/orbit/README.md b/orbit/README.md index 3b3d1da3be..18cd740e52 100644 --- a/orbit/README.md +++ b/orbit/README.md @@ -192,7 +192,7 @@ Yes! Orbit is licensed under an MIT license and all uses are encouraged. ### How does orbit update osquery? And how do the stable and edge channels get triggered to update osquery on a self hosted Fleet instance? -Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/04-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). +Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/4-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). ## Community diff --git a/package.json b/package.json index 340b3a1b8b..cfa86f317f 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,7 @@ "babel-jest": "^23.4.2", "babel-loader": "^8.2.1", "css-loader": "0.28.11", - "cypress": "8.4.0", + "cypress": "^6.9.1", "cypress-wait-until": "^1.7.1", "enzyme": "^3.11.0", "enzyme-adapter-react-16": "^1.15.6", diff --git a/server/authz/authz.go b/server/authz/authz.go index 35d02b96ac..91e8895e4b 100644 --- a/server/authz/authz.go +++ b/server/authz/authz.go @@ -119,41 +119,6 @@ func (a *Authorizer) Authorize(ctx context.Context, object, action interface{}) return nil } -func (a *Authorizer) TeamAuthorize(ctx context.Context, teamID uint, action string) error { - subject := UserFromContext(ctx) - if subject == nil { - return ForbiddenWithInternal("nil subject always forbidden", subject, nil, action) - } - - // global admins and maintainers are authorized to work with teams - if subject.GlobalRole != nil { - switch *subject.GlobalRole { - case fleet.RoleAdmin, fleet.RoleMaintainer: - return nil - case fleet.RoleObserver: - if action == fleet.ActionRead { - return nil - } - } - } - - for _, team := range subject.Teams { - if teamID == team.ID { - switch action { - case fleet.ActionWrite: - if team.Role == fleet.RoleMaintainer { - return nil - } - return ForbiddenWithInternal("team observer cannot write", subject, nil, action) - default: - return nil - } - } - } - - return ForbiddenWithInternal("not a member of the team", subject, nil, action) -} - // AuthzTyper is the interface that may be implemented to get a `type` // property added during marshaling for authorization. Any struct that will be // used as a subject or object in authorization should implement this interface. diff --git a/server/authz/policy.rego b/server/authz/policy.rego index ff9a5ecf8a..9ff87d8780 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -15,7 +15,6 @@ list := "list" write := "write" write_role := "write_role" run := "run" -run_new := "run_new" # Roles admin := "admin" @@ -266,16 +265,6 @@ allow { subject.global_role == maintainer action = run } -allow { - object.type == "query" - subject.global_role == admin - action = run_new -} -allow { - object.type == "query" - subject.global_role == maintainer - action = run_new -} # Team maintainer running a non-observers_can_run query must have the targets # filtered to only teams that they maintain allow { @@ -285,22 +274,6 @@ allow { action == run } -# Team maintainer can run a new query -allow { - object.type == "query" - # If role is maintainer on any team - team_role(subject, subject.teams[_].id) == maintainer - action == run_new -} - -# Team admin can run a new query -allow { - object.type == "query" - # If role is maintainer on any team - team_role(subject, subject.teams[_].id) == admin - action == run_new -} - # (Team) observers can run only if observers_can_run allow { object.type == "query" @@ -369,41 +342,11 @@ allow { } allow { - is_null(object.team_id) object.type == "policy" subject.global_role == maintainer action == [read, write][_] } -allow { - object.type == "policy" - subject.global_role == maintainer - action == [read][_] -} - -# Global Observer users can read policies -allow { - object.type == "policy" - subject.global_role == observer - action == [read][_] -} - -# Team Maintainers can read and write policies -allow { - not is_null(object.team_id) - object.type == "policy" - team_role(subject, subject.teams[_].id) == maintainer - action == [read, write][_] -} - -# Team Observer can read policies -allow { - not is_null(object.team_id) - object.type == "policy" - team_role(subject, subject.teams[_].id) == observer - action == [read][_] -} - ## # Software ## diff --git a/server/config/config.go b/server/config/config.go index d81baa58ba..e687581014 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -37,15 +37,13 @@ type MysqlConfig struct { // RedisConfig defines configs related to Redis type RedisConfig struct { - Address string - Password string - Database int - UseTLS bool `yaml:"use_tls"` - DuplicateResults bool `yaml:"duplicate_results"` - ConnectTimeout time.Duration `yaml:"connect_timeout"` - KeepAlive time.Duration `yaml:"keep_alive"` - ConnectRetryAttempts int `yaml:"connect_retry_attempts"` - ClusterFollowRedirections bool `yaml:"cluster_follow_redirections"` + Address string + Password string + Database int + UseTLS bool `yaml:"use_tls"` + DuplicateResults bool `yaml:"duplicate_results"` + ConnectTimeout time.Duration `yaml:"connect_timeout"` + KeepAlive time.Duration `yaml:"keep_alive"` } const ( @@ -91,12 +89,10 @@ type OsqueryConfig struct { StatusLogPlugin string `yaml:"status_log_plugin"` ResultLogPlugin string `yaml:"result_log_plugin"` LabelUpdateInterval time.Duration `yaml:"label_update_interval"` - PolicyUpdateInterval time.Duration `yaml:"policy_update_interval"` DetailUpdateInterval time.Duration `yaml:"detail_update_interval"` StatusLogFile string `yaml:"status_log_file"` ResultLogFile string `yaml:"result_log_file"` EnableLogRotation bool `yaml:"enable_log_rotation"` - MaxJitterPercent int `yaml:"max_jitter_percent"` } // LoggingConfig defines configs related to logging @@ -247,8 +243,6 @@ func (man Manager) addConfigs() { man.addConfigBool("redis.duplicate_results", false, "Duplicate Live Query results to another Redis channel") man.addConfigDuration("redis.connect_timeout", 5*time.Second, "Timeout at connection time") man.addConfigDuration("redis.keep_alive", 10*time.Second, "Interval between keep alive probes") - man.addConfigInt("redis.connect_retry_attempts", 0, "Number of attempts to retry a failed connection") - man.addConfigBool("redis.cluster_follow_redirections", false, "Automatically follow Redis Cluster redirections") // Server man.addConfigString("server.address", "0.0.0.0:8080", @@ -300,8 +294,6 @@ func (man Manager) addConfigs() { "Log plugin to use for result logs") man.addConfigDuration("osquery.label_update_interval", 1*time.Hour, "Interval to update host label membership (i.e. 1h)") - man.addConfigDuration("osquery.policy_update_interval", 1*time.Hour, - "Interval to update host policy membership (i.e. 1h)") man.addConfigDuration("osquery.detail_update_interval", 1*time.Hour, "Interval to update host details (i.e. 1h)") man.addConfigString("osquery.status_log_file", "", @@ -310,8 +302,6 @@ func (man Manager) addConfigs() { "(DEPRECATED: Use filesystem.result_log_file) Path for osqueryd result logs") man.addConfigBool("osquery.enable_log_rotation", false, "(DEPRECATED: Use filesystem.enable_log_rotation) Enable automatic rotation for osquery log files") - man.addConfigInt("osquery.max_jitter_percent", 10, - "Maximum percentage of the interval to add as jitter") // Logging man.addConfigBool("logging.debug", false, @@ -427,15 +417,13 @@ func (man Manager) LoadConfig() FleetConfig { Mysql: loadMysqlConfig("mysql"), MysqlReadReplica: loadMysqlConfig("mysql_read_replica"), Redis: RedisConfig{ - Address: man.getConfigString("redis.address"), - Password: man.getConfigString("redis.password"), - Database: man.getConfigInt("redis.database"), - UseTLS: man.getConfigBool("redis.use_tls"), - DuplicateResults: man.getConfigBool("redis.duplicate_results"), - ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), - KeepAlive: man.getConfigDuration("redis.keep_alive"), - ConnectRetryAttempts: man.getConfigInt("redis.connect_retry_attempts"), - ClusterFollowRedirections: man.getConfigBool("redis.cluster_follow_redirections"), + Address: man.getConfigString("redis.address"), + Password: man.getConfigString("redis.password"), + Database: man.getConfigInt("redis.database"), + UseTLS: man.getConfigBool("redis.use_tls"), + DuplicateResults: man.getConfigBool("redis.duplicate_results"), + ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), + KeepAlive: man.getConfigDuration("redis.keep_alive"), }, Server: ServerConfig{ Address: man.getConfigString("server.address"), @@ -467,10 +455,8 @@ func (man Manager) LoadConfig() FleetConfig { StatusLogFile: man.getConfigString("osquery.status_log_file"), ResultLogFile: man.getConfigString("osquery.result_log_file"), LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), - PolicyUpdateInterval: man.getConfigDuration("osquery.policy_update_interval"), DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), EnableLogRotation: man.getConfigBool("osquery.enable_log_rotation"), - MaxJitterPercent: man.getConfigInt("osquery.max_jitter_percent"), }, Logging: LoggingConfig{ Debug: man.getConfigBool("logging.debug"), @@ -756,9 +742,7 @@ func TestConfig() FleetConfig { StatusLogPlugin: "filesystem", ResultLogPlugin: "filesystem", LabelUpdateInterval: 1 * time.Hour, - PolicyUpdateInterval: 1 * time.Hour, DetailUpdateInterval: 1 * time.Hour, - MaxJitterPercent: 0, }, Logging: LoggingConfig{ Debug: true, diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index af97ca730f..5dbdc7b6fc 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -10,25 +10,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestActivity(t *testing.T) { +func TestActivityUsernameChange(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"UsernameChange", testActivityUsernameChange}, - {"New", testActivityNew}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testActivityUsernameChange(t *testing.T, ds *Datastore) { u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -67,7 +52,10 @@ func testActivityUsernameChange(t *testing.T, ds *Datastore) { assert.Nil(t, activities[0].ActorGravatar) } -func testActivityNew(t *testing.T, ds *Datastore) { +func TestNewActivity(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -98,12 +86,4 @@ func testActivityNew(t *testing.T, ds *Datastore) { assert.Len(t, activities, 1) assert.Equal(t, "fullname", activities[0].ActorFullName) assert.Equal(t, "test2", activities[0].Type) - - opt = fleet.ListOptions{ - Page: 0, - PerPage: 10, - } - activities, err = ds.ListActivities(context.Background(), opt) - require.NoError(t, err) - assert.Len(t, activities, 2) } diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index 8959912dad..58b0e3906d 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -14,29 +14,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestAppConfig(t *testing.T) { +func TestOrgInfo(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"OrgInfo", testAppConfigOrgInfo}, - {"AdditionalQueries", testAppConfigAdditionalQueries}, - {"EnrollSecrets", testAppConfigEnrollSecrets}, - {"EnrollSecretsCaseSensitive", testAppConfigEnrollSecretsCaseSensitive}, - {"EnrollSecretRoundtrip", testAppConfigEnrollSecretRoundtrip}, - {"EnrollSecretUniqueness", testAppConfigEnrollSecretUniqueness}, - {"Defaults", testAppConfigDefaults}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - c.fn(t, ds) - }) - } -} - -func testAppConfigOrgInfo(t *testing.T, ds *Datastore) { info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ OrgName: "Test", @@ -113,7 +94,10 @@ func testAppConfigOrgInfo(t *testing.T, ds *Datastore) { assert.False(t, verify.SSOEnabled) } -func testAppConfigAdditionalQueries(t *testing.T, ds *Datastore) { +func TestAdditionalQueries(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + additional := ptr.RawMessage(json.RawMessage("not valid json")) info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ @@ -139,8 +123,9 @@ func testAppConfigAdditionalQueries(t *testing.T, ds *Datastore) { assert.JSONEq(t, `{"foo":"bar"}`, string(rawJson)) } -func testAppConfigEnrollSecrets(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestEnrollSecrets(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -194,8 +179,9 @@ func testAppConfigEnrollSecrets(t *testing.T, ds *Datastore) { assert.Equal(t, (*uint)(nil), secret.TeamID) } -func testAppConfigEnrollSecretsCaseSensitive(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestEnrollSecretsCaseSensitive(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() err := ds.ApplyEnrollSecrets( context.Background(), @@ -212,8 +198,9 @@ func testAppConfigEnrollSecretsCaseSensitive(t *testing.T, ds *Datastore) { assert.Error(t, err, "enroll secret with different case should not verify") } -func testAppConfigEnrollSecretRoundtrip(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestEnrollSecretRoundtrip(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -253,8 +240,9 @@ func testAppConfigEnrollSecretRoundtrip(t *testing.T, ds *Datastore) { } -func testAppConfigEnrollSecretUniqueness(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestEnrollSecretUniqueness(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -270,7 +258,10 @@ func testAppConfigEnrollSecretUniqueness(t *testing.T, ds *Datastore) { require.Error(t, err) } -func testAppConfigDefaults(t *testing.T, ds *Datastore) { +func TestAppConfigDefaults(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + insertAppConfigQuery := `INSERT INTO app_config_json(json_value) VALUES(?) ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)` _, err := ds.writer.Exec(insertAppConfigQuery, `{}`) require.NoError(t, err) diff --git a/server/datastore/mysql/campaigns_test.go b/server/datastore/mysql/campaigns_test.go index 87d9316b6a..245795981f 100644 --- a/server/datastore/mysql/campaigns_test.go +++ b/server/datastore/mysql/campaigns_test.go @@ -12,30 +12,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestCampaigns(t *testing.T) { - ds := CreateMySQLDS(t) - - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"DistributedQuery", testCampaignsDistributedQuery}, - {"CleanupDistributedQuery", testCampaignsCleanupDistributedQuery}, - {"SaveDistributedQuery", testCampaignsSaveDistributedQuery}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - - c.fn(t, ds) - }) - } +func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { + targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) + require.Nil(t, err) + assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) + assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) + assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) } -func testCampaignsDistributedQuery(t *testing.T, ds *Datastore) { +func TestDistributedQueryCampaign(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) + mockClock := clock.NewMockClock() + query := test.NewQuery(t, ds, "test", "select * from time", user.ID, false) + campaign := test.NewCampaign(t, ds, query.ID, fleet.QueryRunning, mockClock.Now()) { @@ -77,9 +71,13 @@ func testCampaignsDistributedQuery(t *testing.T, ds *Datastore) { test.AddHostToCampaign(t, ds, campaign.ID, h3.ID) checkTargets(t, ds, campaign.ID, fleet.HostTargets{HostIDs: []uint{h1.ID, h2.ID, h3.ID}, LabelIDs: []uint{l1.ID, l2.ID}}) + } -func testCampaignsCleanupDistributedQuery(t *testing.T, ds *Datastore) { +func TestCleanupDistributedQueryCampaigns(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -151,9 +149,13 @@ func testCampaignsCleanupDistributedQuery(t *testing.T, ds *Datastore) { assert.Equal(t, c2.QueryID, retrieved.QueryID) assert.Equal(t, fleet.QueryComplete, retrieved.Status) } + } -func testCampaignsSaveDistributedQuery(t *testing.T, ds *Datastore) { +func TestSaveDistributedQueryCampaign(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, t.Name(), t.Name()+"zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -172,11 +174,3 @@ func testCampaignsSaveDistributedQuery(t *testing.T, ds *Datastore) { require.NoError(t, err) require.Equal(t, fleet.QueryComplete, gotC.Status) } - -func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { - targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) - require.Nil(t, err) - assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) - assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) - assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) -} diff --git a/server/datastore/mysql/carves_test.go b/server/datastore/mysql/carves_test.go index 2b50ee4298..22f84f772d 100644 --- a/server/datastore/mysql/carves_test.go +++ b/server/datastore/mysql/carves_test.go @@ -14,28 +14,10 @@ import ( var mockCreatedAt = time.Now().UTC().Truncate(time.Second) -func TestCarves(t *testing.T) { +func TestCarveMetadata(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Metadata", testCarvesMetadata}, - {"Blocks", testCarvesBlocks}, - {"Cleanup", testCarvesCleanup}, - {"List", testCarvesList}, - {"Update", testCarvesUpdate}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testCarvesMetadata(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -93,7 +75,10 @@ func testCarvesMetadata(t *testing.T, ds *Datastore) { assert.Equal(t, expectedCarve, carve) } -func testCarvesBlocks(t *testing.T, ds *Datastore) { +func TestCarveBlocks(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -131,9 +116,13 @@ func testCarvesBlocks(t *testing.T, ds *Datastore) { require.NoError(t, err, "get block %d %v", i, expectedBlocks[i]) assert.Equal(t, expectedBlocks[i], data) } + } -func testCarvesCleanup(t *testing.T, ds *Datastore) { +func TestCarveCleanupCarves(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -185,7 +174,10 @@ func testCarvesCleanup(t *testing.T, ds *Datastore) { assert.True(t, carve.Expired) } -func testCarvesList(t *testing.T, ds *Datastore) { +func TestCarveListCarves(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -243,7 +235,10 @@ func testCarvesList(t *testing.T, ds *Datastore) { assert.Len(t, carves, 2) } -func testCarvesUpdate(t *testing.T, ds *Datastore) { +func TestCarveUpdateCarve(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) actualCount := int64(10) diff --git a/server/datastore/mysql/delete.go b/server/datastore/mysql/delete.go index f310769469..384327b8b0 100644 --- a/server/datastore/mysql/delete.go +++ b/server/datastore/mysql/delete.go @@ -10,41 +10,41 @@ import ( // deleteEntity deletes an entity with the given id from the given DB table, // returning a notFound error if appropriate. -func (d *Datastore) deleteEntity(ctx context.Context, dbTable entity, id uint) error { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable.name) +func (d *Datastore) deleteEntity(ctx context.Context, dbTable string, id uint) error { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable) result, err := d.writer.ExecContext(ctx, deleteStmt, id) if err != nil { return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable.name).WithID(id) + return notFound(dbTable).WithID(id) } return nil } // deleteEntityByName deletes an entity with the given name from the given DB // table, returning a notFound error if appropriate. -func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable entity, name string) error { - deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable.name) +func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable string, name string) error { + deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable) result, err := d.writer.ExecContext(ctx, deleteStmt, name) if err != nil { if isMySQLForeignKey(err) { - return foreignKey(dbTable.name, name) + return foreignKey(dbTable, name) } return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable.name).WithName(name) + return notFound(dbTable).WithName(name) } return nil } // deleteEntities deletes the existing entity objects with the provided IDs. // The number of deleted entities is returned along with any error. -func (d *Datastore) deleteEntities(ctx context.Context, dbTable entity, ids []uint) (uint, error) { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable.name) +func (d *Datastore) deleteEntities(ctx context.Context, dbTable string, ids []uint) (uint, error) { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable) query, args, err := sqlx.In(deleteStmt, ids) if err != nil { diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index ea039461d2..abe0cc26fd 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -11,31 +11,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestDelete(t *testing.T) { +func TestDeleteEntity(t *testing.T) { ds := CreateMySQLDS(t) - - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Entity", testDeleteEntity}, - {"EntityByName", testDeleteEntityByName}, - {"Entities", testDeleteEntities}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - c.fn(t, ds) - }) - } -} - -func testDeleteEntity(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) + defer ds.Close() host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1", UUID: t.Name() + "1", @@ -47,33 +29,35 @@ func testDeleteEntity(t *testing.T, ds *Datastore) { require.NoError(t, err) require.NotNil(t, host) - require.NoError(t, ds.deleteEntity(context.Background(), hostsTable, host.ID)) + require.NoError(t, ds.deleteEntity(context.Background(), "hosts", host.ID)) host, err = ds.Host(context.Background(), host.ID) require.Error(t, err) assert.Nil(t, host) } -func testDeleteEntityByName(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestDeleteEntityByName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() query1 := test.NewQuery(t, ds, t.Name()+"time", "select * from time", 0, true) - require.NoError(t, ds.deleteEntityByName(context.Background(), queriesTable, query1.Name)) + require.NoError(t, ds.deleteEntityByName(context.Background(), "queries", query1.Name)) gotQ, err := ds.Query(context.Background(), query1.ID) require.Error(t, err) assert.Nil(t, gotQ) } -func testDeleteEntities(t *testing.T, ds *Datastore) { - defer TruncateTables(t, ds) +func TestDeleteEntities(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() query1 := test.NewQuery(t, ds, t.Name()+"time1", "select * from time", 0, true) query2 := test.NewQuery(t, ds, t.Name()+"time2", "select * from time", 0, true) query3 := test.NewQuery(t, ds, t.Name()+"time3", "select * from time", 0, true) - count, err := ds.deleteEntities(context.Background(), queriesTable, []uint{query1.ID, query2.ID}) + count, err := ds.deleteEntities(context.Background(), "queries", []uint{query1.ID, query2.ID}) require.NoError(t, err) assert.Equal(t, uint(2), count) diff --git a/server/datastore/mysql/email_changes_test.go b/server/datastore/mysql/email_changes_test.go index 1d8a84d5e1..e60a1d2a3f 100644 --- a/server/datastore/mysql/email_changes_test.go +++ b/server/datastore/mysql/email_changes_test.go @@ -11,24 +11,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestEmailChanges(t *testing.T) { +func TestChangeEmail(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Confirm", testEmailChangesConfirm}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testEmailChangesConfirm(t *testing.T, ds *Datastore) { user := &fleet.User{ Password: []byte("foobar"), Email: "bob@bob.com", @@ -59,4 +45,5 @@ func testEmailChangesConfirm(t *testing.T, ds *Datastore) { require.Nil(t, err) _, err = ds.ConfirmPendingEmailChange(context.Background(), otheruser.ID, "uniquetoken") assert.NotNil(t, err) + } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 6e558b8091..f54d922da6 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -21,7 +21,6 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, osquery_host_id, detail_updated_at, label_updated_at, - policy_updated_at, node_key, hostname, uuid, @@ -33,7 +32,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, seen_time, team_id ) - VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,?,? ) + VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,? ) ` result, err := d.writer.ExecContext( ctx, @@ -41,7 +40,6 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, host.OsqueryHostID, host.DetailUpdatedAt, host.LabelUpdatedAt, - host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -67,7 +65,6 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { UPDATE hosts SET detail_updated_at = ?, label_updated_at = ?, - policy_updated_at = ?, node_key = ?, hostname = ?, uuid = ?, @@ -104,7 +101,6 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { _, err := tx.ExecContext(ctx, sqlStatement, host.DetailUpdatedAt, host.LabelUpdatedAt, - host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -150,12 +146,7 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { } } - ac, err := d.AppConfig(ctx) - if err != nil { - return errors.Wrap(err, "failed to get app config to see if we need to update host users and inventory") - } - - if host.HostSoftware.Modified && ac.HostSettings.EnableSoftwareInventory { + if host.HostSoftware.Modified { if err := saveHostSoftwareDB(ctx, tx, host); err != nil { return errors.Wrap(err, "failed to save host software") } @@ -166,10 +157,8 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { return errors.Wrap(err, "failed to save host additional") } - if ac.HostSettings.EnableHostUsers { - if err := saveHostUsersDB(ctx, tx, host); err != nil { - return errors.Wrap(err, "failed to save host users") - } + if err := saveHostUsersDB(ctx, tx, host); err != nil { + return errors.Wrap(err, "failed to save host users") } } @@ -296,7 +285,7 @@ func loadHostUsersDB(ctx context.Context, db sqlx.QueryerContext, host *fleet.Ho } func (d *Datastore) DeleteHost(ctx context.Context, hid uint) error { - err := d.deleteEntity(ctx, hostsTable, hid) + err := d.deleteEntity(ctx, "hosts", hid) if err != nil { return errors.Wrapf(err, "deleting host with id %d", hid) } @@ -496,14 +485,13 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin INSERT INTO hosts ( detail_updated_at, label_updated_at, - policy_updated_at, osquery_host_id, seen_time, node_key, team_id - ) VALUES (?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) + result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) if err != nil { return errors.Wrap(err, "insert host") @@ -566,7 +554,6 @@ func (d *Datastore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee updated_at, detail_updated_at, label_updated_at, - policy_updated_at, node_key, hostname, uuid, diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 37cc857e5f..323cec3d7b 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -8,7 +8,6 @@ import ( "sort" "strconv" "strings" - "sync" "sync/atomic" "testing" "time" @@ -46,57 +45,13 @@ var enrollTests = []struct { }, } -func TestHosts(t *testing.T) { +func TestSaveHosts(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Save", testHostsSave}, - {"DeleteWithSoftware", testHostsDeleteWithSoftware}, - {"SavePackStats", testHostsSavePackStats}, - {"SavePackStatsOverwrites", testHostsSavePackStatsOverwrites}, - {"IgnoresTeamPackStats", testHostsIgnoresTeamPackStats}, - {"Delete", testHostsDelete}, - {"ListFilterAdditional", testHostsListFilterAdditional}, - {"ListStatus", testHostsListStatus}, - {"ListQuery", testHostsListQuery}, - {"Enroll", testHostsEnroll}, - {"Authenticate", testHostsAuthenticate}, - {"AuthenticateCaseSensitive", testHostsAuthenticateCaseSensitive}, - {"Search", testHostsSearch}, - {"SearchLimit", testHostsSearchLimit}, - {"GenerateStatusStatistics", testHostsGenerateStatusStatistics}, - {"MarkSeen", testHostsMarkSeen}, - {"MarkSeenMany", testHostsMarkSeenMany}, - {"CleanupIncoming", testHostsCleanupIncoming}, - {"IDsByName", testHostsIDsByName}, - {"Additional", testHostsAdditional}, - {"ByIdentifier", testHostsByIdentifier}, - {"AddToTeam", testHostsAddToTeam}, - {"SaveUsers", testHostsSaveUsers}, - {"SaveUsersWithoutUid", testHostsSaveUsersWithoutUid}, - {"TotalAndUnseenSince", testHostsTotalAndUnseenSince}, - {"ListByPolicy", testHostsListByPolicy}, - {"SaveTonsOfUsers", testHostsSaveTonsOfUsers}, - {"SavePackStatsConcurrent", testHostsSavePackStatsConcurrent}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - - c.fn(t, ds) - }) - } -} - -func testHostsSave(t *testing.T, ds *Datastore) { - policyUpdatedAt := time.Now().UTC().Truncate(time.Second) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: policyUpdatedAt, SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -116,7 +71,6 @@ func testHostsSave(t *testing.T, ds *Datastore) { assert.Equal(t, "bar.local", host.Hostname) assert.Equal(t, "192.168.1.1", host.PrimaryIP) assert.Equal(t, "30-65-EC-6F-C4-58", host.PrimaryMac) - assert.Equal(t, policyUpdatedAt.UTC(), host.PolicyUpdatedAt) additionalJSON := json.RawMessage(`{"foobar": "bim"}`) host.Additional = &additionalJSON @@ -145,11 +99,13 @@ func testHostsSave(t *testing.T, ds *Datastore) { assert.Nil(t, host) } -func testHostsDeleteWithSoftware(t *testing.T, ds *Datastore) { +func TestDeleteHostWithSoftware(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -179,11 +135,13 @@ func testHostsDeleteWithSoftware(t *testing.T, ds *Datastore) { assert.Nil(t, host) } -func testHostsSavePackStats(t *testing.T, ds *Datastore) { +func TestSaveHostPackStats(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -292,11 +250,13 @@ func testHostsSavePackStats(t *testing.T, ds *Datastore) { require.Len(t, host.PackStats, 2) } -func testHostsSavePackStatsOverwrites(t *testing.T, ds *Datastore) { +func TestSaveHostPackStatsOverwrites(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -436,11 +396,13 @@ func testHostsSavePackStatsOverwrites(t *testing.T, ds *Datastore) { assert.Equal(t, execTime2, gotHost.PackStats[0].QueryStats[0].LastExecuted) } -func testHostsIgnoresTeamPackStats(t *testing.T, ds *Datastore) { +func TestIgnoresTeamPackStats(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -516,11 +478,13 @@ func testHostsIgnoresTeamPackStats(t *testing.T, ds *Datastore) { assert.ElementsMatch(t, host.PackStats[0].QueryStats, stats1) } -func testHostsDelete(t *testing.T, ds *Datastore) { +func TestDeleteHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -536,11 +500,13 @@ func testHostsDelete(t *testing.T, ds *Datastore) { assert.NotNil(t, err) } -func testHostsListFilterAdditional(t *testing.T, ds *Datastore) { +func TestListHostsFilterAdditional(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + h, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -574,12 +540,14 @@ func testHostsListFilterAdditional(t *testing.T, ds *Datastore) { assert.Equal(t, &additional, hosts[0].Additional) } -func testHostsListStatus(t *testing.T, ds *Datastore) { +func TestListHostsStatus(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -611,13 +579,15 @@ func testHostsListStatus(t *testing.T, ds *Datastore) { assert.Equal(t, 10, len(hosts)) } -func testHostsListQuery(t *testing.T, ds *Datastore) { +func TestListHostsQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + hosts := []*fleet.Host{} for i := 0; i < 10; i++ { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -699,7 +669,10 @@ func testHostsListQuery(t *testing.T, ds *Datastore) { assert.Equal(t, 1, len(gotHosts)) } -func testHostsEnroll(t *testing.T, ds *Datastore) { +func TestEnrollHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + test.AddAllHostsLabel(t, ds) team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) @@ -736,7 +709,10 @@ func testHostsEnroll(t *testing.T, ds *Datastore) { } } -func testHostsAuthenticate(t *testing.T, ds *Datastore) { +func TestAuthenticateHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -754,7 +730,10 @@ func testHostsAuthenticate(t *testing.T, ds *Datastore) { assert.Error(t, err) } -func testHostsAuthenticateCaseSensitive(t *testing.T, ds *Datastore) { +func TestAuthenticateHostCaseSensitive(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -765,12 +744,14 @@ func testHostsAuthenticateCaseSensitive(t *testing.T, ds *Datastore) { } } -func testHostsSearch(t *testing.T, ds *Datastore) { +func TestSearchHosts(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + _, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -782,7 +763,6 @@ func testHostsSearch(t *testing.T, ds *Datastore) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -794,7 +774,6 @@ func testHostsSearch(t *testing.T, ds *Datastore) { OsqueryHostID: "99999", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "3", UUID: "abc-def-ghi", @@ -857,14 +836,16 @@ func testHostsSearch(t *testing.T, ds *Datastore) { assert.Equal(t, 1, len(hits)) } -func testHostsSearchLimit(t *testing.T, ds *Datastore) { +func TestSearchHostsLimit(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + filter := fleet.TeamFilter{User: test.UserAdmin} for i := 0; i < 15; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -879,7 +860,10 @@ func testHostsSearchLimit(t *testing.T, ds *Datastore) { assert.Len(t, hosts, 10) } -func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { +func TestGenerateHostStatusStatistics(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + filter := fleet.TeamFilter{User: test.UserAdmin} mockClock := clock.NewMockClock() @@ -897,7 +881,6 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: mockClock.Now().Add(-30 * time.Second), LabelUpdatedAt: mockClock.Now().Add(-30 * time.Second), - PolicyUpdatedAt: mockClock.Now().Add(-30 * time.Second), SeenTime: mockClock.Now().Add(-30 * time.Second), }) require.Nil(t, err) @@ -912,7 +895,6 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "2", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Minute), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Minute), - PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Minute), SeenTime: mockClock.Now().Add(-1 * time.Minute), }) require.Nil(t, err) @@ -927,7 +909,6 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "3", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Hour), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Hour), - PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Hour), SeenTime: mockClock.Now().Add(-1 * time.Hour), }) require.Nil(t, err) @@ -942,7 +923,6 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "4", DetailUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), LabelUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), - PolicyUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)), }) require.Nil(t, err) @@ -962,7 +942,10 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { assert.Equal(t, uint(4), new) } -func testHostsMarkSeen(t *testing.T, ds *Datastore) { +func TestMarkHostSeen(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + mockClock := clock.NewMockClock() anHourAgo := mockClock.Now().Add(-1 * time.Hour).UTC() @@ -975,7 +958,6 @@ func testHostsMarkSeen(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, - PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) assert.Nil(t, err) @@ -998,7 +980,10 @@ func testHostsMarkSeen(t *testing.T, ds *Datastore) { } } -func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { +func TestMarkHostsSeen(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + mockClock := clock.NewMockClock() aSecondAgo := mockClock.Now().Add(-1 * time.Second).UTC() @@ -1012,7 +997,6 @@ func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, - PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1024,7 +1008,6 @@ func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { NodeKey: "2", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, - PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1058,9 +1041,13 @@ func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { require.NotNil(t, h2Verify) assert.WithinDuration(t, aSecondAgo, h2Verify.SeenTime, time.Second) } + } -func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { +func TestCleanupIncomingHosts(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + mockClock := clock.NewMockClock() h1, err := ds.NewHost(context.Background(), &fleet.Host{ @@ -1070,7 +1057,6 @@ func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), - PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1084,7 +1070,6 @@ func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { OsqueryVersion: "3.2.3", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), - PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1108,12 +1093,14 @@ func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { assert.Nil(t, err) } -func testHostsIDsByName(t *testing.T, ds *Datastore) { +func TestHostIDsByName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1130,11 +1117,13 @@ func testHostsIDsByName(t *testing.T, ds *Datastore) { assert.Equal(t, hosts, []uint{2, 3, 6}) } -func testHostsAdditional(t *testing.T, ds *Datastore) { +func TestHostAdditional(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -1202,12 +1191,14 @@ func testHostsAdditional(t *testing.T, ds *Datastore) { assert.Equal(t, &additional, h.Additional) } -func testHostsByIdentifier(t *testing.T, ds *Datastore) { +func TestHostByIdentifier(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 1; i <= 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("osquery_host_id_%d", i), NodeKey: fmt.Sprintf("node_key_%d", i), @@ -1241,7 +1232,10 @@ func testHostsByIdentifier(t *testing.T, ds *Datastore) { require.Error(t, err) } -func testHostsAddToTeam(t *testing.T, ds *Datastore) { +func TestAddHostsToTeam(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -1288,11 +1282,13 @@ func testHostsAddToTeam(t *testing.T, ds *Datastore) { } } -func testHostsSaveUsers(t *testing.T, ds *Datastore) { +func TestSaveHostUsers(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1358,11 +1354,13 @@ func testHostsSaveUsers(t *testing.T, ds *Datastore) { test.ElementsMatchSkipID(t, host.Users, []fleet.HostUser{u1, u2}) } -func testHostsSaveUsersWithoutUid(t *testing.T, ds *Datastore) { +func TestSaveUsersWithoutUid(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1418,7 +1416,6 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-1 * time.Duration(days) * 24 * time.Hour), OsqueryHostID: fmt.Sprintf("%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1431,7 +1428,10 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { require.NotNil(t, host) } -func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { +func TestTotalAndUnseenHostsSince(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + addHostSeenLast(t, ds, 1, 0) total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) @@ -1448,12 +1448,14 @@ func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { assert.Equal(t, 2, unseen) } -func testHostsListByPolicy(t *testing.T, ds *Datastore) { +func TestListHostsByPolicy(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -1504,11 +1506,13 @@ func testHostsListByPolicy(t *testing.T, ds *Datastore) { require.Len(t, hosts, 8) } -func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { +func TestSaveTonsOfUsers(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1523,7 +1527,6 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -1542,12 +1545,7 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { var count1 int32 var count2 int32 - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - for { host1, err := ds.Host(context.Background(), host1.ID) if err != nil { @@ -1586,9 +1584,7 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { errCh <- err return } - if atomic.AddInt32(&count1, 1) >= 100 { - return - } + atomic.AddInt32(&count1, 1) select { case <-ctx.Done(): @@ -1599,8 +1595,6 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { }() go func() { - defer wg.Done() - for { host2, err := ds.Host(context.Background(), host2.ID) if err != nil { @@ -1639,9 +1633,7 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { errCh <- err return } - if atomic.AddInt32(&count2, 1) >= 100 { - return - } + atomic.AddInt32(&count2, 1) select { case <-ctx.Done(): @@ -1652,28 +1644,24 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { }() ticker := time.NewTicker(10 * time.Second) - go func() { - wg.Wait() - cancelFunc() - }() select { case err := <-errCh: - cancelFunc() require.NoError(t, err) - case <-ctx.Done(): + cancelFunc() case <-ticker.C: - require.Fail(t, "timed out") } - t.Log("Count1", atomic.LoadInt32(&count1)) - t.Log("Count2", atomic.LoadInt32(&count2)) + fmt.Println("Count1", atomic.LoadInt32(&count1)) + fmt.Println("Count2", atomic.LoadInt32(&count2)) } -func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { +func TestSaveHostPackStatsConcurrent(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1688,7 +1676,6 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -1756,18 +1743,14 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { }, }, } - return ds.SaveHost(context.Background(), host) + return ds.SaveHost(context.Background(), host1) } errCh := make(chan error) var counter int32 - const total = int32(100) - - var wg sync.WaitGroup + total := int32(1000) loopAndSaveHost := func(host *fleet.Host) { - defer wg.Done() - for { err := saveHostRandomStats(host) if err != nil { @@ -1787,13 +1770,10 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { } } - wg.Add(3) go loopAndSaveHost(host1) go loopAndSaveHost(host2) go func() { - defer wg.Done() - for { specs := []*fleet.PackSpec{ { @@ -1837,14 +1817,13 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { } }() - ticker := time.NewTicker(10 * time.Second) + ticker := time.NewTicker(1 * time.Minute) select { case err := <-errCh: - cancelFunc() require.NoError(t, err) case <-ctx.Done(): - wg.Wait() + return case <-ticker.C: - require.Fail(t, "timed out") + t.Fail() } } diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go index c2d9371c75..bd92bb8aaa 100644 --- a/server/datastore/mysql/invites.go +++ b/server/datastore/mysql/invites.go @@ -138,7 +138,7 @@ func (d *Datastore) InviteByToken(ctx context.Context, token string) (*fleet.Inv } func (d *Datastore) DeleteInvite(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, invitesTable, id) + return d.deleteEntity(ctx, "invites", id) } func (d *Datastore) loadTeamsForInvites(ctx context.Context, invites []*fleet.Invite) error { diff --git a/server/datastore/mysql/invites_test.go b/server/datastore/mysql/invites_test.go index b1f6a0e587..a1b28edbc5 100644 --- a/server/datastore/mysql/invites_test.go +++ b/server/datastore/mysql/invites_test.go @@ -12,29 +12,10 @@ import ( "gopkg.in/guregu/null.v3" ) -func TestInvites(t *testing.T) { +func TestCreateInvite(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Create", testInvitesCreate}, - {"List", testInvitesList}, - {"Delete", testInvitesDelete}, - {"ByToken", testInvitesByToken}, - {"ByEmail", testInvitesByEmail}, - {"Invite", testInvitesInvite}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testInvitesCreate(t *testing.T, ds *Datastore) { for i := 0; i < 3; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -83,9 +64,13 @@ func setupTestInvites(t *testing.T, ds fleet.Datastore) { _, err := ds.NewInvite(context.Background(), &i) require.NoError(t, err, "Failure creating user", user) } + } -func testInvitesList(t *testing.T, ds *Datastore) { +func TestListInvites(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupTestInvites(t, ds) opt := fleet.ListOptions{ @@ -109,9 +94,13 @@ func testInvitesList(t *testing.T, ds *Datastore) { assert.Nil(t, err) assert.Equal(t, 4, len(result)) // allow for admin we created assert.Equal(t, "User00", result[3].Name) + } -func testInvitesDelete(t *testing.T, ds *Datastore) { +func TestDeleteInvite(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupTestInvites(t, ds) invite, err := ds.InviteByEmail(context.Background(), "user0@foo.com") @@ -125,9 +114,13 @@ func testInvitesDelete(t *testing.T, ds *Datastore) { invite, err = ds.InviteByEmail(context.Background(), "user0@foo.com") assert.NotNil(t, err) assert.Nil(t, invite) + } -func testInvitesByToken(t *testing.T, ds *Datastore) { +func TestInviteByToken(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupTestInvites(t, ds) var inviteTests = []struct { @@ -144,7 +137,7 @@ func testInvitesByToken(t *testing.T, ds *Datastore) { } for _, tt := range inviteTests { - t.Run(tt.token, func(t *testing.T) { + t.Run("", func(t *testing.T) { invite, err := ds.InviteByToken(context.Background(), tt.token) if tt.wantErr != nil { require.NotNil(t, err) @@ -153,11 +146,15 @@ func testInvitesByToken(t *testing.T, ds *Datastore) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) + }) } } -func testInvitesByEmail(t *testing.T, ds *Datastore) { +func TestInviteByEmail(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupTestInvites(t, ds) var inviteTests = []struct { @@ -174,7 +171,7 @@ func testInvitesByEmail(t *testing.T, ds *Datastore) { } for _, tt := range inviteTests { - t.Run(tt.email, func(t *testing.T) { + t.Run("", func(t *testing.T) { invite, err := ds.InviteByEmail(context.Background(), tt.email) if tt.wantErr != nil { require.NotNil(t, err) @@ -183,11 +180,15 @@ func testInvitesByEmail(t *testing.T, ds *Datastore) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) + }) } } -func testInvitesInvite(t *testing.T, ds *Datastore) { +func TestInvite(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + admin := &fleet.Invite{ Email: "admin@foo.com", Name: "Xadmin", diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 9b46ac0714..5da41d18da 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -217,35 +217,33 @@ func (d *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...fl } func (d *Datastore) SaveLabel(ctx context.Context, label *fleet.Label) (*fleet.Label, error) { - query := `UPDATE labels SET name = ?, description = ? WHERE id = ?` + query := ` + UPDATE labels SET + name = ?, + description = ? + WHERE id = ? + ` _, err := d.writer.ExecContext(ctx, query, label.Name, label.Description, label.ID) if err != nil { return nil, errors.Wrap(err, "saving label") } - return labelDB(ctx, label.ID, d.writer) + return label, nil } // DeleteLabel deletes a fleet.Label func (d *Datastore) DeleteLabel(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, labelsTable, name) + return d.deleteEntityByName(ctx, "labels", name) } // Label returns a fleet.Label identified by lid if one exists. func (d *Datastore) Label(ctx context.Context, lid uint) (*fleet.Label, error) { - return labelDB(ctx, lid, d.reader) -} - -func labelDB(ctx context.Context, lid uint, q sqlx.QueryerContext) (*fleet.Label, error) { sql := ` - SELECT - l.*, - (SELECT COUNT(1) FROM label_membership lm JOIN hosts h ON (lm.host_id = h.id) WHERE label_id = l.id) AS host_count - FROM labels l - WHERE id = ? + SELECT * FROM labels + WHERE id = ? ` label := &fleet.Label{} - if err := sqlx.GetContext(ctx, q, label, sql, lid); err != nil { + if err := sqlx.GetContext(ctx, d.reader, label, sql, lid); err != nil { return nil, errors.Wrap(err, "selecting label") } @@ -285,12 +283,39 @@ func platformForHost(host *fleet.Host) string { return host.Platform } -func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { +func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { var rows *sql.Rows var err error platform := platformForHost(host) - query := `SELECT id, query FROM labels WHERE platform = ? OR platform = '' AND label_membership_type = ?` - rows, err = d.reader.QueryContext(ctx, query, platform, fleet.LabelMembershipTypeDynamic) + if host.LabelUpdatedAt.Before(cutoff) { + // Retrieve all labels (with matching platform) for this host + sql := ` + SELECT id, query + FROM labels + WHERE platform = ? OR platform = '' + AND label_membership_type = ? +` + rows, err = d.reader.QueryContext(ctx, sql, platform, fleet.LabelMembershipTypeDynamic) + } else { + // Retrieve all labels (with matching platform) iff there is a label + // that has been created since this host last reported label query + // executions + sql := ` + SELECT id, query + FROM labels + WHERE ((SELECT max(created_at) FROM labels WHERE platform = ? OR platform = '') > ?) + AND (platform = ? OR platform = '') + AND label_membership_type = ? +` + rows, err = d.reader.QueryContext( + ctx, + sql, + platform, + host.LabelUpdatedAt, + platform, + fleet.LabelMembershipTypeDynamic, + ) + } if err != nil && err != sql.ErrNoRows { return nil, errors.Wrap(err, "selecting label queries for host") @@ -345,8 +370,14 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Complete inserts if necessary if len(vals) > 0 { - sql := `INSERT INTO label_membership (updated_at, label_id, host_id) VALUES ` - sql += strings.Join(bindvars, ",") + ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` + sql := ` + INSERT IGNORE INTO label_membership (updated_at, label_id, host_id) VALUES + ` + sql += strings.Join(bindvars, ",") + + ` + ON DUPLICATE KEY UPDATE + updated_at = VALUES(updated_at) + ` _, err := tx.ExecContext(ctx, sql, vals...) if err != nil { @@ -356,7 +387,9 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. // Complete deletions if necessary if len(removes) > 0 { - sql := `DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?)` + sql := ` + DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?) + ` query, args, err := sqlx.In(sql, host.ID, removes) if err != nil { return errors.Wrap(err, "IN for DELETE FROM label_membership") @@ -633,12 +666,5 @@ func (d *Datastore) LabelIDsByName(ctx context.Context, labels []string) ([]uint } return labelIDs, nil -} -func (d *Datastore) CleanupOrphanLabelMembership(ctx context.Context) error { - _, err := d.writer.ExecContext(ctx, `DELETE FROM label_membership where not exists (select 1 from labels where id=label_id) or not exists (select 1 from hosts where id=host_id)`) - if err != nil { - return errors.Wrap(err, "cleaning orphan label_membership by label") - } - return nil } diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index c610ce7859..30fef5d73e 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -37,37 +37,9 @@ func TestBatchHostnamesLarge(t *testing.T) { } func TestLabels(t *testing.T) { - ds := CreateMySQLDS(t) + db := CreateMySQLDS(t) + defer db.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"AddAllHosts", testLabelsAddAllHosts}, - {"Search", testLabelsSearch}, - {"ListHostsInLabel", testLabelsListHostsInLabel}, - {"ListHostsInLabelAndStatus", testLabelsListHostsInLabelAndStatus}, - {"ListHostsInLabelAndTeamFilter", testLabelsListHostsInLabelAndTeamFilter}, - {"BuiltIn", testLabelsBuiltIn}, - {"ListUniqueHostsInLabels", testLabelsListUniqueHostsInLabels}, - {"ChangeDetails", testLabelsChangeDetails}, - {"GetSpec", testLabelsGetSpec}, - {"ApplySpecsRoundtrip", testLabelsApplySpecsRoundtrip}, - {"IDsByName", testLabelsIDsByName}, - {"Save", testLabelsSave}, - {"QueriesForCentOSHost", testLabelsQueriesForCentOSHost}, - {"RecordNonExistentQueryLabelExecution", testLabelsRecordNonexistentQueryLabelExecution}, - {"LabelMembershipCleanup", testLabelMembershipCleanup}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testLabelsAddAllHosts(t *testing.T, db *Datastore) { test.AddAllHostsLabel(t, db) hosts := []fleet.Host{} var host *fleet.Host @@ -80,8 +52,10 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { host.Platform = "darwin" require.NoError(t, db.SaveHost(context.Background(), host)) + baseTime := time.Now() + // No labels to check - queries, err := db.LabelQueriesForHost(context.Background(), host) + queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) assert.Nil(t, err) assert.Len(t, queries, 0) @@ -125,7 +99,7 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { host.Platform = "darwin" // Now queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host) + queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime) assert.Nil(t, err) assert.Equal(t, expectQueries, queries) @@ -134,8 +108,6 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { assert.Nil(t, err) assert.Len(t, labels, 1) - baseTime := time.Now() - // Record a query execution err = db.RecordLabelQueryExecutions( context.Background(), @@ -148,6 +120,14 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { require.NoError(t, err) host.LabelUpdatedAt = baseTime + // Now no queries should be returned + queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) + assert.Nil(t, err) + assert.Len(t, queries, 0) + + // Ensure enough gap in created_at + time.Sleep(2 * time.Second) + // A new label targeting another platform should not effect the labels for // this host err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -158,9 +138,9 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { }, }) require.NoError(t, err) - queries, err = db.LabelQueriesForHost(context.Background(), host) + queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) assert.Nil(t, err) - assert.Len(t, queries, 4) + assert.Len(t, queries, 0) // If a new label is added, all labels should be returned err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -172,7 +152,29 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { }) require.NoError(t, err) expectQueries["7"] = "query6" - queries, err = db.LabelQueriesForHost(context.Background(), host) + queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) + assert.Nil(t, err) + assert.Len(t, queries, 5) + + // After expiration, all queries should be returned + queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add((2 * time.Minute))) + assert.Nil(t, err) + assert.Equal(t, expectQueries, queries) + + // Now the two matching labels should be returned + labels, err = db.ListLabelsForHost(context.Background(), host.ID) + assert.Nil(t, err) + if assert.Len(t, labels, 2) { + labelNames := []string{labels[0].Name, labels[1].Name} + sort.Strings(labelNames) + assert.Equal(t, "All Hosts", labelNames[0]) + assert.Equal(t, "label1", labelNames[1]) + } + + // A host that hasn't executed any label queries should still be asked + // to execute those queries + hosts[0].Platform = "darwin" + queries, err = db.LabelQueriesForHost(context.Background(), &hosts[0], time.Now()) assert.Nil(t, err) assert.Len(t, queries, 5) @@ -183,7 +185,10 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { assert.Len(t, labels, 1) } -func testLabelsSearch(t *testing.T, db *Datastore) { +func TestSearchLabels(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + specs := []*fleet.LabelSpec{ {ID: 1, Name: "foo"}, {ID: 2, Name: "bar"}, @@ -236,11 +241,13 @@ func testLabelsSearch(t *testing.T, db *Datastore) { assert.Contains(t, labels, all) } -func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { +func TestListHostsInLabel(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -252,7 +259,6 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "2", NodeKey: "2", @@ -264,7 +270,6 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h3, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "3", NodeKey: "3", @@ -301,11 +306,13 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { } } -func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { +func TestListHostsInLabelAndStatus(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -318,7 +325,6 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, - PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -356,11 +362,13 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { } } -func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { +func TestListHostsInLabelAndTeamFilter(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -373,7 +381,6 @@ func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, - PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -432,7 +439,10 @@ func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { } } -func testLabelsBuiltIn(t *testing.T, db *Datastore) { +func TestBuiltInLabels(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + require.Nil(t, db.MigrateData(context.Background())) user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} @@ -446,13 +456,15 @@ func testLabelsBuiltIn(t *testing.T, db *Datastore) { assert.Equal(t, fleet.LabelTypeBuiltIn, hits[1].LabelType) } -func testLabelsListUniqueHostsInLabels(t *testing.T, db *Datastore) { +func TestListUniqueHostsInLabels(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + hosts := []*fleet.Host{} for i := 0; i < 4; i++ { h, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -496,9 +508,13 @@ func testLabelsListUniqueHostsInLabels(t *testing.T, db *Datastore) { labels, err := db.ListLabels(context.Background(), filter, fleet.ListOptions{}) require.Nil(t, err) require.Len(t, labels, 2) + } -func testLabelsChangeDetails(t *testing.T, db *Datastore) { +func TestChangeLabelDetails(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + label := fleet.LabelSpec{ ID: 1, Name: "my label", @@ -523,7 +539,6 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -568,7 +583,10 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { return expectedSpecs } -func testLabelsGetSpec(t *testing.T, ds *Datastore) { +func TestGetLabelSpec(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + expectedSpecs := setupLabelSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -578,7 +596,10 @@ func testLabelsGetSpec(t *testing.T, ds *Datastore) { } } -func testLabelsApplySpecsRoundtrip(t *testing.T, ds *Datastore) { +func TestApplyLabelSpecsRoundtrip(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + expectedSpecs := setupLabelSpecsTest(t, ds) specs, err := ds.GetLabelSpecs(context.Background()) @@ -593,7 +614,10 @@ func testLabelsApplySpecsRoundtrip(t *testing.T, ds *Datastore) { test.ElementsMatchSkipTimestampsID(t, expectedSpecs, specs) } -func testLabelsIDsByName(t *testing.T, ds *Datastore) { +func TestLabelIDsByName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupLabelSpecsTest(t, ds) labels, err := ds.LabelIDsByName(context.Background(), []string{"foo", "bar", "bing"}) @@ -602,18 +626,9 @@ func testLabelsIDsByName(t *testing.T, ds *Datastore) { assert.Equal(t, []uint{1, 2, 3}, labels) } -func testLabelsSave(t *testing.T, db *Datastore) { - h1, err := db.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - OsqueryHostID: "1", - NodeKey: "1", - UUID: "1", - Hostname: "foo.local", - }) - require.NoError(t, err) +func TestSaveLabel(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() label := &fleet.Label{ Name: "my label", @@ -621,23 +636,22 @@ func testLabelsSave(t *testing.T, db *Datastore) { Query: "select 1 from processes;", Platform: "darwin", } - label, err = db.NewLabel(context.Background(), label) - require.NoError(t, err) + label, err := db.NewLabel(context.Background(), label) + require.Nil(t, err) label.Name = "changed name" label.Description = "changed description" - - require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) - _, err = db.SaveLabel(context.Background(), label) - require.NoError(t, err) + require.Nil(t, err) saved, err := db.Label(context.Background(), label.ID) - require.NoError(t, err) + require.Nil(t, err) assert.Equal(t, label.Name, saved.Name) assert.Equal(t, label.Description, saved.Description) - assert.Equal(t, 1, saved.HostCount) } -func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { +func TestLabelQueriesForCentOSHost(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + host, err := db.EnrollHost(context.Background(), "0", "0", nil, 0) require.Nil(t, err, "enrollment should succeed") host.Platform = "rhel" @@ -658,17 +672,21 @@ func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { }) require.NoError(t, err) - queries, err := db.LabelQueriesForHost(context.Background(), host) + baseTime := time.Now().Add(-5 * time.Minute) + + queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) require.NoError(t, err) require.Len(t, queries, 1) assert.Equal(t, "select 1;", queries[fmt.Sprint(label.ID)]) } -func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) { +func TestRecordNonexistentQueryLabelExecution(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() + h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -687,69 +705,3 @@ func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{99999: ptr.Bool(true)}, time.Now())) } - -func testLabelMembershipCleanup(t *testing.T, ds *Datastore) { - setupTest := func() (*fleet.Host, *fleet.Label) { - host, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - OsqueryHostID: "1", - }) - require.NoError(t, err) - require.NotNil(t, host) - - label := &fleet.Label{ - Name: "foo", - Query: "select * from foo;", - } - label, err = ds.NewLabel(context.Background(), label) - require.NoError(t, err) - - require.NoError(t, ds.RecordLabelQueryExecutions(context.Background(), host, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) - return host, label - } - - checkCount := func(before, after int) { - var count int - require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) - assert.Equal(t, before, count) - - require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) - - require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) - assert.Equal(t, after, count) - } - - t.Run("none gone", func(t *testing.T) { - host, label := setupTest() - checkCount(1, 1) - require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) - require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) - require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) - }) - t.Run("label gone", func(t *testing.T) { - host, label := setupTest() - require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) - checkCount(1, 0) - require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) - }) - t.Run("host gone", func(t *testing.T) { - host, label := setupTest() - require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) - checkCount(1, 0) - require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) - }) - t.Run("both gone", func(t *testing.T) { - host, label := setupTest() - require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) - require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) - checkCount(1, 0) - }) -} diff --git a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go index 4de803112b..f1721cf57b 100644 --- a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go +++ b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go @@ -31,13 +31,13 @@ func Up_20210819131107(tx *sql.Tx) error { } // Clear any orphan software and host_software - _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_host_software AS SELECT * FROM host_software;`) + _, err = tx.Exec(`DELETE FROM host_software WHERE NOT EXISTS (select 1 from hosts h where h.id=host_software.host_id)`) if err != nil { - return errors.Wrap(err, "save current host software to a temp table") + return errors.Wrap(err, "clearing orphan host_software") } - _, err = tx.Exec(`DELETE FROM host_software;`) + _, err = tx.Exec(`DELETE FROM software WHERE NOT EXISTS (select 1 from host_software hs where hs.software_id=software.id)`) if err != nil { - return errors.Wrap(err, "clear all host software") + return errors.Wrap(err, "clearing orphan software") } if _, err := tx.Exec(` @@ -48,16 +48,6 @@ func Up_20210819131107(tx *sql.Tx) error { return errors.Wrap(err, "add fk on host_software hosts & software") } - _, err = tx.Exec(`INSERT IGNORE INTO host_software SELECT * FROM temp_host_software;`) - if err != nil { - return errors.Wrap(err, "reinserting host software") - } - - _, err = tx.Exec(`DROP TABLE temp_host_software;`) - if err != nil { - return errors.Wrap(err, "dropping temp table") - } - return nil } diff --git a/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go b/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go deleted file mode 100644 index 40cec1783a..0000000000 --- a/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go +++ /dev/null @@ -1,25 +0,0 @@ -package tables - -import ( - "database/sql" - - "github.com/pkg/errors" -) - -func init() { - MigrationClient.AddMigration(Up_20210915144307, Down_20210915144307) -} - -func Up_20210915144307(tx *sql.Tx) error { - if _, err := tx.Exec(`ALTER TABLE policies - ADD COLUMN team_id INT UNSIGNED, - ADD FOREIGN KEY fk_policies_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE - `); err != nil { - return errors.Wrap(err, "add column team_id") - } - return nil -} - -func Down_20210915144307(tx *sql.Tx) error { - return nil -} diff --git a/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go b/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go deleted file mode 100644 index 6c7466e722..0000000000 --- a/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go +++ /dev/null @@ -1,24 +0,0 @@ -package tables - -import ( - "database/sql" - - "github.com/pkg/errors" -) - -func init() { - MigrationClient.AddMigration(Up_20210920155130, Down_20210920155130) -} - -func Up_20210920155130(tx *sql.Tx) error { - _, err := tx.Exec( - `ALTER TABLE label_membership DROP FOREIGN KEY fk_lm_host_id, DROP FOREIGN KEY fk_lm_label_id;`) - if err != nil { - return errors.Wrap(err, "dropping foreign keys for label_membership") - } - return nil -} - -func Down_20210920155130(tx *sql.Tx) error { - return nil -} diff --git a/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go b/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go deleted file mode 100644 index 43385178ee..0000000000 --- a/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go +++ /dev/null @@ -1,29 +0,0 @@ -package tables - -import ( - "database/sql" - - "github.com/pkg/errors" -) - -func init() { - MigrationClient.AddMigration(Up_20210927143115, Down_20210927143115) -} - -func Up_20210927143115(tx *sql.Tx) error { - _, err := tx.Exec("ALTER TABLE hosts ADD COLUMN policy_updated_at timestamp NOT NULL DEFAULT '2000-01-01 00:00:00'") - if err != nil { - return errors.Wrap(err, "adding policy_updated_at column") - } - - _, err = tx.Exec("delete from policy_membership_history") - if err != nil { - return errors.Wrap(err, "clearing policy_membership_history") - } - - return err -} - -func Down_20210927143115(tx *sql.Tx) error { - return nil -} diff --git a/server/datastore/mysql/migrations_test.go b/server/datastore/mysql/migrations_test.go index 57d469eec4..f30833f41c 100644 --- a/server/datastore/mysql/migrations_test.go +++ b/server/datastore/mysql/migrations_test.go @@ -5,9 +5,12 @@ import ( "context" "os/exec" "testing" + "time" "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -65,3 +68,54 @@ func createMySQLDSForMigrationTests(t *testing.T, dbName string) *Datastore { require.NoError(t, err) return ds } + +func Test20210819131107_AddCascadeToHostSoftware(t *testing.T) { + ds := createMySQLDSForMigrationTests(t, t.Name()) + defer ds.Close() + + for { + version, err := tables.MigrationClient.GetDBVersion(ds.writer.DB) + require.NoError(t, err) + + // break right before the the constraint migration + if version == 20210818182258 { + break + } + require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) + } + + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) + + soft1 := fleet.HostSoftware{ + Modified: true, + Software: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + }, + } + host1.HostSoftware = soft1 + soft2 := fleet.HostSoftware{ + Modified: true, + Software: []fleet.Software{ + {Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, + }, + } + host2.HostSoftware = soft2 + host2.Modified = true + + require.NoError(t, ds.SaveHostSoftware(context.Background(), host1)) + require.NoError(t, ds.SaveHostSoftware(context.Background(), host2)) + + require.NoError(t, ds.DeleteHost(context.Background(), host1.ID)) + + require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) + + // Make sure we don't delete more than we need + hostCheck, err := ds.Host(context.Background(), host2.ID) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(context.Background(), hostCheck)) + require.Len(t, hostCheck.Software, 3) +} diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 9664df5111..6d72fab858 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -62,22 +62,6 @@ type Datastore struct { type txFn func(sqlx.ExtContext) error -type entity struct { - name string -} - -var ( - hostsTable = entity{"hosts"} - invitesTable = entity{"invites"} - labelsTable = entity{"labels"} - packsTable = entity{"packs"} - queriesTable = entity{"queries"} - scheduledQueriesTable = entity{"scheduled_queries"} - sessionsTable = entity{"sessions"} - teamsTable = entity{"teams"} - usersTable = entity{"users"} -) - // retryableError determines whether a MySQL error can be retried. By default // errors are considered non-retryable. Only errors that we know have a // possibility of succeeding on a retry should return true in this function. diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index 3e10b99d20..68b40cbcdd 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -50,7 +50,6 @@ func TestDatastoreReplica(t *testing.T) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index 3e03afc873..20b2e89fed 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -365,7 +365,7 @@ func (d *Datastore) SavePack(ctx context.Context, pack *fleet.Pack) error { // DeletePack deletes a fleet.Pack so that it won't show up in results. func (d *Datastore) DeletePack(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, packsTable, name) + return d.deleteEntityByName(ctx, "packs", name) } // Pack fetch fleet.Pack with matching ID diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index c571b3b36f..710d752a22 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -15,39 +15,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestPacks(t *testing.T) { +func TestDeletePack(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Delete", testPacksDelete}, - {"Save", testPacksSave}, - {"GetByName", testPacksGetByName}, - {"List", testPacksList}, - {"ApplySpecRoundtrip", testPacksApplySpecRoundtrip}, - {"GetSpec", testPacksGetSpec}, - {"ApplySpecMissingQueries", testPacksApplySpecMissingQueries}, - {"ApplySpecMissingName", testPacksApplySpecMissingName}, - {"ListForHost", testPacksListForHost}, - {"EnsureGlobal", testPacksEnsureGlobal}, - {"EnsureTeam", testPacksEnsureTeam}, - {"TeamNameChangesTeamSchedule", testPacksTeamNameChangesTeamSchedule}, - {"TeamScheduleNamesMigrateToNewFormat", testPacksTeamScheduleNamesMigrateToNewFormat}, - {"ApplySpecFailsOnTargetIDNull", testPacksApplySpecFailsOnTargetIDNull}, - {"ApplyStatsNotLocking", testPacksApplyStatsNotLocking}, - {"ApplyStatsNotLockingTryTwo", testPacksApplyStatsNotLockingTryTwo}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testPacksDelete(t *testing.T, ds *Datastore) { pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -62,7 +33,10 @@ func testPacksDelete(t *testing.T, ds *Datastore) { assert.NotNil(t, err) } -func testPacksSave(t *testing.T, ds *Datastore) { +func TestSavePack(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + expectedPack := &fleet.Pack{ Name: "foo", HostIDs: []uint{1}, @@ -96,7 +70,10 @@ func testPacksSave(t *testing.T, ds *Datastore) { test.EqualSkipTimestampsID(t, expectedPack, pack) } -func testPacksGetByName(t *testing.T, ds *Datastore) { +func TestGetPackByName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -110,9 +87,13 @@ func testPacksGetByName(t *testing.T, ds *Datastore) { require.Nil(t, err) assert.False(t, ok) assert.Nil(t, pack) + } -func testPacksList(t *testing.T, ds *Datastore) { +func TestListPacks(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + p1 := &fleet.PackSpec{ ID: 1, Name: "foo_pack", @@ -242,7 +223,10 @@ func setupPackSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.PackSpec { return expectedSpecs } -func testPacksApplySpecRoundtrip(t *testing.T, ds *Datastore) { +func TestApplyPackSpecRoundtrip(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + expectedSpecs := setupPackSpecsTest(t, ds) gotSpec, err := ds.GetPackSpecs(context.Background()) @@ -250,7 +234,10 @@ func testPacksApplySpecRoundtrip(t *testing.T, ds *Datastore) { assert.Equal(t, expectedSpecs, gotSpec) } -func testPacksGetSpec(t *testing.T, ds *Datastore) { +func TestGetPackSpec(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + expectedSpecs := setupPackSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -260,7 +247,10 @@ func testPacksGetSpec(t *testing.T, ds *Datastore) { } } -func testPacksApplySpecMissingQueries(t *testing.T, ds *Datastore) { +func TestApplyPackSpecMissingQueries(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -285,7 +275,10 @@ func testPacksApplySpecMissingQueries(t *testing.T, ds *Datastore) { } } -func testPacksApplySpecMissingName(t *testing.T, ds *Datastore) { +func TestApplyPackSpecMissingName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + setupPackSpecsTest(t, ds) specs := []*fleet.PackSpec{ @@ -311,7 +304,10 @@ func testPacksApplySpecMissingName(t *testing.T, ds *Datastore) { assert.Equal(t, "foo", spec.Queries[0].Name) } -func testPacksListForHost(t *testing.T, ds *Datastore) { +func TestListPacksForHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + mockClock := clock.NewMockClock() l1 := &fleet.LabelSpec{ @@ -420,7 +416,10 @@ func testPacksListForHost(t *testing.T, ds *Datastore) { } } -func testPacksEnsureGlobal(t *testing.T, ds *Datastore) { +func TestEnsureGlobalPack(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + test.AddAllHostsLabel(t, ds) packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) @@ -451,7 +450,10 @@ func testPacksEnsureGlobal(t *testing.T, ds *Datastore) { assert.Equal(t, "global", *gp.Type) } -func testPacksEnsureTeam(t *testing.T, ds *Datastore) { +func TestEnsureTeamPack(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) require.Nil(t, err) assert.Len(t, packs, 0) @@ -497,7 +499,10 @@ func testPacksEnsureTeam(t *testing.T, ds *Datastore) { assert.Equal(t, []uint{team2.ID}, tp2.TeamIDs) } -func testPacksTeamNameChangesTeamSchedule(t *testing.T, ds *Datastore) { +func TestTeamNameChangesTeamSchedule(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -516,7 +521,10 @@ func testPacksTeamNameChangesTeamSchedule(t *testing.T, ds *Datastore) { assert.Equal(t, teamScheduleName(team1), tp.Name) } -func testPacksTeamScheduleNamesMigrateToNewFormat(t *testing.T, ds *Datastore) { +func TestTeamScheduleNamesMigrateToNewFormat(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -539,7 +547,10 @@ func testPacksTeamScheduleNamesMigrateToNewFormat(t *testing.T, ds *Datastore) { require.Equal(t, teamScheduleName(team1), tp.Name) } -func testPacksApplySpecFailsOnTargetIDNull(t *testing.T, ds *Datastore) { +func TestApplyPackSpecFailsOnTargetIDNull(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -591,15 +602,17 @@ func randomPackStatsForHost(hostID, packID uint, scheduledQueries []*fleet.Sched } } -func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { +func TestPackApplyStatsNotLocking(t *testing.T) { t.Skip("This can be too much for the test db if you're running all tests") + ds := CreateMySQLDS(t) + defer ds.Close() + specs := setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -637,15 +650,17 @@ func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { cancelFunc() } -func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { +func TestPackApplyStatsNotLockingTryTwo(t *testing.T) { t.Skip("This can be too much for the test db if you're running all tests") + ds := CreateMySQLDS(t) + defer ds.Close() + setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/password_reset_test.go b/server/datastore/mysql/password_reset_test.go index ce9b2fcc40..71c0c5ca62 100644 --- a/server/datastore/mysql/password_reset_test.go +++ b/server/datastore/mysql/password_reset_test.go @@ -9,25 +9,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPasswordReset(t *testing.T) { - ds := CreateMySQLDS(t) +func TestPasswordResetRequests(t *testing.T) { + db := CreateMySQLDS(t) + defer db.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Requests", testPasswordResetRequests}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testPasswordResetRequests(t *testing.T, ds *Datastore) { - createTestUsers(t, ds) + createTestUsers(t, db) now := time.Now().UTC() tomorrow := now.Add(time.Hour * 24) var passwordResetTests = []struct { @@ -44,7 +30,7 @@ func testPasswordResetRequests(t *testing.T, ds *Datastore) { ExpiresAt: tt.expires, Token: tt.token, } - req, err := ds.NewPasswordResetRequest(context.Background(), r) + req, err := db.NewPasswordResetRequest(context.Background(), r) assert.Nil(t, err) assert.Equal(t, tt.userID, req.UserID) } diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 1b0a196d7e..256d11a1f5 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -22,30 +22,23 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, queryID uint) (*fleet. return nil, errors.Wrap(err, "getting last id after inserting policy") } - return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) + return policyDB(ctx, ds.writer, uint(lastIdInt64)) } func (ds *Datastore) Policy(ctx context.Context, id uint) (*fleet.Policy, error) { - return policyDB(ctx, ds.reader, id, nil) + return policyDB(ctx, ds.reader, id) } -func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint, teamID *uint) (*fleet.Policy, error) { - teamWhere := "TRUE" - args := []interface{}{id} - if teamID != nil { - teamWhere = "team_id = ?" - args = append(args, *teamID) - } - +func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.Policy, error) { var policy fleet.Policy err := sqlx.GetContext(ctx, q, &policy, - fmt.Sprintf(`SELECT + `SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=? AND %s`, teamWhere), - args...) + FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=?`, + id) if err != nil { return nil, errors.Wrap(err, "getting policy") } @@ -85,27 +78,17 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee } func (ds *Datastore) ListGlobalPolicies(ctx context.Context) ([]*fleet.Policy, error) { - return listPoliciesDB(ctx, ds.reader, nil) -} - -func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID *uint) ([]*fleet.Policy, error) { - teamWhere := "p.team_id is NULL" - var args []interface{} - if teamID != nil { - teamWhere = "p.team_id = ?" - args = append(args, *teamID) - } var policies []*fleet.Policy err := sqlx.SelectContext( ctx, - q, + ds.reader, &policies, - fmt.Sprintf(`SELECT + `SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE %s`, teamWhere), args..., + FROM policies p JOIN queries q ON (p.query_id=q.id)`, ) if err != nil { return nil, errors.Wrap(err, "listing policies") @@ -114,91 +97,33 @@ func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID *uint) ([ } func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { - return deletePolicyDB(ctx, ds.writer, ids, nil) -} - -func deletePolicyDB(ctx context.Context, q sqlx.ExtContext, ids []uint, teamID *uint) ([]uint, error) { - stmt := `DELETE FROM policies WHERE id IN (?) AND %s` + stmt := `DELETE FROM policies WHERE id IN (?)` stmt, args, err := sqlx.In(stmt, ids) if err != nil { return nil, errors.Wrap(err, "IN for DELETE FROM policies") } - stmt = q.Rebind(stmt) - - teamWhere := "TRUE" - if teamID != nil { - teamWhere = "team_id = ?" - args = append(args, *teamID) - } - - if _, err := q.ExecContext(ctx, fmt.Sprintf(stmt, teamWhere), args...); err != nil { + stmt = ds.writer.Rebind(stmt) + if _, err := ds.writer.ExecContext(ctx, stmt, args...); err != nil { return nil, errors.Wrap(err, "delete policies") } return ids, nil } -func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { - var globalRows, teamRows []struct { +func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, _ *fleet.Host) (map[string]string, error) { + var rows []struct { Id string `db:"id"` Query string `db:"query"` } - err := sqlx.SelectContext( - ctx, - ds.reader, - &globalRows, - `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id is NULL`, - ) + err := sqlx.SelectContext(ctx, ds.reader, &rows, `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id)`) if err != nil { return nil, errors.Wrap(err, "selecting policies for host") } results := map[string]string{} - if host.TeamID != nil { - err := sqlx.SelectContext( - ctx, - ds.reader, - &teamRows, - `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id = ?`, - *host.TeamID, - ) - if err != nil { - return nil, errors.Wrap(err, "selecting policies for host in team") - } - } - - for _, row := range globalRows { - results[row.Id] = row.Query - } - - for _, row := range teamRows { + for _, row := range rows { results[row.Id] = row.Query } return results, nil } - -func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { - res, err := ds.writer.ExecContext(ctx, `INSERT INTO policies (query_id, team_id) VALUES (?, ?)`, queryID, teamID) - if err != nil { - return nil, errors.Wrap(err, "inserting new team policy") - } - lastIdInt64, err := res.LastInsertId() - if err != nil { - return nil, errors.Wrap(err, "getting last id after inserting policy") - } - - return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) -} - -func (ds *Datastore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { - return listPoliciesDB(ctx, ds.reader, &teamID) -} - -func (ds *Datastore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { - return deletePolicyDB(ctx, ds.writer, ids, &teamID) -} - -func (ds *Datastore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { - return policyDB(ctx, ds.reader, policyID, &teamID) -} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 8897d98995..fca687aa4c 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -12,25 +12,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestPolicies(t *testing.T) { +func TestNewGlobalPolicy(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"NewGlobalPolicy", testPoliciesNewGlobalPolicy}, - {"MembershipView", testPoliciesMembershipView}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testPoliciesNewGlobalPolicy(t *testing.T, ds *Datastore) { q, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "query1", Description: "query1 desc", @@ -73,12 +58,14 @@ func testPoliciesNewGlobalPolicy(t *testing.T, ds *Datastore) { require.NoError(t, ds.DeleteQuery(context.Background(), q.Name)) } -func testPoliciesMembershipView(t *testing.T, ds *Datastore) { +func TestPolicyMembershipView(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -90,7 +77,6 @@ func testPoliciesMembershipView(t *testing.T, ds *Datastore) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -156,138 +142,9 @@ func testPoliciesMembershipView(t *testing.T, ds *Datastore) { require.NoError(t, err) assert.Equal(t, policies[0], policy) - queries, err := ds.PolicyQueriesForHost(context.Background(), host1) + queries, err := ds.PolicyQueriesForHost(context.Background(), nil) require.NoError(t, err) require.Len(t, queries, 2) assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) } - -func TestTeamPolicy(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - - team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) - require.NoError(t, err) - - q, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query1", - Description: "query1 desc", - Query: "select 1;", - Saved: true, - }) - require.NoError(t, err) - - team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) - require.NoError(t, err) - - q2, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query2", - Description: "query2 desc", - Query: "select 1;", - Saved: true, - }) - require.NoError(t, err) - - prevPolicies, err := ds.ListGlobalPolicies(context.Background()) - require.NoError(t, err) - - _, err = ds.NewTeamPolicy(context.Background(), 99999999, q.ID) - require.Error(t, err) - - p, err := ds.NewTeamPolicy(context.Background(), team1.ID, q.ID) - require.NoError(t, err) - - assert.Equal(t, "query1", p.QueryName) - - globalPolicies, err := ds.ListGlobalPolicies(context.Background()) - require.NoError(t, err) - require.Len(t, globalPolicies, len(prevPolicies)) - - _, err = ds.NewTeamPolicy(context.Background(), team2.ID, q2.ID) - require.NoError(t, err) - - teamPolicies, err := ds.ListTeamPolicies(context.Background(), team1.ID) - require.NoError(t, err) - require.Len(t, teamPolicies, 1) - assert.Equal(t, q.ID, teamPolicies[0].QueryID) - - team2Policies, err := ds.ListTeamPolicies(context.Background(), team2.ID) - require.NoError(t, err) - require.Len(t, team2Policies, 1) - assert.Equal(t, q2.ID, team2Policies[0].QueryID) - - _, err = ds.DeleteTeamPolicies(context.Background(), team1.ID, []uint{teamPolicies[0].ID}) - require.NoError(t, err) - - teamPolicies, err = ds.ListGlobalPolicies(context.Background()) - require.NoError(t, err) - require.Len(t, teamPolicies, 0) -} - -func TestPolicyQueriesForHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - - team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) - require.NoError(t, err) - - host1, err := ds.NewHost(context.Background(), &fleet.Host{ - OsqueryHostID: "1234", - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1", - UUID: "1", - Hostname: "foo.local", - }) - require.NoError(t, err) - - require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID})) - host1, err = ds.Host(context.Background(), host1.ID) - require.NoError(t, err) - - host2, err := ds.NewHost(context.Background(), &fleet.Host{ - OsqueryHostID: "5679", - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "2", - UUID: "2", - Hostname: "bar.local", - }) - require.NoError(t, err) - - q, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query1", - Description: "query1 desc", - Query: "select 1;", - Saved: true, - }) - require.NoError(t, err) - _, err = ds.NewGlobalPolicy(context.Background(), q.ID) - require.NoError(t, err) - - q2, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query2", - Description: "query2 desc", - Query: "select 42;", - Saved: true, - }) - require.NoError(t, err) - _, err = ds.NewTeamPolicy(context.Background(), team1.ID, q2.ID) - require.NoError(t, err) - - queries, err := ds.PolicyQueriesForHost(context.Background(), host1) - require.NoError(t, err) - require.Len(t, queries, 2) - assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) - assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) - - queries, err = ds.PolicyQueriesForHost(context.Background(), host2) - require.NoError(t, err) - require.Len(t, queries, 1) - assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) -} diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index 8a6eabeafd..56c98ad104 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -138,13 +138,13 @@ func (d *Datastore) SaveQuery(ctx context.Context, q *fleet.Query) error { // DeleteQuery deletes Query identified by Query.ID. func (d *Datastore) DeleteQuery(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, queriesTable, name) + return d.deleteEntityByName(ctx, "queries", name) } // DeleteQueries deletes the existing query objects with the provided IDs. The // number of deleted queries is returned along with any error. func (d *Datastore) DeleteQueries(ctx context.Context, ids []uint) (uint, error) { - return d.deleteEntities(ctx, queriesTable, ids) + return d.deleteEntities(ctx, "queries", ids) } // Query returns a single Query identified by id, if such exists. @@ -170,7 +170,7 @@ func (d *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { // ListQueries returns a list of queries with sort order and results limit // determined by passed in fleet.ListOptions -func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { +func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { sql := ` SELECT q.*, COALESCE(u.name, '') AS author_name FROM queries q @@ -178,11 +178,7 @@ func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ON q.author_id = u.id WHERE saved = true ` - if opt.OnlyObserverCanRun { - sql += " AND q.observer_can_run=true" - } - sql = appendListOptionsToSQL(sql, opt.ListOptions) - + sql = appendListOptionsToSQL(sql, opt) results := []*fleet.Query{} if err := sqlx.SelectContext(ctx, d.reader, &results, sql); err != nil { diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index 5217522ff5..593193c8b7 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -12,31 +12,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestQueries(t *testing.T) { +func TestApplyQueries(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Apply", testQueriesApply}, - {"Delete", testQueriesDelete}, - {"GetByName", testQueriesGetByName}, - {"DeleteMany", testQueriesDeleteMany}, - {"Save", testQueriesSave}, - {"List", testQueriesList}, - {"LoadPacksForQueries", testQueriesLoadPacksForQueries}, - {"DuplicateNew", testQueriesDuplicateNew}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testQueriesApply(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) @@ -50,7 +29,7 @@ func testQueriesApply(t *testing.T, ds *Datastore) { err := ds.ApplyQueries(context.Background(), zwass.ID, expectedQueries) require.Nil(t, err) - queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -68,7 +47,7 @@ func testQueriesApply(t *testing.T, ds *Datastore) { err = ds.ApplyQueries(context.Background(), groob.ID, expectedQueries) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -86,7 +65,7 @@ func testQueriesApply(t *testing.T, ds *Datastore) { err = ds.ApplyQueries(context.Background(), zwass.ID, []*fleet.Query{expectedQueries[2]}) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -100,7 +79,10 @@ func testQueriesApply(t *testing.T, ds *Datastore) { assert.Equal(t, &zwass.ID, queries[2].AuthorID) } -func testQueriesDelete(t *testing.T, ds *Datastore) { +func TestDeleteQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -121,7 +103,10 @@ func testQueriesDelete(t *testing.T, ds *Datastore) { assert.NotNil(t, err) } -func testQueriesGetByName(t *testing.T, ds *Datastore) { +func TestGetQueryByName(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) actual, err := ds.QueryByName(context.Background(), "q1") @@ -134,7 +119,10 @@ func testQueriesGetByName(t *testing.T, ds *Datastore) { assert.True(t, fleet.IsNotFound(err)) } -func testQueriesDeleteMany(t *testing.T, ds *Datastore) { +func TestDeleteQueries(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) q1 := test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) @@ -142,7 +130,7 @@ func testQueriesDeleteMany(t *testing.T, ds *Datastore) { q3 := test.NewQuery(t, ds, "q3", "select 1", user.ID, true) q4 := test.NewQuery(t, ds, "q4", "select * from osquery_info", user.ID, true) - queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) assert.Len(t, queries, 4) @@ -150,7 +138,7 @@ func testQueriesDeleteMany(t *testing.T, ds *Datastore) { require.Nil(t, err) assert.Equal(t, uint(2), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) assert.Len(t, queries, 2) @@ -158,7 +146,7 @@ func testQueriesDeleteMany(t *testing.T, ds *Datastore) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) assert.Len(t, queries, 1) @@ -166,12 +154,16 @@ func testQueriesDeleteMany(t *testing.T, ds *Datastore) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) require.Nil(t, err) assert.Len(t, queries, 0) + } -func testQueriesSave(t *testing.T, ds *Datastore) { +func TestSaveQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -198,7 +190,10 @@ func testQueriesSave(t *testing.T, ds *Datastore) { assert.True(t, queryVerify.ObserverCanRun) } -func testQueriesList(t *testing.T, ds *Datastore) { +func TestListQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) for i := 0; i < 10; i++ { @@ -220,13 +215,16 @@ func testQueriesList(t *testing.T, ds *Datastore) { }) require.Nil(t, err) - opts := fleet.ListQueryOptions{} + opts := fleet.ListOptions{} results, err := ds.ListQueries(context.Background(), opts) assert.Nil(t, err) assert.Equal(t, 10, len(results)) } -func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { +func TestLoadPacksForQueries(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "q1", Query: "select * from time"}, @@ -236,9 +234,9 @@ func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { require.Nil(t, err) specs := []*fleet.PackSpec{ - {Name: "p1"}, - {Name: "p2"}, - {Name: "p3"}, + &fleet.PackSpec{Name: "p1"}, + &fleet.PackSpec{Name: "p2"}, + &fleet.PackSpec{Name: "p3"}, } err = ds.ApplyPackSpecs(context.Background(), specs) require.Nil(t, err) @@ -252,10 +250,10 @@ func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - { + &fleet.PackSpec{ Name: "p2", Queries: []fleet.PackSpecQuery{ - { + fleet.PackSpecQuery{ Name: "q0", QueryName: queries[0].Name, Interval: 60, @@ -277,19 +275,19 @@ func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - { + &fleet.PackSpec{ Name: "p1", Queries: []fleet.PackSpecQuery{ - { + fleet.PackSpecQuery{ QueryName: queries[1].Name, Interval: 60, }, }, }, - { + &fleet.PackSpec{ Name: "p3", Queries: []fleet.PackSpecQuery{ - { + fleet.PackSpecQuery{ QueryName: queries[1].Name, Interval: 60, }, @@ -314,15 +312,15 @@ func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { } specs = []*fleet.PackSpec{ - { + &fleet.PackSpec{ Name: "p3", Queries: []fleet.PackSpecQuery{ - { + fleet.PackSpecQuery{ Name: "q0", QueryName: queries[0].Name, Interval: 60, }, - { + fleet.PackSpecQuery{ Name: "q1", QueryName: queries[1].Name, Interval: 60, @@ -350,7 +348,10 @@ func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { } } -func testQueriesDuplicateNew(t *testing.T, ds *Datastore) { +func TestDuplicateNewQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := test.NewUser(t, ds, "Mike Arpaia", "mike@fleet.co", true) q1, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "foo", @@ -369,40 +370,3 @@ func testQueriesDuplicateNew(t *testing.T, ds *Datastore) { // is private to the individual datastore implementations assert.Contains(t, err.Error(), "already exists") } - -func TestListQueryFiltersObserver(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - - _, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query1", - Query: "select 1;", - Saved: true, - }) - require.NoError(t, err) - _, err = ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query2", - Query: "select 1;", - Saved: true, - }) - require.NoError(t, err) - query3, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "query3", - Query: "select 1;", - Saved: true, - ObserverCanRun: true, - }) - require.NoError(t, err) - - queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) - require.NoError(t, err) - require.Len(t, queries, 3) - - queries, err = ds.ListQueries( - context.Background(), - fleet.ListQueryOptions{OnlyObserverCanRun: true, ListOptions: fleet.ListOptions{PerPage: 1}}, - ) - require.NoError(t, err) - require.Len(t, queries, 1) - assert.Equal(t, query3.ID, queries[0].ID) -} diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 560ce5bec9..8605f60b35 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -123,7 +123,7 @@ func saveScheduledQueryDB(ctx context.Context, exec sqlx.ExecerContext, sq *flee } func (d *Datastore) DeleteScheduledQuery(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, scheduledQueriesTable, id) + return d.deleteEntity(ctx, "scheduled_queries", id) } func (d *Datastore) ScheduledQuery(ctx context.Context, id uint) (*fleet.ScheduledQuery, error) { diff --git a/server/datastore/mysql/scheduled_queries_test.go b/server/datastore/mysql/scheduled_queries_test.go index a6268c665b..8345c08118 100644 --- a/server/datastore/mysql/scheduled_queries_test.go +++ b/server/datastore/mysql/scheduled_queries_test.go @@ -12,29 +12,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestScheduledQueries(t *testing.T) { +func TestListScheduledQueriesInPack(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"ListInPack", testScheduledQueriesListInPack}, - {"New", testScheduledQueriesNew}, - {"Get", testScheduledQueriesGet}, - {"Delete", testScheduledQueriesDelete}, - {"CascadingDelete", testScheduledQueriesCascadingDelete}, - {"CleanupOrphanStats", testScheduledQueriesCleanupOrphanStats}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testScheduledQueriesListInPack(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -100,7 +81,10 @@ func testScheduledQueriesListInPack(t *testing.T, ds *Datastore) { require.Len(t, gotQueries, 3) } -func testScheduledQueriesNew(t *testing.T, ds *Datastore) { +func TestNewScheduledQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -116,7 +100,10 @@ func testScheduledQueriesNew(t *testing.T, ds *Datastore) { assert.Equal(t, "select * from time;", query.Query) } -func testScheduledQueriesGet(t *testing.T, ds *Datastore) { +func TestScheduledQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -140,7 +127,10 @@ func testScheduledQueriesGet(t *testing.T, ds *Datastore) { assert.False(t, *query.Denylist) } -func testScheduledQueriesDelete(t *testing.T, ds *Datastore) { +func TestDeleteScheduledQuery(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -157,7 +147,10 @@ func testScheduledQueriesDelete(t *testing.T, ds *Datastore) { require.NotNil(t, err) } -func testScheduledQueriesCascadingDelete(t *testing.T, ds *Datastore) { +func TestCascadingDeletionOfQueries(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -206,7 +199,10 @@ func testScheduledQueriesCascadingDelete(t *testing.T, ds *Datastore) { require.Len(t, gotQueries, 1) } -func testScheduledQueriesCleanupOrphanStats(t *testing.T, ds *Datastore) { +func TestCleanupOrphanScheduledQueryStats(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 00f3bea508..8db6f3112e 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -221,7 +221,6 @@ CREATE TABLE `hosts` ( `team_id` int(10) unsigned DEFAULT NULL, `gigs_disk_space_available` float NOT NULL DEFAULT '0', `percent_disk_space_available` float NOT NULL DEFAULT '0', - `policy_updated_at` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', PRIMARY KEY (`id`), UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`), UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), @@ -269,7 +268,9 @@ CREATE TABLE `label_membership` ( `label_id` int(10) unsigned NOT NULL, `host_id` int(10) unsigned NOT NULL, PRIMARY KEY (`host_id`,`label_id`), - KEY `idx_lm_label_id` (`label_id`) + KEY `idx_lm_label_id` (`label_id`), + CONSTRAINT `fk_lm_host_id` FOREIGN KEY (`host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_lm_label_id` FOREIGN KEY (`label_id`) REFERENCES `labels` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; @@ -309,9 +310,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=105 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=102 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -399,12 +400,9 @@ CREATE TABLE `policies` ( `query_id` int(10) unsigned NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `team_id` int(10) unsigned DEFAULT NULL, PRIMARY KEY (`id`), KEY `fk_policies_query_id` (`query_id`), - KEY `fk_policies_team_id` (`team_id`), - CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`), - CONSTRAINT `policies_ibfk_2` FOREIGN KEY (`team_id`) REFERENCES `teams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; SET @saved_cs_client = @@character_set_client; diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index 02e8273beb..16fb65c8fb 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -70,7 +70,7 @@ func (d *Datastore) NewSession(ctx context.Context, session *fleet.Session) (*fl } func (d *Datastore) DestroySession(ctx context.Context, session *fleet.Session) error { - err := d.deleteEntity(ctx, sessionsTable, session.ID) + err := d.deleteEntity(ctx, "sessions", session.ID) if err != nil { return errors.Wrap(err, "deleting session") } diff --git a/server/datastore/mysql/sessions_test.go b/server/datastore/mysql/sessions_test.go index 446052e653..14c225a8a8 100644 --- a/server/datastore/mysql/sessions_test.go +++ b/server/datastore/mysql/sessions_test.go @@ -10,24 +10,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestSessions(t *testing.T) { +func TestSessionGetters(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Getters", testSessionsGetters}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testSessionsGetters(t *testing.T, ds *Datastore) { user, err := ds.NewUser(context.Background(), &fleet.User{ Password: []byte("supersecret"), Email: "other@bobcom", diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index c8d754e526..04952635bf 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -332,18 +332,11 @@ func (d *Datastore) AllSoftwareWithoutCPEIterator(ctx context.Context) (fleet.So } func (d *Datastore) AddCPEForSoftware(ctx context.Context, software fleet.Software, cpe string) error { - _, err := addCPEForSoftwareDB(ctx, d.writer, software, cpe) - return err -} - -func addCPEForSoftwareDB(ctx context.Context, exec sqlx.ExecerContext, software fleet.Software, cpe string) (uint, error) { sql := `INSERT INTO software_cpe (software_id, cpe) VALUES (?, ?)` - res, err := exec.ExecContext(ctx, sql, software.ID, cpe) - if err != nil { - return 0, errors.Wrap(err, "insert software cpe") + if _, err := d.writer.ExecContext(ctx, sql, software.ID, cpe); err != nil { + return errors.Wrap(err, "insert software cpe") } - id, _ := res.LastInsertId() // cannot fail with the mysql driver - return uint(id), nil + return nil } func (d *Datastore) AllCPEs(ctx context.Context) ([]string, error) { diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 099bdeb5a1..9e52730939 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -11,37 +11,14 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/test" - "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestSoftware(t *testing.T) { +func TestSaveHostSoftware(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"SaveHost", testSoftwareSaveHost}, - {"CPE", testSoftwareCPE}, - {"InsertCVEs", testSoftwareInsertCVEs}, - {"HostDuplicates", testSoftwareHostDuplicates}, - {"LoadVulnerabilities", testSoftwareLoadVulnerabilities}, - {"AllCPEs", testSoftwareAllCPEs}, - {"NothingChanged", testSoftwareNothingChanged}, - {"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs}, - {"List", testSoftwareList}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testSoftwareSaveHost(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) @@ -126,7 +103,10 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) { test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software) } -func testSoftwareCPE(t *testing.T, ds *Datastore) { +func TestSoftwareCPE(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft1 := fleet.HostSoftware{ @@ -196,7 +176,10 @@ func testSoftwareCPE(t *testing.T, ds *Datastore) { require.NoError(t, iterator.Close()) } -func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) { +func TestInsertCVEs(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -214,7 +197,10 @@ func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) { require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) } -func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { +func TestHostSoftwareDuplicates(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) longName := strings.Repeat("a", 260) @@ -246,7 +232,10 @@ func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { require.NoError(t, tx.Commit()) } -func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { +func TestLoadSoftwareVulnerabilities(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -281,7 +270,10 @@ func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { require.Len(t, host.Software[1].Vulnerabilities, 0) } -func testSoftwareAllCPEs(t *testing.T, ds *Datastore) { +func TestAllCPEs(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -304,7 +296,7 @@ func testSoftwareAllCPEs(t *testing.T, ds *Datastore) { assert.ElementsMatch(t, cpes, []string{"somecpe", "someothercpewithoutvulns"}) } -func testSoftwareNothingChanged(t *testing.T, ds *Datastore) { +func TestNothingChanged(t *testing.T) { assert.False(t, nothingChanged(nil, []fleet.Software{{}})) assert.True(t, nothingChanged(nil, nil)) assert.True(t, nothingChanged( @@ -321,7 +313,10 @@ func testSoftwareNothingChanged(t *testing.T, ds *Datastore) { )) } -func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { +func TestLoadSupportsTonsOfCVEs(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -337,26 +332,15 @@ func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) sort.Slice(host.Software, func(i, j int) bool { return host.Software[i].Name < host.Software[j].Name }) - + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) - - require.NoError(t, ds.withTx(context.Background(), func(tx sqlx.ExtContext) error { - somecpeID, err := addCPEForSoftwareDB(context.Background(), tx, host.Software[0], "somecpe") - if err != nil { - return err - } - sql := `INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES (?, ?)` - for i := 0; i < 1000; i++ { - part1 := rand.Intn(1000) - part2 := rand.Intn(1000) - part3 := rand.Intn(1000) - cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) - if _, err := tx.ExecContext(context.Background(), sql, somecpeID, cve); err != nil { - return err - } - } - return nil - })) + for i := 0; i < 1000; i++ { + part1 := rand.Intn(1000) + part2 := rand.Intn(1000) + part3 := rand.Intn(1000) + cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) + require.NoError(t, ds.InsertCVEForCPE(context.Background(), cve, []string{"somecpe"})) + } require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) @@ -379,7 +363,10 @@ func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { } } -func testSoftwareList(t *testing.T, ds *Datastore) { +func TestListSoftware(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index 34583b0ab7..201175e7c2 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -10,28 +10,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestStatistics(t *testing.T) { +func TestShouldSendStatistics(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"ShouldSend", testStatisticsShouldSend}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -63,7 +48,6 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err = ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index 1f2cd2000c..dabd73a76d 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -14,27 +14,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestTargets(t *testing.T) { +func TestCountHostsInTargets(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"CountHosts", testTargetsCountHosts}, - {"HostStatus", testTargetsHostStatus}, - {"HostIDsInTargets", testTargetsHostIDsInTargets}, - {"HostIDsInTargetsTeam", testTargetsHostIDsInTargetsTeam}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testTargetsCountHosts(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -47,7 +30,6 @@ func testTargetsCountHosts(t *testing.T, ds *Datastore) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), - PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, @@ -174,9 +156,13 @@ func testTargetsCountHosts(t *testing.T, ds *Datastore) { assert.Equal(t, uint(0), metrics.OnlineHosts) assert.Equal(t, uint(5), metrics.OfflineHosts) assert.Equal(t, uint(1), metrics.MissingInActionHosts) + } -func testTargetsHostStatus(t *testing.T, ds *Datastore) { +func TestHostStatus(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + test.AddAllHostsLabel(t, ds) mockClock := clock.NewMockClock() @@ -236,7 +222,10 @@ func testTargetsHostStatus(t *testing.T, ds *Datastore) { } } -func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { +func TestHostIDsInTargets(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -248,7 +237,6 @@ func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { NodeKey: strconv.Itoa(hostCount), DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) @@ -314,7 +302,10 @@ func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { assert.Equal(t, []uint{1, 3, 4, 5, 6}, ids) } -func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { +func TestHostIDsInTargetsTeam(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -327,7 +318,6 @@ func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), - PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, diff --git a/server/datastore/mysql/teams.go b/server/datastore/mysql/teams.go index 183f6c4dac..122a734cc5 100644 --- a/server/datastore/mysql/teams.go +++ b/server/datastore/mysql/teams.go @@ -80,7 +80,7 @@ func saveTeamSecretsDB(ctx context.Context, exec sqlx.ExecerContext, team *fleet } func (d *Datastore) DeleteTeam(ctx context.Context, tid uint) error { - if err := d.deleteEntity(ctx, teamsTable, tid); err != nil { + if err := d.deleteEntity(ctx, "teams", tid); err != nil { return errors.Wrapf(err, "delete team id %d", tid) } return nil diff --git a/server/datastore/mysql/teams_test.go b/server/datastore/mysql/teams_test.go index 744299f0b9..21bb004915 100644 --- a/server/datastore/mysql/teams_test.go +++ b/server/datastore/mysql/teams_test.go @@ -13,28 +13,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestTeams(t *testing.T) { +func TestTeamGetSetDelete(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"GetSetDelete", testTeamsGetSetDelete}, - {"Users", testTeamsUsers}, - {"List", testTeamsList}, - {"Search", testTeamsSearch}, - {"EnrollSecrets", testTeamsEnrollSecrets}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testTeamsGetSetDelete(t *testing.T, ds *Datastore) { var createTests = []struct { name, description string }{ @@ -43,7 +25,7 @@ func testTeamsGetSetDelete(t *testing.T, ds *Datastore) { } for _, tt := range createTests { - t.Run(tt.name, func(t *testing.T) { + t.Run("", func(t *testing.T) { team, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: tt.name, Description: tt.description, @@ -70,7 +52,10 @@ func testTeamsGetSetDelete(t *testing.T, ds *Datastore) { } } -func testTeamsUsers(t *testing.T, ds *Datastore) { +func TestTeamUsers(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID} @@ -126,7 +111,10 @@ func testTeamsUsers(t *testing.T, ds *Datastore) { assert.ElementsMatch(t, team2Users, team2.Users) } -func testTeamsList(t *testing.T, ds *Datastore) { +func TestTeamListTeams(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID, GlobalRole: ptr.String(fleet.RoleAdmin)} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID, GlobalRole: ptr.String(fleet.RoleObserver)} @@ -180,7 +168,10 @@ func testTeamsList(t *testing.T, ds *Datastore) { assert.Equal(t, 1, teams[1].UserCount) } -func testTeamsSearch(t *testing.T, ds *Datastore) { +func TestTeamSearchTeams(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -216,7 +207,10 @@ func testTeamsSearch(t *testing.T, ds *Datastore) { assert.Len(t, teams, 0) } -func testTeamsEnrollSecrets(t *testing.T, ds *Datastore) { +func TestTeamEnrollSecrets(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + secrets := []*fleet.EnrollSecret{{Secret: "secret1"}, {Secret: "secret2"}} team1, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: "team1", diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index e90a9a3302..6d84ca5e8a 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -233,9 +233,7 @@ func createMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto strings.TrimPrefix(details.Name(), "github.com/fleetdm/fleet/v4/"), "/", "_", ) cleanName = strings.ReplaceAll(cleanName, ".", "_") - ds := initializeDatabase(t, cleanName, opts) - t.Cleanup(func() { ds.Close() }) - return ds + return initializeDatabase(t, cleanName, opts) } func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datastore { @@ -245,71 +243,3 @@ func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto func CreateMySQLDS(t *testing.T) *Datastore { return createMySQLDSWithOptions(t, nil) } - -func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { - if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { - t.Skip("MySQL tests are disabled") - } - - t.Parallel() - ds := initializeDatabase(t, name, new(DatastoreTestOptions)) - t.Cleanup(func() { ds.Close() }) - return ds -} - -// TruncateTables truncates the specified tables, in order, using ds.writer. -// Note that the order is typically not important because FK checks are -// disabled while truncating. If no table is provided, all tables (except -// those that are seeded by the SQL schema file) are truncated. -func TruncateTables(t *testing.T, ds *Datastore, tables ...string) { - // those tables are seeded with the schema.sql and as such must not - // be truncated - a more precise approach must be used for those, e.g. - // delete where id > max before test, or something like that. - nonEmptyTables := map[string]bool{ - "app_config_json": true, - "app_configs": true, - "migration_status_tables": true, - "osquery_options": true, - } - - ctx := context.Background() - - require.NoError(t, ds.withTx(ctx, func(tx sqlx.ExtContext) error { - var skipSeeded bool - - if len(tables) == 0 { - skipSeeded = true - sql := ` - SELECT - table_name - FROM - information_schema.tables - WHERE - table_schema = database() AND - table_type = 'BASE TABLE' - ` - if err := sqlx.SelectContext(ctx, tx, &tables, sql); err != nil { - return err - } - } - - if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=0`); err != nil { - return err - } - for _, tbl := range tables { - if nonEmptyTables[tbl] { - if skipSeeded { - continue - } - return fmt.Errorf("cannot truncate table %s, it contains seed data from schema.sql", tbl) - } - if _, err := tx.ExecContext(ctx, "TRUNCATE TABLE "+tbl); err != nil { - return err - } - } - if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=1`); err != nil { - return err - } - return nil - })) -} diff --git a/server/datastore/mysql/unicode_test.go b/server/datastore/mysql/unicode_test.go index a06a991bfa..64bfb3adb7 100644 --- a/server/datastore/mysql/unicode_test.go +++ b/server/datastore/mysql/unicode_test.go @@ -33,7 +33,6 @@ func TestUnicode(t *testing.T) { Hostname: "🍌", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go index 4f801295ca..29399b1114 100644 --- a/server/datastore/mysql/users.go +++ b/server/datastore/mysql/users.go @@ -267,5 +267,5 @@ func saveTeamsForUserDB(ctx context.Context, tx sqlx.ExtContext, user *fleet.Use // DeleteUser deletes the associated user func (d *Datastore) DeleteUser(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, usersTable, id) + return d.deleteEntity(ctx, "users", id) } diff --git a/server/datastore/mysql/users_test.go b/server/datastore/mysql/users_test.go index 240e2a338e..86562c97f6 100644 --- a/server/datastore/mysql/users_test.go +++ b/server/datastore/mysql/users_test.go @@ -15,30 +15,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestUsers(t *testing.T) { +func TestCreateUser(t *testing.T) { ds := CreateMySQLDS(t) + defer ds.Close() - cases := []struct { - name string - fn func(t *testing.T, ds *Datastore) - }{ - {"Create", testUsersCreate}, - {"ByID", testUsersByID}, - {"Save", testUsersSave}, - {"List", testUsersList}, - {"Teams", testUsersTeams}, - {"CreateWithTeams", testUsersCreateWithTeams}, - {"SaveMany", testUsersSaveMany}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - defer TruncateTables(t, ds) - c.fn(t, ds) - }) - } -} - -func testUsersCreate(t *testing.T, ds *Datastore) { var createTests = []struct { password, email string isAdmin, passwordReset, sso bool @@ -68,7 +48,10 @@ func testUsersCreate(t *testing.T, ds *Datastore) { } } -func testUsersByID(t *testing.T, ds *Datastore) { +func TestUserByID(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + users := createTestUsers(t, ds) for _, tt := range users { returned, err := ds.UserByID(context.Background(), tt.ID) @@ -109,7 +92,10 @@ func createTestUsers(t *testing.T, ds fleet.Datastore) []*fleet.User { return users } -func testUsersSave(t *testing.T, ds *Datastore) { +func TestSaveUser(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + users := createTestUsers(t, ds) testUserGlobalRole(t, ds, users) testEmailAttribute(t, ds, users) @@ -164,7 +150,10 @@ func testUserGlobalRole(t *testing.T, ds fleet.Datastore, users []*fleet.User) { assert.Equal(t, "Cannot specify both Global Role and Team Roles", flErr.Message) } -func testUsersList(t *testing.T, ds *Datastore) { +func TestListUsers(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + createTestUsers(t, ds) users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) @@ -182,7 +171,10 @@ func testUsersList(t *testing.T, ds *Datastore) { assert.Equal(t, "mike@fleet.co", users[0].Email) } -func testUsersTeams(t *testing.T, ds *Datastore) { +func TestUserTeams(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -272,7 +264,10 @@ func testUsersTeams(t *testing.T, ds *Datastore) { assert.Len(t, users[1].Teams, 0) } -func testUsersCreateWithTeams(t *testing.T, ds *Datastore) { +func TestUserCreateWithTeams(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -311,7 +306,10 @@ func testUsersCreateWithTeams(t *testing.T, ds *Datastore) { assert.Equal(t, "maintainer", user.Teams[2].Role) } -func testUsersSaveMany(t *testing.T, ds *Datastore) { +func TestSaveUsers(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + u1 := test.NewUser(t, ds, t.Name()+"Admin1", t.Name()+"admin1@fleet.co", true) u2 := test.NewUser(t, ds, t.Name()+"Admin2", t.Name()+"admin2@fleet.co", true) u3 := test.NewUser(t, ds, t.Name()+"Admin3", t.Name()+"admin3@fleet.co", true) diff --git a/server/datastore/redis/redis.go b/server/datastore/redis/redis.go index 3679b7a61a..d9046af6e8 100644 --- a/server/datastore/redis/redis.go +++ b/server/datastore/redis/redis.go @@ -1,11 +1,9 @@ package redis import ( - "net" "strings" "time" - "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" @@ -19,64 +17,29 @@ type standalonePool struct { addr string } -func (p *standalonePool) ConfigureDoer(conn redis.Conn) redis.Conn { - return conn -} - func (p *standalonePool) Stats() map[string]redis.PoolStats { return map[string]redis.PoolStats{ p.addr: p.Pool.Stats(), } } -type clusterPool struct { - *redisc.Cluster - followRedirs bool -} - -// ConfigureDoer configures conn to follow redirections if the redis -// configuration requested it. If the conn is already in error, or -// if it is not a redisc cluster connection, it is returned unaltered. -func (p *clusterPool) ConfigureDoer(conn redis.Conn) redis.Conn { - if err := conn.Err(); err == nil && p.followRedirs { - rc, err := redisc.RetryConn(conn, 3, 300*time.Millisecond) - if err == nil { - return rc - } - } - return conn -} - -// PoolConfig holds the redis pool configuration options. -type PoolConfig struct { - Server string - Password string - Database int - UseTLS bool - ConnTimeout time.Duration - KeepAlive time.Duration - ConnectRetryAttempts int - ClusterFollowRedirections bool - - // allows for testing dial retries and other dial-related scenarios - testRedisDialFunc func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) -} - // NewRedisPool creates a Redis connection pool using the provided server // address, password and database. -func NewRedisPool(config PoolConfig) (fleet.RedisPool, error) { - cluster := newCluster(config) +func NewRedisPool( + server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration, +) (fleet.RedisPool, error) { + cluster := newCluster(server, password, database, useTLS, connTimeout, keepAlive) if err := cluster.Refresh(); err != nil { if isClusterDisabled(err) || isClusterCommandUnknown(err) { // not a Redis Cluster setup, use a standalone Redis pool - pool, _ := cluster.CreatePool(config.Server) + pool, _ := cluster.CreatePool(server) cluster.Close() - return &standalonePool{pool, config.Server}, nil + return &standalonePool{pool, server}, nil } return nil, errors.Wrap(err, "refresh cluster") } - return &clusterPool{cluster, config.ClusterFollowRedirections}, nil + return cluster, nil } // SplitRedisKeysBySlot takes a list of redis keys and groups them by hash slot @@ -86,7 +49,7 @@ func NewRedisPool(config PoolConfig) (fleet.RedisPool, error) { // simply returns all keys in the same group (i.e. the top-level slice has a // length of 1). func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { - if _, isCluster := pool.(*clusterPool); isCluster { + if _, isCluster := pool.(*redisc.Cluster); isCluster { return redisc.SplitBySlot(keys...) } return [][]string{keys} @@ -98,7 +61,7 @@ func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { // of nodes stops and EachRedisNode returns that error. For standalone redis, // fn is called only once. func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { - if cluster, isCluster := pool.(*clusterPool); isCluster { + if cluster, isCluster := pool.(*redisc.Cluster); isCluster { return cluster.EachNode(false, func(_ string, conn redis.Conn) error { return fn(conn) }) @@ -109,64 +72,35 @@ func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { return fn(conn) } -func newCluster(config PoolConfig) *redisc.Cluster { - opts := []redis.DialOption{ - redis.DialDatabase(config.Database), - redis.DialUseTLS(config.UseTLS), - redis.DialConnectTimeout(config.ConnTimeout), - redis.DialKeepAlive(config.KeepAlive), - // Read/Write timeouts not set here because we may see results - // only rarely on the pub/sub channel. - } - if config.Password != "" { - opts = append(opts, redis.DialPassword(config.Password)) - } - - dialFn := redis.Dial - if config.testRedisDialFunc != nil { - dialFn = config.testRedisDialFunc - } - +func newCluster(server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration) *redisc.Cluster { return &redisc.Cluster{ - StartupNodes: []string{config.Server}, - CreatePool: func(server string, _ ...redis.DialOption) (*redis.Pool, error) { + StartupNodes: []string{server}, + CreatePool: func(server string, opts ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ MaxIdle: 3, IdleTimeout: 240 * time.Second, - Dial: func() (redis.Conn, error) { - var conn redis.Conn - op := func() error { - c, err := dialFn("tcp", server, opts...) - - var netErr net.Error - if errors.As(err, &netErr) { - if netErr.Temporary() || netErr.Timeout() { - // retryable error - return err - } - } - if err != nil { - // at this point, this is a non-retryable error - return backoff.Permanent(err) - } - - // success, store the connection to use - conn = c - return nil - } - - if config.ConnectRetryAttempts > 0 { - boff := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(config.ConnectRetryAttempts)) - if err := backoff.Retry(op, boff); err != nil { - return nil, err - } - } else if err := op(); err != nil { + c, err := redis.Dial( + "tcp", + server, + redis.DialDatabase(database), + redis.DialUseTLS(useTLS), + redis.DialConnectTimeout(connTimeout), + redis.DialKeepAlive(keepAlive), + // Read/Write timeouts not set here because we may see results + // only rarely on the pub/sub channel. + ) + if err != nil { return nil, err } - return conn, nil + if password != "" { + if _, err := c.Do("AUTH", password); err != nil { + c.Close() + return nil, err + } + } + return c, err }, - TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Minute { return nil diff --git a/server/datastore/redis/redis_test.go b/server/datastore/redis/redis_test.go index 30b55a3c16..ad3da7d6b5 100644 --- a/server/datastore/redis/redis_test.go +++ b/server/datastore/redis/redis_test.go @@ -2,171 +2,15 @@ package redis import ( "fmt" - "io" - "runtime" "testing" "time" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" - "github.com/pkg/errors" "github.com/stretchr/testify/require" ) -type netError struct { - error - timeout bool - temporary bool - allowedCalls int // once this reaches 0, mockDial does not return an error - countCalls int -} - -func (t *netError) Timeout() bool { return t.timeout } -func (t *netError) Temporary() bool { return t.temporary } - -var errFromConn = errors.New("SUCCESS") - -type redisConn struct{} - -func (redisConn) Close() error { return errFromConn } -func (redisConn) Err() error { return errFromConn } -func (redisConn) Do(_ string, _ ...interface{}) (interface{}, error) { return nil, errFromConn } -func (redisConn) Send(_ string, _ ...interface{}) error { return errFromConn } -func (redisConn) Flush() error { return errFromConn } -func (redisConn) Receive() (interface{}, error) { return nil, errFromConn } - -func TestConnectRetry(t *testing.T) { - mockDial := func(err error) func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { - return func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { - var ne *netError - if errors.As(err, &ne) { - ne.countCalls++ - if ne.allowedCalls <= 0 { - return redisConn{}, nil - } - ne.allowedCalls-- - } - return nil, err - } - } - - cases := []struct { - err error - retries int - wantCalls int - min, max time.Duration - }{ - // the min-max time intervals are based on the backoff default configuration as - // used in the Dial func of the redis pool. It starts with 500ms interval, - // multiplies by 1.5 on each attempt, and has a randomization of 0.5 that must - // be accounted for. Example ranges of intervals are given at - // https://github.com/fleetdm/fleet/pull/1962#issue-729635664 - // and were used to calculate the (approximate) expected range. - { - io.EOF, 0, 1, 0, 100 * time.Millisecond, - }, // non-retryable, no retry configured - { - &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 0, 1, 0, 100 * time.Millisecond, - }, // retryable, but no retry configured - { - io.EOF, 3, 1, 0, 100 * time.Millisecond, - }, // non-retryable, retry configured - { - &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, - }, // retryable, retry configured - { - &netError{error: io.EOF, temporary: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, - }, // retryable, retry configured - { - &netError{error: io.EOF, allowedCalls: 10}, 2, 1, 0, 100 * time.Millisecond, - }, // net error, but non-retryable - { - &netError{error: io.EOF, timeout: true, allowedCalls: 1}, 10, 2, 250 * time.Millisecond, 750 * time.Millisecond, - }, // retryable, but succeeded after one retry - } - for _, c := range cases { - t.Run(c.err.Error(), func(t *testing.T) { - start := time.Now() - _, err := NewRedisPool(PoolConfig{ - Server: "127.0.0.1:12345", - ConnectRetryAttempts: c.retries, - testRedisDialFunc: mockDial(c.err), - }) - diff := time.Since(start) - require.GreaterOrEqual(t, diff, c.min) - require.LessOrEqual(t, diff, c.max) - require.Error(t, err) - - wantErr := io.EOF - var ne *netError - if errors.As(c.err, &ne) { - require.Equal(t, c.wantCalls, ne.countCalls) - if ne.allowedCalls == 0 { - wantErr = errFromConn - } - } else { - require.Equal(t, c.wantCalls, 1) - } - - // the error is returned as part of the cluster.Refresh error, hence the - // check with Contains. - require.Contains(t, err.Error(), wantErr.Error()) - }) - } -} - -func TestRedisPoolConfigureDoer(t *testing.T) { - const prefix = "TestRedisPoolConfigureDoer:" - - t.Run("standalone", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, false, false) - defer teardown() - - c1 := pool.Get() - defer c1.Close() - c2 := pool.ConfigureDoer(pool.Get()) - defer c2.Close() - - // both conns work equally well, get nil because keys do not exist, - // but no redirection error (this is standalone redis). - _, err := redis.String(c1.Do("GET", prefix+"{a}")) - require.Equal(t, redis.ErrNil, err) - _, err = redis.String(c1.Do("GET", prefix+"{b}")) - require.Equal(t, redis.ErrNil, err) - - _, err = redis.String(c2.Do("GET", prefix+"{a}")) - require.Equal(t, redis.ErrNil, err) - _, err = redis.String(c2.Do("GET", prefix+"{b}")) - require.Equal(t, redis.ErrNil, err) - }) - - t.Run("cluster", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, true, true) - defer teardown() - - c1 := pool.Get() - defer c1.Close() - c2 := pool.ConfigureDoer(pool.Get()) - defer c2.Close() - - // unconfigured conn gets MOVED error on the second key - // (it is bound to {a}, {b} is on a different node) - _, err := redis.String(c1.Do("GET", prefix+"{a}")) - require.Equal(t, redis.ErrNil, err) - _, err = redis.String(c1.Do("GET", prefix+"{b}")) - rerr := redisc.ParseRedir(err) - require.Error(t, rerr) - require.Equal(t, "MOVED", rerr.Type) - - // configured conn gets the nil value, it redirected automatically - _, err = redis.String(c2.Do("GET", prefix+"{a}")) - require.Equal(t, redis.ErrNil, err) - _, err = redis.String(c2.Do("GET", prefix+"{b}")) - require.Equal(t, redis.ErrNil, err) - }) -} - func TestEachRedisNode(t *testing.T) { const prefix = "TestEachRedisNode:" @@ -205,23 +49,19 @@ func TestEachRedisNode(t *testing.T) { } t.Run("standalone", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, false, false) + pool, teardown := setupRedisForTest(t, false) defer teardown() runTest(t, pool) }) t.Run("cluster", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, true, false) + pool, teardown := setupRedisForTest(t, true) defer teardown() runTest(t, pool) }) } -func setupRedisForTest(t *testing.T, cluster, redir bool) (pool fleet.RedisPool, teardown func()) { - if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { - t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) - } - +func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardown func()) { var ( addr = "127.0.0.1:" password = "" @@ -234,15 +74,7 @@ func setupRedisForTest(t *testing.T, cluster, redir bool) (pool fleet.RedisPool, } addr += port - pool, err := NewRedisPool(PoolConfig{ - Server: addr, - Password: password, - Database: database, - UseTLS: useTLS, - ConnTimeout: 5 * time.Second, - KeepAlive: 10 * time.Second, - ClusterFollowRedirections: redir, - }) + pool, err := NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) conn := pool.Get() diff --git a/server/fleet/app.go b/server/fleet/app.go index 93f7644607..043dc0259d 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -243,12 +243,6 @@ type ListOptions struct { MatchQuery string } -type ListQueryOptions struct { - ListOptions - - OnlyObserverCanRun bool -} - // EnrollSecret contains information about an enroll secret, name, and active // status. Enroll secrets are used for osquery authentication. type EnrollSecret struct { @@ -322,7 +316,6 @@ type Logging struct { type UpdateIntervalConfig struct { OSQueryDetail time.Duration `json:"osquery_detail"` - OSQueryPolicy time.Duration `json:"osquery_policy"` } type LoggingPlugin struct { diff --git a/server/fleet/authz.go b/server/fleet/authz.go index 118dcce871..94cda02869 100644 --- a/server/fleet/authz.go +++ b/server/fleet/authz.go @@ -11,6 +11,4 @@ const ( ActionWriteRole = "write_role" // ActionRun is the action for running a live query. ActionRun = "run" - // ActionRunNew is the action for running a new live query. - ActionRunNew = "run_new" ) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 50b960b84a..c333c010a4 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -61,7 +61,7 @@ type Datastore interface { Query(ctx context.Context, id uint) (*Query, error) // ListQueries returns a list of queries with the provided sorting and paging options. Associated packs should also // be loaded. - ListQueries(ctx context.Context, opt ListQueryOptions) ([]*Query, error) + ListQueries(ctx context.Context, opt ListOptions) ([]*Query, error) // QueryByName looks up a query by name. QueryByName(ctx context.Context, name string, opts ...OptionalArg) (*Query, error) @@ -139,9 +139,10 @@ type Datastore interface { Label(ctx context.Context, lid uint) (*Label, error) ListLabels(ctx context.Context, filter TeamFilter, opt ListOptions) ([]*Label, error) - // LabelQueriesForHost returns the label queries that should be executed for the given host. - // Results are returned in a map of label id -> query - LabelQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) + // LabelQueriesForHost returns the label queries that should be executed for the given host. The cutoff is the + // minimum timestamp a query execution should have to be considered "fresh". Executions that are not fresh will be + // repeated. Results are returned in a map of label id -> query + LabelQueriesForHost(ctx context.Context, host *Host, cutoff time.Time) (map[string]string, error) // RecordLabelQueryExecutions saves the results of label queries. The results map is a map of label id -> whether or // not the label matches. The time parameter is the timestamp to save with the query execution. @@ -286,7 +287,6 @@ type Datastore interface { DeleteScheduledQuery(ctx context.Context, id uint) error ScheduledQuery(ctx context.Context, id uint) (*ScheduledQuery, error) CleanupOrphanScheduledQueryStats(ctx context.Context) error - CleanupOrphanLabelMembership(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// // TeamStore @@ -331,8 +331,7 @@ type Datastore interface { RecordStatisticsSent(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// - // GlobalPoliciesStore - + // GlobalPoliciesStore interface { NewGlobalPolicy(ctx context.Context, queryID uint) (*Policy, error) Policy(ctx context.Context, id uint) (*Policy, error) RecordPolicyQueryExecutions(ctx context.Context, host *Host, results map[uint]*bool, updated time.Time) error @@ -350,14 +349,6 @@ type Datastore interface { MigrationStatus(ctx context.Context) (MigrationStatus, error) ListSoftware(ctx context.Context, teamId *uint, opt ListOptions) ([]Software, error) - - /////////////////////////////////////////////////////////////////////////////// - // Team Policies - - NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) - ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) - DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) - TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } type MigrationStatus int diff --git a/server/fleet/global_policies.go b/server/fleet/global_policies.go index 7034f0fd2b..e41bacb87d 100644 --- a/server/fleet/global_policies.go +++ b/server/fleet/global_policies.go @@ -6,11 +6,10 @@ type Policy struct { QueryName string `json:"query_name" db:"query_name"` PassingHostCount uint `json:"passing_host_count" db:"passing_host_count"` FailingHostCount uint `json:"failing_host_count" db:"failing_host_count"` - TeamID *uint `json:"team_id" db:"team_id"` UpdateCreateTimestamps } -func (p Policy) AuthzType() string { +func (Policy) AuthzType() string { return "policy" } diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 5ef79d568f..5c0e19f1d9 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -64,7 +64,6 @@ type Host struct { OsqueryHostID string `json:"-" db:"osquery_host_id"` DetailUpdatedAt time.Time `json:"detail_updated_at" db:"detail_updated_at"` // Time that the host details were last updated LabelUpdatedAt time.Time `json:"label_updated_at" db:"label_updated_at"` // Time that the host labels were last updated - PolicyUpdatedAt time.Time `json:"policy_updated_at" db:"policy_updated_at"` // Time that the host policies were last updated LastEnrolledAt time.Time `json:"last_enrolled_at" db:"last_enrolled_at"` // Time that the host last enrolled SeenTime time.Time `json:"seen_time" db:"seen_time"` // Time that the host was last "seen" RefetchRequested bool `json:"refetch_requested" db:"refetch_requested"` diff --git a/server/fleet/redis_pool.go b/server/fleet/redis_pool.go index e1d86daf2c..18fa088b77 100644 --- a/server/fleet/redis_pool.go +++ b/server/fleet/redis_pool.go @@ -5,17 +5,7 @@ import "github.com/gomodule/redigo/redis" // RedisPool is the common interface for redigo's Pool for standalone Redis // and redisc's Cluster for Redis Cluster. type RedisPool interface { - // Get returns a redis connection. It must always be closed after use. Get() redis.Conn - - // Close closes the redis connection. Close() error - - // Stats returns a map of redis pool statistics for each server address. Stats() map[string]redis.PoolStats - - // ConfigureDoer returns a redis connection that is properly configured - // to execute Do commands. This should only be called when the actions - // to execute are all done with conn.Do. - ConfigureDoer(redis.Conn) redis.Conn } diff --git a/server/fleet/service.go b/server/fleet/service.go index 8c7f4568be..3f4e8c2b91 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -358,7 +358,6 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // ActivitiesService - ListActivities(ctx context.Context, opt ListOptions) ([]*Activity, error) /////////////////////////////////////////////////////////////////////////////// @@ -369,7 +368,6 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // GlobalScheduleService - GlobalScheduleQuery(ctx context.Context, sq *ScheduledQuery) (*ScheduledQuery, error) GetGlobalScheduledQueries(ctx context.Context, opts ListOptions) ([]*ScheduledQuery, error) ModifyGlobalScheduledQueries(ctx context.Context, id uint, q ScheduledQueryPayload) (*ScheduledQuery, error) @@ -377,12 +375,10 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // TranslatorService - Translate(ctx context.Context, payloads []TranslatePayload) ([]TranslatePayload, error) /////////////////////////////////////////////////////////////////////////////// // TeamScheduleService - TeamScheduleQuery(ctx context.Context, teamID uint, sq *ScheduledQuery) (*ScheduledQuery, error) GetTeamScheduledQueries(ctx context.Context, teamID uint, opts ListOptions) ([]*ScheduledQuery, error) ModifyTeamScheduledQueries( @@ -402,12 +398,4 @@ type Service interface { // Software ListSoftware(ctx context.Context, teamID *uint, opt ListOptions) ([]Software, error) - - /////////////////////////////////////////////////////////////////////////////// - // Team Policies - - NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) - ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) - DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) - GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } diff --git a/server/live_query/redis_live_query.go b/server/live_query/redis_live_query.go index a54e480b37..b81b88b3fb 100644 --- a/server/live_query/redis_live_query.go +++ b/server/live_query/redis_live_query.go @@ -194,7 +194,7 @@ func (r *redisLiveQuery) RunQuery(name, sql string, hostIDs []uint) error { } func (r *redisLiveQuery) StopQuery(name string) error { - conn := r.pool.ConfigureDoer(r.pool.Get()) + conn := r.pool.Get() defer conn.Close() targetKey, sqlKey := generateKeys(name) @@ -279,7 +279,7 @@ func (r *redisLiveQuery) collectBatchQueriesForHost(hostID uint, queryKeys []str } func (r *redisLiveQuery) QueryCompletedByHost(name string, hostID uint) error { - conn := r.pool.ConfigureDoer(r.pool.Get()) + conn := r.pool.Get() defer conn.Close() targetKey, _ := generateKeys(name) diff --git a/server/live_query/redis_live_query_test.go b/server/live_query/redis_live_query_test.go index 5a13411b32..bdabe609f2 100644 --- a/server/live_query/redis_live_query_test.go +++ b/server/live_query/redis_live_query_test.go @@ -1,7 +1,6 @@ package live_query import ( - "runtime" "testing" "time" @@ -100,10 +99,6 @@ func TestMigrateKeys(t *testing.T) { } func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, teardown func()) { - if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { - t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) - } - var ( addr = "127.0.0.1:" password = "" @@ -116,14 +111,7 @@ func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, tea } addr += port - pool, err := redis.NewRedisPool(redis.PoolConfig{ - Server: addr, - Password: password, - Database: database, - UseTLS: useTLS, - ConnTimeout: 5 * time.Second, - KeepAlive: 10 * time.Second, - }) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) store = NewRedisLiveQuery(pool) diff --git a/server/logging/filesystem.go b/server/logging/filesystem.go index 5b9a7e98bf..ac82ade98b 100644 --- a/server/logging/filesystem.go +++ b/server/logging/filesystem.go @@ -24,42 +24,35 @@ type filesystemLogWriter struct { // NewFilesystemLogWriter creates a log file for osquery status/result logs. // The logFile can be rotated by sending a `SIGHUP` signal to Fleet if // enableRotation is true -// -// The enableCompression argument is only used when enableRotation is true. func NewFilesystemLogWriter(path string, appLogger log.Logger, enableRotation bool, enableCompression bool) (*filesystemLogWriter, error) { - // Fail early if the process does not have the necessary - // permissions to open the file at path. - file, err := openFile(path) - if err != nil { - return nil, errors.Wrap(err, "perm check") - } - if !enableRotation { - // no log rotation, use "raw" bufio implementation - return &filesystemLogWriter{ - writer: newRawLogWriter(file), - }, nil - } - // Use lumberjack logger that supports rotation - file.Close() - osquerydLogger := &lumberjack.Logger{ - Filename: path, - MaxSize: 500, // megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: enableCompression, - } - appLogger = log.With(appLogger, "component", "osqueryd-logger") - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGHUP) - go func() { - for { - <-sig //block on signal - if err := osquerydLogger.Rotate(); err != nil { - appLogger.Log("err", err) - } + if enableRotation { + // Use lumberjack logger that supports rotation + osquerydLogger := &lumberjack.Logger{ + Filename: path, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: enableCompression, } - }() - return &filesystemLogWriter{osquerydLogger}, nil + appLogger = log.With(appLogger, "component", "osqueryd-logger") + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP) + go func() { + for { + <-sig //block on signal + if err := osquerydLogger.Rotate(); err != nil { + appLogger.Log("err", err) + } + } + }() + return &filesystemLogWriter{osquerydLogger}, nil + } + // no log rotation, use "raw" bufio implementation + writer, err := newRawLogWriter(path) + if err != nil { + return nil, errors.Wrap(err, "create new raw logger") + } + return &filesystemLogWriter{writer}, nil } // If writer is based on bufio we want to flush after a batch of @@ -92,9 +85,13 @@ type rawLogWriter struct { mtx sync.Mutex } -func newRawLogWriter(file *os.File) *rawLogWriter { +func newRawLogWriter(path string) (*rawLogWriter, error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if err != nil { + return nil, err + } buff := bufio.NewWriter(file) - return &rawLogWriter{file: file, buff: buff} + return &rawLogWriter{file: file, buff: buff}, nil } // Write bytes to file @@ -144,7 +141,3 @@ func (l *rawLogWriter) Close() error { return nil } - -func openFile(path string) (*os.File, error) { - return os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) -} diff --git a/server/logging/filesystem_test.go b/server/logging/filesystem_test.go index 912e3e5542..ae6328ec44 100644 --- a/server/logging/filesystem_test.go +++ b/server/logging/filesystem_test.go @@ -4,8 +4,6 @@ import ( "context" "crypto/rand" "encoding/json" - stderrors "errors" - "io/fs" "io/ioutil" "os" "path" @@ -61,27 +59,6 @@ func TestFilesystemLogger(t *testing.T) { } -// TestFilesystemLoggerPermission tests that NewFilesystemLogWriter fails -// if the process does not have permissions to write to the provided path. -func TestFilesystemLoggerPermission(t *testing.T) { - tempPath := t.TempDir() - require.NoError(t, os.Chmod(tempPath, 0000)) - fileName := path.Join(tempPath, "filesystemLogWriter") - for _, tc := range []struct { - name string - rotation bool - }{ - {name: "with-rotation", rotation: true}, - {name: "without-rotation", rotation: false}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := NewFilesystemLogWriter(fileName, log.NewNopLogger(), tc.rotation, false) - require.Error(t, err) - require.True(t, stderrors.Is(err, fs.ErrPermission), err) - }) - } -} - func BenchmarkFilesystemLogger(b *testing.B) { ctx := context.Background() tempPath, err := ioutil.TempDir("", "test") diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 206d4433f4..666db34709 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -59,7 +59,7 @@ type DeleteQueriesFunc func(ctx context.Context, ids []uint) (uint, error) type QueryFunc func(ctx context.Context, id uint) (*fleet.Query, error) -type ListQueriesFunc func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) +type ListQueriesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) type QueryByNameFunc func(ctx context.Context, name string, opts ...fleet.OptionalArg) (*fleet.Query, error) @@ -115,7 +115,7 @@ type LabelFunc func(ctx context.Context, lid uint) (*fleet.Label, error) type ListLabelsFunc func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Label, error) -type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) +type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) type RecordLabelQueryExecutionsFunc func(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error @@ -221,8 +221,6 @@ type ScheduledQueryFunc func(ctx context.Context, id uint) (*fleet.ScheduledQuer type CleanupOrphanScheduledQueryStatsFunc func(ctx context.Context) error -type CleanupOrphanLabelMembershipFunc func(ctx context.Context) error - type NewTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) type SaveTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) @@ -279,14 +277,6 @@ type MigrationStatusFunc func(ctx context.Context) (fleet.MigrationStatus, error type ListSoftwareFunc func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) -type NewTeamPolicyFunc func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) - -type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) - -type DeleteTeamPoliciesFunc func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) - -type TeamPolicyFunc func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) - type DataStore struct { NewCarveFunc NewCarveFunc NewCarveFuncInvoked bool @@ -603,9 +593,6 @@ type DataStore struct { CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFuncInvoked bool - CleanupOrphanLabelMembershipFunc CleanupOrphanLabelMembershipFunc - CleanupOrphanLabelMembershipFuncInvoked bool - NewTeamFunc NewTeamFunc NewTeamFuncInvoked bool @@ -689,18 +676,6 @@ type DataStore struct { ListSoftwareFunc ListSoftwareFunc ListSoftwareFuncInvoked bool - - NewTeamPolicyFunc NewTeamPolicyFunc - NewTeamPolicyFuncInvoked bool - - ListTeamPoliciesFunc ListTeamPoliciesFunc - ListTeamPoliciesFuncInvoked bool - - DeleteTeamPoliciesFunc DeleteTeamPoliciesFunc - DeleteTeamPoliciesFuncInvoked bool - - TeamPolicyFunc TeamPolicyFunc - TeamPolicyFuncInvoked bool } func (s *DataStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { @@ -823,7 +798,7 @@ func (s *DataStore) Query(ctx context.Context, id uint) (*fleet.Query, error) { return s.QueryFunc(ctx, id) } -func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { +func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { s.ListQueriesFuncInvoked = true return s.ListQueriesFunc(ctx, opt) } @@ -963,9 +938,9 @@ func (s *DataStore) ListLabels(ctx context.Context, filter fleet.TeamFilter, opt return s.ListLabelsFunc(ctx, filter, opt) } -func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { +func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { s.LabelQueriesForHostFuncInvoked = true - return s.LabelQueriesForHostFunc(ctx, host) + return s.LabelQueriesForHostFunc(ctx, host, cutoff) } func (s *DataStore) RecordLabelQueryExecutions(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error { @@ -1228,11 +1203,6 @@ func (s *DataStore) CleanupOrphanScheduledQueryStats(ctx context.Context) error return s.CleanupOrphanScheduledQueryStatsFunc(ctx) } -func (s *DataStore) CleanupOrphanLabelMembership(ctx context.Context) error { - s.CleanupOrphanLabelMembershipFuncInvoked = true - return s.CleanupOrphanLabelMembershipFunc(ctx) -} - func (s *DataStore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { s.NewTeamFuncInvoked = true return s.NewTeamFunc(ctx, team) @@ -1372,23 +1342,3 @@ func (s *DataStore) ListSoftware(ctx context.Context, teamId *uint, opt fleet.Li s.ListSoftwareFuncInvoked = true return s.ListSoftwareFunc(ctx, teamId, opt) } - -func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { - s.NewTeamPolicyFuncInvoked = true - return s.NewTeamPolicyFunc(ctx, teamID, queryID) -} - -func (s *DataStore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { - s.ListTeamPoliciesFuncInvoked = true - return s.ListTeamPoliciesFunc(ctx, teamID) -} - -func (s *DataStore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { - s.DeleteTeamPoliciesFuncInvoked = true - return s.DeleteTeamPoliciesFunc(ctx, teamID, ids) -} - -func (s *DataStore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { - s.TeamPolicyFuncInvoked = true - return s.TeamPolicyFunc(ctx, teamID, policyID) -} diff --git a/server/pubsub/testing_utils.go b/server/pubsub/testing_utils.go index 9ff5b39c1d..2b8a02cc41 100644 --- a/server/pubsub/testing_utils.go +++ b/server/pubsub/testing_utils.go @@ -1,7 +1,6 @@ package pubsub import ( - "runtime" "testing" "time" @@ -11,10 +10,6 @@ import ( ) func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, teardown func()) { - if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { - t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) - } - var ( addr = "127.0.0.1:" password = "" @@ -28,14 +23,7 @@ func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, te } addr += port - pool, err := redis.NewRedisPool(redis.PoolConfig{ - Server: addr, - Password: password, - Database: database, - UseTLS: useTLS, - ConnTimeout: 5 * time.Second, - KeepAlive: 10 * time.Second, - }) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) store = NewRedisQueryResults(pool, dupResults) diff --git a/server/service/client.go b/server/service/client.go index 4e7a86b500..abb98c15af 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -230,7 +230,7 @@ func (l *logRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { fmt.Fprintf(os.Stderr, "Read body error: %v", err) return nil, err } - res.Body = io.NopCloser(resBody) + res.Body = ioutil.NopCloser(resBody) return res, nil } diff --git a/server/service/global_policies_test.go b/server/service/global_policies_test.go deleted file mode 100644 index 284cccb4eb..0000000000 --- a/server/service/global_policies_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package service - -import ( - "context" - "testing" - - "github.com/fleetdm/fleet/v4/server/contexts/viewer" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/mock" - "github.com/fleetdm/fleet/v4/server/ptr" -) - -func TestGlobalPoliciesAuth(t *testing.T) { - ds := new(mock.Store) - svc := newTestService(ds, nil, nil) - - ds.NewGlobalPolicyFunc = func(ctx context.Context, queryID uint) (*fleet.Policy, error) { - return nil, nil - } - ds.ListGlobalPoliciesFunc = func(ctx context.Context) ([]*fleet.Policy, error) { - return nil, nil - } - ds.PolicyFunc = func(ctx context.Context, id uint) (*fleet.Policy, error) { - return nil, nil - } - ds.DeleteGlobalPoliciesFunc = func(ctx context.Context, ids []uint) ([]uint, error) { - return nil, nil - } - - var testCases = []struct { - name string - user *fleet.User - shouldFailWrite bool - shouldFailRead bool - }{ - { - "global admin", - &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, - false, - false, - }, - { - "global maintainer", - &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, - false, - false, - }, - { - "global observer", - &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, - true, - false, - }, - { - "team maintainer", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, - true, - true, - }, - { - "team observer, belongs to team", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, - true, - true, - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) - - _, err := svc.NewGlobalPolicy(ctx, 2) - checkAuthErr(t, tt.shouldFailWrite, err) - - _, err = svc.ListGlobalPolicies(ctx) - checkAuthErr(t, tt.shouldFailRead, err) - - _, err = svc.GetPolicyByIDQueries(ctx, 1) - checkAuthErr(t, tt.shouldFailRead, err) - - _, err = svc.DeleteGlobalPolicies(ctx, []uint{1}) - checkAuthErr(t, tt.shouldFailWrite, err) - }) - } -} diff --git a/server/service/handler.go b/server/service/handler.go index edbb178d86..90518ab52b 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -695,35 +695,16 @@ func attachNewStyleFleetAPIRoutes(r *mux.Router, svc fleet.Service, opts []kitht e.POST("/api/v1/fleet/translate", translatorEndpoint, translatorRequest{}) e.POST("/api/v1/fleet/spec/teams", applyTeamSpecsEndpoint, applyTeamSpecsRequest{}) - // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ e.GET("/api/v1/fleet/team/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) e.POST("/api/v1/fleet/team/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) e.PATCH("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) e.DELETE("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) - // End alias - - e.GET("/api/v1/fleet/teams/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) - e.POST("/api/v1/fleet/teams/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) - e.PATCH("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) - e.DELETE("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) e.POST("/api/v1/fleet/global/policies", globalPolicyEndpoint, globalPolicyRequest{}) e.GET("/api/v1/fleet/global/policies", listGlobalPoliciesEndpoint, nil) e.GET("/api/v1/fleet/global/policies/{policy_id}", getPolicyByIDEndpoint, getPolicyByIDRequest{}) e.POST("/api/v1/fleet/global/policies/delete", deleteGlobalPoliciesEndpoint, deleteGlobalPoliciesRequest{}) - // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ - e.POST("/api/v1/fleet/team/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) - e.GET("/api/v1/fleet/team/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) - e.GET("/api/v1/fleet/team/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) - e.POST("/api/v1/fleet/team/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) - // End alias - - e.POST("/api/v1/fleet/teams/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) - e.GET("/api/v1/fleet/teams/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) - e.GET("/api/v1/fleet/teams/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) - e.POST("/api/v1/fleet/teams/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) - e.GET("/api/v1/fleet/software", listSoftwareEndpoint, listSoftwareRequest{}) } diff --git a/server/service/http_auth.go b/server/service/http_auth.go index 7a1c1756ef..bf61a4c616 100644 --- a/server/service/http_auth.go +++ b/server/service/http_auth.go @@ -18,11 +18,9 @@ func setRequestsContexts(svc fleet.Service) kithttp.RequestFunc { return func(ctx context.Context, r *http.Request) context.Context { bearer := token.FromHTTPRequest(r) ctx = token.NewContext(ctx, bearer) - if bearer != "" { - v, err := authViewer(ctx, string(bearer), svc) - if err == nil { - ctx = viewer.NewContext(ctx, *v) - } + v, err := authViewer(ctx, string(bearer), svc) + if err == nil { + ctx = viewer.NewContext(ctx, *v) } // get the user-id for request diff --git a/server/service/http_auth_test.go b/server/service/http_auth_test.go index eea04c6164..3c9533de85 100644 --- a/server/service/http_auth_test.go +++ b/server/service/http_auth_test.go @@ -21,6 +21,7 @@ import ( func TestLogin(t *testing.T) { ds, users, server := setupAuthTest(t) + defer server.Close() var loginTests = []struct { email string status int @@ -59,7 +60,7 @@ func TestLogin(t *testing.T) { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := io.NopCloser(bytes.NewBuffer(j)) + requestBody := &nopCloser{bytes.NewBuffer(j)} resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, tt.status, resp.StatusCode) @@ -172,7 +173,7 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := io.NopCloser(bytes.NewBuffer(j)) + requestBody := &nopCloser{bytes.NewBuffer(j)} resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) @@ -190,6 +191,7 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { func TestNoHeaderErrorsDifferently(t *testing.T) { _, _, server := setupAuthTest(t) + defer server.Close() req, _ := http.NewRequest("GET", server.URL+"/api/v1/fleet/users", nil) client := &http.Client{} @@ -228,3 +230,10 @@ func TestNoHeaderErrorsDifferently(t *testing.T) { } `, string(bodyBytes)) } + +// an io.ReadCloser for new request body +type nopCloser struct { + io.Reader +} + +func (nopCloser) Close() error { return nil } diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go deleted file mode 100644 index 25dafc96ca..0000000000 --- a/server/service/integration_core_test.go +++ /dev/null @@ -1,394 +0,0 @@ -package service - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/ghodss/yaml" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -type integrationTestSuite struct { - suite.Suite - - withServer -} - -func (s *integrationTestSuite) SetupSuite() { - s.withServer.SetupSuite("integrationTestSuite") -} - -func TestIntegrations(t *testing.T) { - testingSuite := new(integrationTestSuite) - testingSuite.s = &testingSuite.Suite - suite.Run(t, testingSuite) -} - -func (s *integrationTestSuite) TestDoubleUserCreationErrors() { - t := s.T() - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String(fleet.RoleObserver), - } - - s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusOK) - respSecond := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusConflict) - - assertBodyContains(t, respSecond, `Error 1062: Duplicate entry 'email@asd.com'`) -} - -func (s *integrationTestSuite) TestUserWithoutRoleErrors() { - t := s.T() - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - } - - resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") -} - -func (s *integrationTestSuite) TestUserWithWrongRoleErrors() { - t := s.T() - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String("wrongrole"), - } - resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") -} - -func (s *integrationTestSuite) TestUserCreationWrongTeamErrors() { - t := s.T() - - teams := []fleet.UserTeam{ - { - Team: fleet.Team{ - ID: 9999, - }, - Role: fleet.RoleObserver, - }, - } - - params := fleet.UserPayload{ - Name: ptr.String("user2"), - Email: ptr.String("email2@asd.com"), - Password: ptr.String("pass"), - Teams: &teams, - } - resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) - assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) -} - -func (s *integrationTestSuite) TestQueryCreationLogsActivity() { - t := s.T() - - admin1 := s.users["admin1@example.com"] - admin1.GravatarURL = "http://iii.com" - err := s.ds.SaveUser(context.Background(), &admin1) - require.NoError(t, err) - - params := fleet.QueryPayload{ - Name: ptr.String("user1"), - Query: ptr.String("select * from time;"), - } - s.Do("POST", "/api/v1/fleet/queries", ¶ms, http.StatusOK) - - activities := listActivitiesResponse{} - s.DoJSON("GET", "/api/v1/fleet/activities", nil, http.StatusOK, &activities) - - assert.Len(t, activities.Activities, 1) - assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0].ActorFullName) - require.NotNil(t, activities.Activities[0].ActorGravatar) - assert.Equal(t, "http://iii.com", *activities.Activities[0].ActorGravatar) - assert.Equal(t, "created_saved_query", activities.Activities[0].Type) -} -func (s *integrationTestSuite) TestAppConfigAdditionalQueriesCanBeRemoved() { - t := s.T() - - spec := []byte(` - host_expiry_settings: - host_expiry_enabled: true - host_expiry_window: 0 - host_settings: - additional_queries: - time: SELECT * FROM time - enable_host_users: true -`) - s.applyConfig(spec) - - spec = []byte(` - host_settings: - enable_host_users: true - additional_queries: null -`) - s.applyConfig(spec) - - config := s.getConfig() - assert.Nil(t, config.HostSettings.AdditionalQueries) - assert.True(t, config.HostExpirySettings.HostExpiryEnabled) -} - -func (s *integrationTestSuite) TestAppConfigDefaultValues() { - config := s.getConfig() - s.Run("Update interval", func() { - require.Equal(s.T(), 1*time.Hour, config.UpdateInterval.OSQueryDetail) - }) - - s.Run("has logging", func() { - require.NotNil(s.T(), config.Logging) - }) -} - -func (s *integrationTestSuite) TestUserRolesSpec() { - t := s.T() - - _, err := s.ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - - email := t.Name() + "@asd.com" - u := &fleet.User{ - Password: []byte("asd"), - Name: t.Name(), - Email: email, - GravatarURL: "http://asd.com", - GlobalRole: ptr.String(fleet.RoleObserver), - } - user, err := s.ds.NewUser(context.Background(), u) - require.NoError(t, err) - assert.Len(t, user.Teams, 0) - - spec := []byte(fmt.Sprintf(` - roles: - %s: - global_role: null - teams: - - role: maintainer - team: team1 -`, - email)) - - var userRoleSpec applyUserRoleSpecsRequest - err = yaml.Unmarshal(spec, &userRoleSpec.Spec) - require.NoError(t, err) - - s.Do("POST", "/api/v1/fleet/users/roles/spec", &userRoleSpec, http.StatusOK) - - user, err = s.ds.UserByEmail(context.Background(), email) - require.NoError(t, err) - require.Len(t, user.Teams, 1) - assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) -} - -func (s *integrationTestSuite) TestGlobalSchedule() { - t := s.T() - - gs := fleet.GlobalSchedulePayload{} - s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 0) - - qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery1", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} - r := globalScheduleQueryResponse{} - s.DoJSON("POST", "/api/v1/fleet/global/schedule", gsParams, http.StatusOK, &r) - - gs = fleet.GlobalSchedulePayload{} - s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) - assert.Equal(t, "TestQuery1", gs.GlobalSchedule[0].Name) - id := gs.GlobalSchedule[0].ID - - gs = fleet.GlobalSchedulePayload{} - gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} - s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), gsParams, http.StatusOK, &gs) - - gs = fleet.GlobalSchedulePayload{} - s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) - - r = globalScheduleQueryResponse{} - s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), nil, http.StatusOK, &r) - - gs = fleet.GlobalSchedulePayload{} - s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 0) -} - -func (s *integrationTestSuite) TestTranslator() { - t := s.T() - - payload := translatorResponse{} - params := translatorRequest{List: []fleet.TranslatePayload{ - { - Type: fleet.TranslatorTypeUserEmail, - Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, - }, - }} - s.DoJSON("POST", "/api/v1/fleet/translate", ¶ms, http.StatusOK, &payload) - require.Len(t, payload.List, 1) - - assert.Equal(t, s.users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) -} - -func (s *integrationTestSuite) TestVulnerableSoftware() { - t := s.T() - - host, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: t.Name() + "1", - UUID: t.Name() + "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - require.NotNil(t, host) - - soft := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "bar", Version: "0.0.3", Source: "apps"}, - }, - } - host.HostSoftware = soft - require.NoError(t, s.ds.SaveHostSoftware(context.Background(), host)) - require.NoError(t, s.ds.LoadHostSoftware(context.Background(), host)) - - soft1 := host.Software[0] - if soft1.Name != "bar" { - soft1 = host.Software[1] - } - - require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) - require.NoError(t, s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) - - resp := s.Do("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), nil, http.StatusOK) - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - expectedJSONSoft2 := `"name": "bar", - "version": "0.0.3", - "source": "apps", - "generated_cpe": "somecpe", - "vulnerabilities": [ - { - "cve": "cve-123-123-132", - "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" - } - ]` - expectedJSONSoft1 := `"name": "foo", - "version": "0.0.1", - "source": "chrome_extensions", - "generated_cpe": "", - "vulnerabilities": null` - // We are doing Contains instead of equals to test the output for software in particular - // ignoring other things like timestamps and things that are outside the cope of this ticket - assert.Contains(t, string(bodyBytes), expectedJSONSoft2) - assert.Contains(t, string(bodyBytes), expectedJSONSoft1) -} - -func (s *integrationTestSuite) TestGlobalPolicies() { - t := s.T() - - for i := 0; i < 3; i++ { - _, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), - OsqueryHostID: strconv.Itoa(i), - NodeKey: fmt.Sprintf("%d", i), - UUID: fmt.Sprintf("%d", i), - Hostname: fmt.Sprintf("foo.local%d", i), - }) - require.NoError(t, err) - } - - qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery3", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gpParams := globalPolicyRequest{QueryID: qr.ID} - gpResp := globalPolicyResponse{} - s.DoJSON("POST", "/api/v1/fleet/global/policies", gpParams, http.StatusOK, &gpResp) - require.NotNil(t, gpResp.Policy) - assert.Equal(t, qr.ID, gpResp.Policy.QueryID) - - policiesResponse := listGlobalPoliciesResponse{} - s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 1) - assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) - - singlePolicyResponse := getPolicyByIDResponse{} - singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) - s.DoJSON("GET", singlePolicyURL, nil, http.StatusOK, &singlePolicyResponse) - assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) - assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) - - listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) - listHostsResp := listHostsResponse{} - s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 3) - - h1 := listHostsResp.Hosts[0] - h2 := listHostsResp.Hosts[1] - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 0) - - require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) - require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 1) - - deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} - deletePolicyResp := deleteGlobalPoliciesResponse{} - s.DoJSON("POST", "/api/v1/fleet/global/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp) - - policiesResponse = listGlobalPoliciesResponse{} - s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 0) -} diff --git a/server/service/integration_ds_only_test.go b/server/service/integration_ds_only_test.go deleted file mode 100644 index 780a281fb2..0000000000 --- a/server/service/integration_ds_only_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package service - -import ( - "net/http" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -type integrationDSTestSuite struct { - withDS - suite.Suite -} - -func TestIntegrationDSTestSuite(t *testing.T) { - testingSuite := new(integrationDSTestSuite) - testingSuite.s = &testingSuite.Suite - suite.Run(t, testingSuite) -} - -func (s *integrationDSTestSuite) SetupSuite() { - s.withDS.SetupSuite("integrationDSTestSuite") -} - -func (s *integrationDSTestSuite) TestLicenseExpiration() { - testCases := []struct { - name string - tier string - expiration time.Time - shouldHaveHeader bool - }{ - {"basic expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, - {"basic not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, - {"core expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, - {"core not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, - } - - createTestUsers(s.T(), s.ds) - for _, tt := range testCases { - s.Run(tt.name, func() { - t := s.T() - - license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} - _, server := RunServerForTestsWithDS(t, s.ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) - - ts := withServer{server: server} - ts.s = &s.Suite - ts.token = ts.getTestAdminToken() - - resp := ts.Do("GET", "/api/v1/fleet/config", nil, http.StatusOK) - if tt.shouldHaveHeader { - require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) - } else { - require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) - } - }) - } -} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go deleted file mode 100644 index 5370f50b82..0000000000 --- a/server/service/integration_enterprise_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "testing" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -func TestIntegrationsEnterprise(t *testing.T) { - testingSuite := new(integrationEnterpriseTestSuite) - testingSuite.s = &testingSuite.Suite - suite.Run(t, testingSuite) -} - -type integrationEnterpriseTestSuite struct { - withServer - suite.Suite -} - -func (s *integrationEnterpriseTestSuite) SetupSuite() { - s.withDS.SetupSuite("integrationEnterpriseTestSuite") - - users, server := RunServerForTestsWithDS( - s.T(), s.ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) - s.server = server - s.users = users - s.token = s.getTestAdminToken() -} - -func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { - t := s.T() - - // create a team through the service so it initializes the agent ops - teamName := t.Name() + "team1" - team := &fleet.Team{ - Name: teamName, - Description: "desc team1", - } - - s.Do("POST", "/api/v1/fleet/teams", team, http.StatusOK) - - // updates a team - agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) - teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: teamName, AgentOptions: &agentOpts}}} - s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) - - team, err := s.ds.TeamByName(context.Background(), teamName) - require.NoError(t, err) - - assert.Len(t, team.Secrets, 0) - require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) - - // creates a team with default agent options - user, err := s.ds.UserByEmail(context.Background(), "admin1@example.com") - require.NoError(t, err) - - teams, err := s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - require.True(t, len(teams) >= 1) - - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} - s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) - - teams, err = s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - assert.True(t, len(teams) >= 2) - - team, err = s.ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` - assert.Len(t, team.Secrets, 0) - require.NotNil(t, team.AgentOptions) - require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) - - // updates secrets - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} - s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) - - team, err = s.ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - require.Len(t, team.Secrets, 1) - assert.Equal(t, "ABC", team.Secrets[0].Secret) -} - -func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { - t := s.T() - - team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - - ts := getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 0) - - qr, err := s.ds.NewQuery( - context.Background(), - &fleet.Query{Name: "TestQueryTeamPolicy", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}, - ) - require.NoError(t, err) - - gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} - r := teamScheduleQueryResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), gsParams, http.StatusOK, &r) - - ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(42), ts.Scheduled[0].Interval) - assert.Equal(t, "TestQueryTeamPolicy", ts.Scheduled[0].Name) - assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) - id := ts.Scheduled[0].ID - - modifyResp := modifyTeamScheduleResponse{} - modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} - s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), modifyParams, http.StatusOK, &modifyResp) - - // just to satisfy my paranoia, wanted to make sure the contents of the json would work - s.DoRaw("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), []byte(`{"interval": 77}`), http.StatusOK) - - ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(77), ts.Scheduled[0].Interval) - - deleteResp := deleteTeamScheduleResponse{} - s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), nil, http.StatusOK, &deleteResp) - - ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 0) -} - -func (s *integrationEnterpriseTestSuite) TestTeamPolicies() { - t := s.T() - - team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1" + t.Name(), - Description: "desc team1", - }) - require.NoError(t, err) - - oldToken := s.token - t.Cleanup(func() { - s.token = oldToken - }) - - password := "garbage" - email := "testteam@user.com" - - u := &fleet.User{ - Name: "test team user", - Email: email, - GlobalRole: nil, - Teams: []fleet.UserTeam{ - { - Team: *team1, - Role: fleet.RoleMaintainer, - }, - }, - } - require.NoError(t, u.SetPassword(password, 10, 10)) - _, err = s.ds.NewUser(context.Background(), u) - require.NoError(t, err) - - s.token = s.getTestToken(email, password) - - ts := listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Policies, 0) - - qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) - require.NoError(t, err) - - tpParams := teamPolicyRequest{QueryID: qr.ID} - r := teamPolicyResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), tpParams, http.StatusOK, &r) - - ts = listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Policies, 1) - assert.Equal(t, "TestQuery2", ts.Policies[0].QueryName) - assert.Equal(t, qr.ID, ts.Policies[0].QueryID) - - deletePolicyParams := deleteTeamPoliciesRequest{IDs: []uint{ts.Policies[0].ID}} - deletePolicyResp := deleteTeamPoliciesResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies/delete", team1.ID), deletePolicyParams, http.StatusOK, &deletePolicyResp) - - ts = listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) - require.Len(t, ts.Policies, 0) -} diff --git a/server/service/integration_logger_test.go b/server/service/integration_logger_test.go deleted file mode 100644 index 2f39f97496..0000000000 --- a/server/service/integration_logger_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package service - -import ( - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "strings" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -func TestIntegrationLoggerTestSuite(t *testing.T) { - testingSuite := new(integrationLoggerTestSuite) - testingSuite.s = &testingSuite.Suite - suite.Run(t, testingSuite) -} - -type integrationLoggerTestSuite struct { - withServer - suite.Suite - - buf *bytes.Buffer -} - -func (s *integrationLoggerTestSuite) SetupSuite() { - s.withDS.SetupSuite("integrationLoggerTestSuite") - - s.buf = new(bytes.Buffer) - logger := log.NewJSONLogger(s.buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - users, server := RunServerForTestsWithDS(s.T(), s.ds, TestServerOpts{Logger: logger}) - s.server = server - s.users = users -} - -func (s *integrationLoggerTestSuite) TearDownTest() { - s.buf.Reset() -} - -func (s *integrationLoggerTestSuite) TestLogger() { - t := s.T() - - s.token = getTestAdminToken(t, s.server) - - s.getConfig() - - params := fleet.QueryPayload{ - Name: ptr.String("somequery"), - Description: ptr.String("desc"), - Query: ptr.String("select 1 from osquery;"), - } - payload := createQueryRequest{} - s.DoJSON("POST", "/api/v1/fleet/queries", params, http.StatusOK, &payload) - - logs := s.buf.String() - parts := strings.Split(strings.TrimSpace(logs), "\n") - assert.Len(t, parts, 3) - for i, part := range parts { - kv := make(map[string]string) - err := json.Unmarshal([]byte(part), &kv) - require.NoError(t, err) - - assert.NotEqual(t, "", kv["took"]) - - switch i { - case 0: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) - case 1: - assert.Equal(t, "debug", kv["level"]) - assert.Equal(t, "GET", kv["method"]) - assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - case 2: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - assert.Equal(t, "somequery", kv["name"]) - assert.Equal(t, "select 1 from osquery;", kv["sql"]) - default: - t.Fail() - } - } -} - -func (s *integrationLoggerTestSuite) TestOsqueryEndpointsLogErrors() { - t := s.T() - - _, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: t.Name() + "1234", - UUID: "1", - Hostname: "foo.local", - OsqueryHostID: t.Name(), - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - requestBody := io.NopCloser(bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))) - req, _ := http.NewRequest("POST", s.server.URL+"/api/v1/osquery/log", requestBody) - client := &http.Client{} - _, err = client.Do(req) - require.Nil(t, err) - - logString := s.buf.String() - assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} -`, logString) -} - -func (s *integrationLoggerTestSuite) TestSubmitStatusLog() { - t := s.T() - - _, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: t.Name() + "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - OsqueryHostID: t.Name(), - }) - require.NoError(t, err) - - req := submitLogsRequest{ - NodeKey: "1234", - LogType: "status", - Data: nil, - } - res := submitLogsResponse{} - s.DoJSON("POST", "/api/v1/osquery/log", req, http.StatusOK, &res) - - logString := s.buf.String() - assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) - assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) -} - -func (s *integrationLoggerTestSuite) TestEnrollAgentLogsErrors() { - t := s.T() - _, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - j, err := json.Marshal(&enrollAgentRequest{ - EnrollSecret: "1234", - HostIdentifier: "4321", - HostDetails: nil, - }) - require.NoError(t, err) - - s.DoRawNoAuth("POST", "/api/v1/osquery/enroll", j, http.StatusUnauthorized) - - parts := strings.Split(strings.TrimSpace(s.buf.String()), "\n") - require.Len(t, parts, 1) - logData := make(map[string]json.RawMessage) - require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) - assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) -} diff --git a/server/service/integration_test.go b/server/service/integration_test.go new file mode 100644 index 0000000000..ca24a626c2 --- /dev/null +++ b/server/service/integration_test.go @@ -0,0 +1,931 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/datastore/mysql" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/ghodss/yaml" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDoubleUserCreationErrors(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String(fleet.RoleObserver), + } + j, err := json.Marshal(¶ms) + assert.Nil(t, err) + + requestBody := &nopCloser{bytes.NewBuffer(j)} + req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + requestBody = &nopCloser{bytes.NewBuffer(j)} + req, _ = http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err = client.Do(req) + require.Nil(t, err) + assert.Equal(t, http.StatusConflict, resp.StatusCode) + assertBodyContains(t, resp, `Error 1062: Duplicate entry 'email@asd.com'`) +} + +func TestUserWithoutRoleErrors(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + } + j, err := json.Marshal(¶ms) + assert.Nil(t, err) + + requestBody := &nopCloser{bytes.NewBuffer(j)} + req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") +} + +func TestUserWithWrongRoleErrors(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String("wrongrole"), + } + j, err := json.Marshal(¶ms) + assert.Nil(t, err) + + requestBody := &nopCloser{bytes.NewBuffer(j)} + req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") +} + +func TestUserCreationWrongTeamErrors(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + teams := []fleet.UserTeam{ + { + Team: fleet.Team{ + ID: 9999, + }, + Role: fleet.RoleObserver, + }, + } + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + Teams: &teams, + } + method := "POST" + path := "/api/v1/fleet/users/admin" + expectedStatusCode := http.StatusUnprocessableEntity + + resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) + defer closeFunc() + assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) +} + +func doReq( + t *testing.T, + params interface{}, + method string, + server *httptest.Server, + path string, + token string, + expectedStatusCode int, +) (*http.Response, func()) { + j, err := json.Marshal(¶ms) + assert.Nil(t, err) + + requestBody := &nopCloser{bytes.NewBuffer(j)} + req, _ := http.NewRequest(method, server.URL+path, requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + assert.Equal(t, expectedStatusCode, resp.StatusCode) + return resp, func() { + thisResp := resp + thisResp.Body.Close() + } +} + +func doRawReq( + t *testing.T, + body []byte, + method string, + server *httptest.Server, + path string, + token string, + expectedStatusCode int, +) *http.Response { + requestBody := &nopCloser{bytes.NewBuffer(body)} + req, _ := http.NewRequest(method, server.URL+path, requestBody) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + assert.Equal(t, expectedStatusCode, resp.StatusCode) + return resp +} + +func doJSONReq( + t *testing.T, + params interface{}, + method string, + server *httptest.Server, + path string, + token string, + expectedStatusCode int, + v interface{}, +) { + resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) + defer closeFunc() + err := json.NewDecoder(resp.Body).Decode(v) + require.Nil(t, err) +} + +func assertBodyContains(t *testing.T, resp *http.Response, expectedError string) { + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + bodyString := string(bodyBytes) + assert.Contains(t, bodyString, expectedError) +} + +func TestQueryCreationLogsActivity(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + users, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + admin1 := users["admin1@example.com"] + admin1.GravatarURL = "http://iii.com" + err := ds.SaveUser(context.Background(), &admin1) + require.NoError(t, err) + + params := fleet.QueryPayload{ + Name: ptr.String("user1"), + Query: ptr.String("select * from time;"), + } + _, closeFunc := doReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK) + defer closeFunc() + type activitiesRespose struct { + Activities []map[string]interface{} `json:"activities"` + } + activities := activitiesRespose{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/activities", token, http.StatusOK, &activities) + + assert.Len(t, activities.Activities, 1) + assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0]["actor_full_name"]) + assert.Equal(t, "http://iii.com", activities.Activities[0]["actor_gravatar"]) + assert.Equal(t, "created_saved_query", activities.Activities[0]["type"]) +} + +func getJSON(r *http.Response, target interface{}) error { + return json.NewDecoder(r.Body).Decode(target) +} + +func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { + err := &fleet.Error{} + require.Nil(t, getJSON(resp, err)) + assert.Equal(t, code, err.Code) + assert.Equal(t, message, err.Message) +} + +func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + spec := []byte(` + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 0 + host_settings: + additional_queries: + time: SELECT * FROM time + enable_host_users: true +`) + applyConfig(t, spec, server, token) + + spec = []byte(` + host_settings: + enable_host_users: true + additional_queries: null +`) + applyConfig(t, spec, server, token) + + config := getConfig(t, server, token) + assert.Nil(t, config.HostSettings.AdditionalQueries) + assert.True(t, config.HostExpirySettings.HostExpiryEnabled) +} + +func TestAppConfigUpdateInterval(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + config := getConfig(t, server, token) + require.Equal(t, 1*time.Hour, config.UpdateInterval.OSQueryDetail) +} + +func TestAppConfigHasLogging(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + config := getConfig(t, server, token) + require.NotNil(t, config.Logging) +} + +func applyConfig(t *testing.T, spec []byte, server *httptest.Server, token string) { + var appConfigSpec interface{} + err := yaml.Unmarshal(spec, &appConfigSpec) + require.NoError(t, err) + + _, closeFunc := doReq(t, appConfigSpec, "PATCH", server, "/api/v1/fleet/config", token, http.StatusOK) + closeFunc() +} + +func getConfig(t *testing.T, server *httptest.Server, token string) *appConfigResponse { + var responseBody *appConfigResponse + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK, &responseBody) + return responseBody +} + +func TestUserRolesSpec(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + _, err := ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + token := getTestAdminToken(t, server) + + user, err := ds.UserByEmail(context.Background(), "user1@example.com") + require.NoError(t, err) + assert.Len(t, user.Teams, 0) + + spec := []byte(` + roles: + user1@example.com: + global_role: null + teams: + - role: maintainer + team: team1 +`) + + var userRoleSpec applyUserRoleSpecsRequest + err = yaml.Unmarshal(spec, &userRoleSpec.Spec) + require.NoError(t, err) + + _, closeFunc := doReq(t, userRoleSpec, "POST", server, "/api/v1/fleet/users/roles/spec", token, http.StatusOK) + closeFunc() + + user, err = ds.UserByEmail(context.Background(), "user1@example.com") + require.NoError(t, err) + require.Len(t, user.Teams, 1) + assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) + + // But users are not deleted + users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) + require.NoError(t, err) + assert.Len(t, users, 3) +} + +func TestGlobalSchedule(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + test.AddAllHostsLabel(t, ds) + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + gs := fleet.GlobalSchedulePayload{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) + assert.Len(t, gs.GlobalSchedule, 0) + + qr, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} + r := globalScheduleQueryResponse{} + doJSONReq(t, gsParams, "POST", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &r) + require.Nil(t, r.Err) + + gs = fleet.GlobalSchedulePayload{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) + assert.Equal(t, "TestQuery", gs.GlobalSchedule[0].Name) + id := gs.GlobalSchedule[0].ID + + gs = fleet.GlobalSchedulePayload{} + gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} + doJSONReq( + t, gsParams, "PATCH", server, + fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), + token, http.StatusOK, &gs, + ) + + gs = fleet.GlobalSchedulePayload{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) + + r = globalScheduleQueryResponse{} + doJSONReq( + t, nil, "DELETE", server, + fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), + token, http.StatusOK, &r, + ) + require.Nil(t, r.Err) + + gs = fleet.GlobalSchedulePayload{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 0) +} + +func TestTeamSpecs(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) + defer server.Close() + token := getTestAdminToken(t, server) + + // create a team through the service so it initializes the agent ops + team := &fleet.Team{ + Name: "team1", + Description: "desc team1", + } + _, closeFunc := doReq(t, team, "POST", server, "/api/v1/fleet/teams", token, http.StatusOK) + defer closeFunc() + + // updates a team + agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) + teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team1", AgentOptions: &agentOpts}}} + _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) + defer closeFunc() + + team, err := ds.TeamByName(context.Background(), "team1") + require.NoError(t, err) + + assert.Len(t, team.Secrets, 0) + require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) + + // creates a team with default agent options + user, err := ds.UserByEmail(context.Background(), "admin1@example.com") + require.NoError(t, err) + + teams, err := ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + assert.Len(t, teams, 1) + + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} + _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) + defer closeFunc() + + teams, err = ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + assert.Len(t, teams, 2) + + team, err = ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` + assert.Len(t, team.Secrets, 0) + require.NotNil(t, team.AgentOptions) + require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) + + // updates secrets + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} + _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) + defer closeFunc() + + team, err = ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + require.Len(t, team.Secrets, 1) + assert.Equal(t, "ABC", team.Secrets[0].Secret) +} + +func TestTranslator(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + users, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + payload := translatorResponse{} + params := translatorRequest{List: []fleet.TranslatePayload{ + { + Type: fleet.TranslatorTypeUserEmail, + Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, + }, + }} + doJSONReq(t, ¶ms, "POST", server, "/api/v1/fleet/translate", token, http.StatusOK, &payload) + + require.Nil(t, payload.Err) + assert.Len(t, payload.List, 1) + + assert.Equal(t, users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) +} + +func TestTeamSchedule(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + test.AddAllHostsLabel(t, ds) + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + team1, err := ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + ts := getTeamScheduleResponse{} + doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) + assert.Len(t, ts.Scheduled, 0) + + qr, err := ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) + require.NoError(t, err) + + gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} + r := teamScheduleQueryResponse{} + doJSONReq(t, gsParams, "POST", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &r) + require.Nil(t, r.Err) + + ts = getTeamScheduleResponse{} + doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(42), ts.Scheduled[0].Interval) + assert.Equal(t, "TestQuery", ts.Scheduled[0].Name) + assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) + id := ts.Scheduled[0].ID + + modifyResp := modifyTeamScheduleResponse{} + modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} + doJSONReq( + t, modifyParams, "PATCH", server, + fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), + token, http.StatusOK, &modifyResp, + ) + + // just to satisfy my paranoia, wanted to make sure the contents of the json would work + doRawReq(t, []byte(`{"interval": 77}`), "PATCH", server, + fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), + token, http.StatusOK) + + ts = getTeamScheduleResponse{} + doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) + assert.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(77), ts.Scheduled[0].Interval) + + deleteResp := deleteTeamScheduleResponse{} + doJSONReq( + t, nil, "DELETE", server, + fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), + token, http.StatusOK, &deleteResp, + ) + require.Nil(t, r.Err) + + ts = getTeamScheduleResponse{} + doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) + assert.Len(t, ts.Scheduled, 0) +} + +func TestLogger(t *testing.T) { + buf := new(bytes.Buffer) + logger := log.NewJSONLogger(buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) + defer server.Close() + token := getTestAdminToken(t, server) + + getConfig(t, server, token) + params := fleet.QueryPayload{ + Name: ptr.String("somequery"), + Description: ptr.String("desc"), + Query: ptr.String("select 1 from osquery;"), + } + payload := createQueryRequest{} + doJSONReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK, &payload) + + logs := buf.String() + parts := strings.Split(strings.TrimSpace(logs), "\n") + assert.Len(t, parts, 3) + for i, part := range parts { + kv := make(map[string]string) + err := json.Unmarshal([]byte(part), &kv) + require.NoError(t, err) + + assert.NotEqual(t, "", kv["took"]) + + switch i { + case 0: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) + case 1: + assert.Equal(t, "debug", kv["level"]) + assert.Equal(t, "GET", kv["method"]) + assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + case 2: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + assert.Equal(t, "somequery", kv["name"]) + assert.Equal(t, "select 1 from osquery;", kv["sql"]) + default: + t.Fail() + } + } +} + +func TestVulnerableSoftware(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + require.NotNil(t, host) + + soft := fleet.HostSoftware{ + Modified: true, + Software: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "apps"}, + }, + } + host.HostSoftware = soft + require.NoError(t, ds.SaveHostSoftware(context.Background(), host)) + require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) + + soft1 := host.Software[0] + if soft1.Name != "bar" { + soft1 = host.Software[1] + } + + require.NoError(t, ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) + require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) + + path := fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID) + resp, closeFunc := doReq(t, nil, "GET", server, path, token, http.StatusOK) + defer closeFunc() + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + expectedJSONSoft2 := `"name": "bar", + "version": "0.0.3", + "source": "apps", + "generated_cpe": "somecpe", + "vulnerabilities": [ + { + "cve": "cve-123-123-132", + "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" + } + ]` + expectedJSONSoft1 := `"name": "foo", + "version": "0.0.1", + "source": "chrome_extensions", + "generated_cpe": "", + "vulnerabilities": null` + // We are doing Contains instead of equals to test the output for software in particular + // ignoring other things like timestamps and things that are outside the cope of this ticket + assert.Contains(t, string(bodyBytes), expectedJSONSoft2) + assert.Contains(t, string(bodyBytes), expectedJSONSoft1) +} + +func TestGlobalPolicies(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds) + defer server.Close() + token := getTestAdminToken(t, server) + + for i := 0; i < 3; i++ { + _, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), + OsqueryHostID: strconv.Itoa(i), + NodeKey: fmt.Sprintf("%d", i), + UUID: fmt.Sprintf("%d", i), + Hostname: fmt.Sprintf("foo.local%d", i), + }) + require.NoError(t, err) + } + + qr, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gpParams := globalPolicyRequest{QueryID: qr.ID} + gpResp := globalPolicyResponse{} + doJSONReq(t, gpParams, "POST", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &gpResp) + require.NotNil(t, gpResp.Policy) + assert.Equal(t, qr.ID, gpResp.Policy.QueryID) + + policiesResponse := listGlobalPoliciesResponse{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 1) + assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) + + singlePolicyResponse := getPolicyByIDResponse{} + singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) + doJSONReq(t, nil, "GET", server, singlePolicyURL, token, http.StatusOK, &singlePolicyResponse) + assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) + assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) + + listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) + listHostsResp := listHostsResponse{} + doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 3) + + h1 := listHostsResp.Hosts[0] + h2 := listHostsResp.Hosts[1] + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 0) + + require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) + require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 1) + + deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} + deletePolicyResp := deleteGlobalPoliciesResponse{} + doJSONReq(t, deletePolicyParams, "POST", server, "/api/v1/fleet/global/policies/delete", token, http.StatusOK, &deletePolicyResp) + + policiesResponse = listGlobalPoliciesResponse{} + doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 0) +} + +func TestOsqueryEndpointsLogErrors(t *testing.T) { + buf := new(bytes.Buffer) + logger := log.NewJSONLogger(buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) + defer server.Close() + + _, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + requestBody := &nopCloser{bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))} + req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/log", requestBody) + client := &http.Client{} + _, err = client.Do(req) + require.Nil(t, err) + + logString := buf.String() + assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} +`, logString) +} + +func TestSubmitStatusLog(t *testing.T) { + buf := new(bytes.Buffer) + logger := log.NewJSONLogger(buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) + defer server.Close() + token := getTestAdminToken(t, server) + + _, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + req := submitLogsRequest{ + NodeKey: "1234", + LogType: "status", + Data: nil, + } + res := submitLogsResponse{} + doJSONReq(t, req, "POST", server, "/api/v1/osquery/log", token, http.StatusOK, &res) + + logString := buf.String() + assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) + assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) +} + +func TestEnrollAgentLogsErrors(t *testing.T) { + buf := new(bytes.Buffer) + logger := log.NewJSONLogger(buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) + defer server.Close() + + _, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + j, err := json.Marshal(&enrollAgentRequest{ + EnrollSecret: "1234", + HostIdentifier: "4321", + HostDetails: nil, + }) + require.NoError(t, err) + + requestBody := &nopCloser{bytes.NewBuffer(j)} + req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/enroll", requestBody) + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + parts := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, parts, 1) + logData := make(map[string]json.RawMessage) + require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) + assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) +} + +func TestLicenseExpiration(t *testing.T) { + ds := mysql.CreateMySQLDS(t) + defer ds.Close() + + testCases := []struct { + name string + tier string + expiration time.Time + shouldHaveHeader bool + }{ + {"premium expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, + {"premium not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, + {"free expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, + {"free not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, + } + + _ = createTestUsers(t, ds) + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} + _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) + defer server.Close() + + token := getTestAdminToken(t, server) + + resp, closeFunc := doReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK) + defer closeFunc() + if tt.shouldHaveHeader { + require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) + } else { + require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) + } + }) + } +} diff --git a/server/service/service.go b/server/service/service.go index 62cec4c931..e9cf216b89 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -38,19 +38,10 @@ type Service struct { } // NewService creates a new service from the config struct -func NewService( - ds fleet.Datastore, - resultStore fleet.QueryResultStore, - logger kitlog.Logger, - osqueryLogger *logging.OsqueryLogger, - config config.FleetConfig, - mailService fleet.MailService, - c clock.Clock, - sso sso.SessionStore, - lq fleet.LiveQueryStore, - carveStore fleet.CarveStore, - license fleet.LicenseInfo, -) (fleet.Service, error) { +func NewService(ds fleet.Datastore, resultStore fleet.QueryResultStore, + logger kitlog.Logger, osqueryLogger *logging.OsqueryLogger, config config.FleetConfig, mailService fleet.MailService, + c clock.Clock, sso sso.SessionStore, lq fleet.LiveQueryStore, carveStore fleet.CarveStore, + license fleet.LicenseInfo) (fleet.Service, error) { var svc fleet.Service authorizer, err := authz.NewAuthorizer() diff --git a/server/service/service_appconfig.go b/server/service/service_appconfig.go index 1c9ce979e7..e15ecf851d 100644 --- a/server/service/service_appconfig.go +++ b/server/service/service_appconfig.go @@ -181,10 +181,7 @@ func (svc *Service) SetupRequired(ctx context.Context) (bool, error) { } func (svc *Service) UpdateIntervalConfig(ctx context.Context) (*fleet.UpdateIntervalConfig, error) { - return &fleet.UpdateIntervalConfig{ - OSQueryDetail: svc.config.Osquery.DetailUpdateInterval, - OSQueryPolicy: svc.config.Osquery.PolicyUpdateInterval, - }, nil + return &fleet.UpdateIntervalConfig{OSQueryDetail: svc.config.Osquery.DetailUpdateInterval}, nil } func (svc *Service) LoggingConfig(ctx context.Context) (*fleet.Logging, error) { diff --git a/server/service/service_campaign_test.go b/server/service/service_campaign_test.go index 5437cca55f..6a6039cac5 100644 --- a/server/service/service_campaign_test.go +++ b/server/service/service_campaign_test.go @@ -36,7 +36,7 @@ func TestStreamCampaignResultsClosesReditOnWSClose(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { diff --git a/server/service/service_campaigns.go b/server/service/service_campaigns.go index ec1cd90e17..0802a2f99c 100644 --- a/server/service/service_campaigns.go +++ b/server/service/service_campaigns.go @@ -61,7 +61,7 @@ func (svc Service) NewDistributedQueryCampaign(ctx context.Context, queryString } queryString = query.Query } else { - if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRunNew); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionWrite); err != nil { return nil, err } query = &fleet.Query{ diff --git a/server/service/service_hosts_test.go b/server/service/service_hosts_test.go index d288d2ff6e..7f276443b7 100644 --- a/server/service/service_hosts_test.go +++ b/server/service/service_hosts_test.go @@ -32,7 +32,6 @@ func TestListHosts(t *testing.T) { SeenTime: storedTime, DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), }) require.NoError(t, err) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index 6787aa7c7b..fc5ab0bfdd 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -2,10 +2,8 @@ package service import ( "context" - "crypto/rand" "encoding/json" "fmt" - "math/big" "strconv" "strings" "time" @@ -406,7 +404,7 @@ const hostDistributedQueryPrefix = "fleet_distributed_query_" // osqueryd to fill in the host details func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map[string]string, error) { queries := make(map[string]string) - if !svc.shouldUpdate(host.DetailUpdatedAt, svc.config.Osquery.DetailUpdateInterval) && !host.RefetchRequested { + if host.DetailUpdatedAt.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) && !host.RefetchRequested { // No need to update already fresh details return queries, nil } @@ -440,19 +438,6 @@ func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map return queries, nil } -func (svc *Service) shouldUpdate(lastUpdated time.Time, interval time.Duration) bool { - var jitter time.Duration - if svc.config.Osquery.MaxJitterPercent > 0 { - maxJitter := time.Duration(svc.config.Osquery.MaxJitterPercent) * interval / time.Duration(100.0) - randDuration, err := rand.Int(rand.Reader, big.NewInt(int64(maxJitter))) - if err == nil { - jitter = time.Duration(randDuration.Int64()) - } - } - cutoff := svc.clock.Now().Add(-(interval + jitter)) - return lastUpdated.Before(cutoff) -} - func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) { // skipauth: Authorization is currently for user endpoints only. svc.authz.SkipAuthorization(ctx) @@ -470,15 +455,14 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin } // Retrieve the label queries that should be updated - if svc.shouldUpdate(host.LabelUpdatedAt, svc.config.Osquery.LabelUpdateInterval) { - labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host) - if err != nil { - return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} - } + cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval) + labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host, cutoff) + if err != nil { + return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} + } - for name, query := range labelQueries { - queries[hostLabelQueryPrefix+name] = query - } + for name, query := range labelQueries { + queries[hostLabelQueryPrefix+name] = query } liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID) @@ -490,15 +474,13 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin queries[hostDistributedQueryPrefix+name] = query } - if svc.shouldUpdate(host.PolicyUpdatedAt, svc.config.Osquery.PolicyUpdateInterval) { - policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) - if err != nil { - return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} - } + policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) + if err != nil { + return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} + } - for name, query := range policyQueries { - queries[hostPolicyQueryPrefix+name] = query - } + for name, query := range policyQueries { + queries[hostPolicyQueryPrefix+name] = query } accelerate := uint(0) @@ -709,8 +691,6 @@ func (svc *Service) SubmitDistributedQueryResults( } if len(policyResults) > 0 { - host.Modified = true - host.PolicyUpdatedAt = svc.clock.Now() err = svc.ds.RecordPolicyQueryExecutions(ctx, &host, policyResults, svc.clock.Now()) if err != nil { logging.WithErr(ctx, err) diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 2f0d3f0fe9..30d7b9fa56 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -324,7 +324,7 @@ func TestLabelQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { @@ -361,7 +361,7 @@ func TestLabelQueries(t *testing.T) { assert.Len(t, queries, 0) assert.Zero(t, acc) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{ "label1": "query1", "label2": "query2", @@ -539,7 +539,7 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -715,7 +715,7 @@ func TestDetailQueries(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -920,7 +920,7 @@ func TestNewDistributedQueryCampaign(t *testing.T) { mockClock := clock.NewMockClock() svc := newTestServiceWithClock(ds, rs, lq, mockClock) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { @@ -989,7 +989,7 @@ func TestDistributedQueryResults(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -1721,7 +1721,7 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { }, nil } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { return nil } @@ -1746,63 +1746,6 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { require.NoError(t, err) } -func TestTeamMaintainerCanRunNewDistributedCampaigns(t *testing.T) { - ds := new(mock.Store) - rs := &mock.QueryResultStore{ - HealthCheckFunc: func() error { - return nil - }, - } - lq := &live_query.MockLiveQuery{} - mockClock := clock.NewMockClock() - svc := newTestServiceWithClock(ds, rs, lq, mockClock) - - ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { - return &fleet.AppConfig{}, nil - } - - ds.NewDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) (*fleet.DistributedQueryCampaign, error) { - return camp, nil - } - ds.QueryFunc = func(ctx context.Context, id uint) (*fleet.Query, error) { - return &fleet.Query{ - ID: 42, - Name: "query", - Query: "select 1;", - ObserverCanRun: false, - }, nil - } - viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{ - User: &fleet.User{ID: 0, Teams: []fleet.UserTeam{{Role: fleet.RoleMaintainer}}}, - }) - - q := "select year, month, day, hour, minutes, seconds from time" - ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { - return nil - } - //var gotQuery *fleet.Query - ds.NewQueryFunc = func(ctx context.Context, query *fleet.Query, opts ...fleet.OptionalArg) (*fleet.Query, error) { - //gotQuery = query - query.ID = 42 - return query, nil - } - ds.NewDistributedQueryCampaignTargetFunc = func(ctx context.Context, target *fleet.DistributedQueryCampaignTarget) (*fleet.DistributedQueryCampaignTarget, error) { - return target, nil - } - ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) { - return fleet.TargetMetrics{}, nil - } - ds.HostIDsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets) ([]uint, error) { - return []uint{1, 3, 5}, nil - } - ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { - return nil - } - lq.On("RunQuery", "0", "select year, month, day, hour, minutes, seconds from time", []uint{1, 3, 5}).Return(nil) - _, err := svc.NewDistributedQueryCampaign(viewerCtx, q, nil, fleet.HostTargets{HostIDs: []uint{2}, LabelIDs: []uint{1}}) - require.NoError(t, err) -} - func TestPolicyQueries(t *testing.T) { mockClock := clock.NewMockClock() ds := new(mock.Store) @@ -1813,14 +1756,13 @@ func TestPolicyQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { return host, nil } - ds.SaveHostFunc = func(ctx context.Context, gotHost *fleet.Host) error { - host = gotHost + ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { return nil } ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { @@ -1873,40 +1815,4 @@ func TestPolicyQueries(t *testing.T) { require.NotNil(t, recordedResults[1]) require.Equal(t, true, *recordedResults[1]) require.Nil(t, recordedResults[2]) - - ctx = hostctx.NewContext(context.Background(), *host) - queries, _, err = svc.GetDistributedQueries(ctx) - require.NoError(t, err) - require.Len(t, queries, expectedDetailQueries) - - // After the first time we get policies and update the host, then there shouldn't be any policies - hasAnyPolicy := false - for name := range queries { - if strings.HasPrefix(name, hostPolicyQueryPrefix) { - hasAnyPolicy = true - break - } - } - assert.False(t, hasAnyPolicy) - - // Let's move time forward, there should be policies now - mockClock.AddTime(2 * time.Hour) - - queries, _, err = svc.GetDistributedQueries(ctx) - require.NoError(t, err) - require.Len(t, queries, expectedDetailQueries+2) - - hasPolicy1, hasPolicy2 = false, false - for name := range queries { - if strings.HasPrefix(name, hostPolicyQueryPrefix) { - if name[len(hostPolicyQueryPrefix):] == "1" { - hasPolicy1 = true - } - if name[len(hostPolicyQueryPrefix):] == "2" { - hasPolicy2 = true - } - } - } - assert.True(t, hasPolicy1) - assert.True(t, hasPolicy2) } diff --git a/server/service/service_queries.go b/server/service/service_queries.go index 3eefd83733..65ea5722df 100644 --- a/server/service/service_queries.go +++ b/server/service/service_queries.go @@ -66,7 +66,7 @@ func (svc Service) GetQuerySpecs(ctx context.Context) ([]*fleet.QuerySpec, error return nil, err } - queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{}) + queries, err := svc.ds.ListQueries(ctx, fleet.ListOptions{}) if err != nil { return nil, errors.Wrap(err, "getting queries") } @@ -90,39 +90,12 @@ func (svc Service) GetQuerySpec(ctx context.Context, name string) (*fleet.QueryS return specFromQuery(query), nil } -func onlyShowObserverCanRunQueries(user *fleet.User) bool { - if user.GlobalRole != nil && *user.GlobalRole == fleet.RoleObserver { - return true - } else if len(user.Teams) > 0 { - allObserver := true - for _, team := range user.Teams { - if team.Role != fleet.RoleObserver { - allObserver = false - break - } - } - return allObserver - } - return false -} - func (svc Service) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRead); err != nil { return nil, err } - user := authz.UserFromContext(ctx) - onlyShowObserverCanRun := onlyShowObserverCanRunQueries(user) - - queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{ - ListOptions: opt, - OnlyObserverCanRun: onlyShowObserverCanRun, - }) - if err != nil { - return nil, err - } - - return queries, nil + return svc.ds.ListQueries(ctx, opt) } func (svc *Service) GetQuery(ctx context.Context, id uint) (*fleet.Query, error) { diff --git a/server/service/service_queries_test.go b/server/service/service_queries_test.go index 5f3087a81a..5167d6b6f7 100644 --- a/server/service/service_queries_test.go +++ b/server/service/service_queries_test.go @@ -4,11 +4,8 @@ import ( "context" "testing" - "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,61 +21,3 @@ func TestNewQueryAttach(t *testing.T) { ) require.Error(t, err) } - -func TestFilterQueriesForObserver(t *testing.T) { - require.True(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)})) - require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)})) - require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)})) - - require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleObserver}}})) - require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ - {Role: fleet.RoleObserver}, - {Role: fleet.RoleObserver}, - }})) - require.False(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ - {Role: fleet.RoleObserver}, - {Role: fleet.RoleMaintainer}, - }})) -} - -func TestListQueries(t *testing.T) { - ds := new(mock.Store) - svc := newTestService(ds, nil, nil) - - cases := [...]struct { - title string - user *fleet.User - expectedOpts fleet.ListQueryOptions - }{ - { - title: "global admin", - user: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, - expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, - }, - { - title: "global observer", - user: &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, - expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: true}, - }, - { - title: "team admin", - user: &fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleAdmin}}}, - expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, - }, - } - - var calledWithOpts fleet.ListQueryOptions - ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { - calledWithOpts = opt - return []*fleet.Query{}, nil - } - - for _, tt := range cases { - t.Run(tt.title, func(t *testing.T) { - viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) - _, err := svc.ListQueries(viewerCtx, fleet.ListOptions{}) - require.NoError(t, err) - assert.Equal(t, tt.expectedOpts, calledWithOpts) - }) - } -} diff --git a/server/service/team_policies.go b/server/service/team_policies.go deleted file mode 100644 index 9ea4febc28..0000000000 --- a/server/service/team_policies.go +++ /dev/null @@ -1,156 +0,0 @@ -package service - -import ( - "context" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" -) - -///////////////////////////////////////////////////////////////////////////////// -// Add -///////////////////////////////////////////////////////////////////////////////// - -type teamPolicyRequest struct { - TeamID uint `url:"team_id"` - QueryID uint `json:"query_id"` -} - -type teamPolicyResponse struct { - Policy *fleet.Policy `json:"policy,omitempty"` - Err error `json:"error,omitempty"` -} - -func (r teamPolicyResponse) error() error { return r.Err } - -func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { - req := request.(*teamPolicyRequest) - resp, err := svc.NewTeamPolicy(ctx, req.TeamID, req.QueryID) - if err != nil { - return teamPolicyResponse{Err: err}, nil - } - return teamPolicyResponse{Policy: resp}, nil -} - -func (svc Service) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { - return nil, err - } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { - return nil, err - } - - return svc.ds.NewTeamPolicy(ctx, teamID, queryID) -} - -///////////////////////////////////////////////////////////////////////////////// -// List -///////////////////////////////////////////////////////////////////////////////// - -type listTeamPoliciesRequest struct { - TeamID uint `url:"team_id"` -} - -type listTeamPoliciesResponse struct { - Policies []*fleet.Policy `json:"policies,omitempty"` - Err error `json:"error,omitempty"` -} - -func (r listTeamPoliciesResponse) error() error { return r.Err } - -func listTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { - req := request.(*listTeamPoliciesRequest) - resp, err := svc.ListTeamPolicies(ctx, req.TeamID) - if err != nil { - return listTeamPoliciesResponse{Err: err}, nil - } - return listTeamPoliciesResponse{Policies: resp}, nil -} - -func (svc Service) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { - return nil, err - } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { - return nil, err - } - - return svc.ds.ListTeamPolicies(ctx, teamID) -} - -///////////////////////////////////////////////////////////////////////////////// -// Get by id -///////////////////////////////////////////////////////////////////////////////// - -type getTeamPolicyByIDRequest struct { - TeamID uint `url:"team_id"` - PolicyID uint `url:"policy_id"` -} - -type getTeamPolicyByIDResponse struct { - Policy *fleet.Policy `json:"policy"` - Err error `json:"error,omitempty"` -} - -func (r getTeamPolicyByIDResponse) error() error { return r.Err } - -func getTeamPolicyByIDEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { - req := request.(*getTeamPolicyByIDRequest) - teamPolicy, err := svc.GetTeamPolicyByIDQueries(ctx, req.TeamID, req.PolicyID) - if err != nil { - return getTeamPolicyByIDResponse{Err: err}, nil - } - return getTeamPolicyByIDResponse{Policy: teamPolicy}, nil -} - -func (svc Service) GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { - return nil, err - } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { - return nil, err - } - - teamPolicy, err := svc.ds.TeamPolicy(ctx, teamID, policyID) - if err != nil { - return nil, err - } - - return teamPolicy, nil -} - -///////////////////////////////////////////////////////////////////////////////// -// Delete -///////////////////////////////////////////////////////////////////////////////// - -type deleteTeamPoliciesRequest struct { - TeamID uint `url:"team_id"` - IDs []uint `json:"ids"` -} - -type deleteTeamPoliciesResponse struct { - Deleted []uint `json:"deleted,omitempty"` - Err error `json:"error,omitempty"` -} - -func (r deleteTeamPoliciesResponse) error() error { return r.Err } - -func deleteTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { - req := request.(*deleteTeamPoliciesRequest) - resp, err := svc.DeleteTeamPolicies(ctx, req.TeamID, req.IDs) - if err != nil { - return deleteTeamPoliciesResponse{Err: err}, nil - } - return deleteTeamPoliciesResponse{Deleted: resp}, nil -} - -func (svc Service) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { - return nil, err - } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { - return nil, err - } - - return svc.ds.DeleteTeamPolicies(ctx, teamID, ids) -} diff --git a/server/service/team_policies_test.go b/server/service/team_policies_test.go deleted file mode 100644 index 662dcce3ea..0000000000 --- a/server/service/team_policies_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package service - -import ( - "context" - "testing" - - "github.com/fleetdm/fleet/v4/server/authz" - "github.com/fleetdm/fleet/v4/server/contexts/viewer" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/mock" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/stretchr/testify/require" -) - -func TestTeamPoliciesAuth(t *testing.T) { - ds := new(mock.Store) - svc := newTestService(ds, nil, nil) - - ds.NewTeamPolicyFunc = func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { - return &fleet.Policy{}, nil - } - ds.ListTeamPoliciesFunc = func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { - return nil, nil - } - ds.TeamPolicyFunc = func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { - return nil, nil - } - ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { - return nil, nil - } - - var testCases = []struct { - name string - user *fleet.User - shouldFailWrite bool - shouldFailRead bool - }{ - { - "global admin", - &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, - false, - false, - }, - { - "global maintainer", - &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, - true, - false, - }, - { - "global observer", - &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, - true, - false, - }, - { - "team maintainer, belongs to team", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, - false, - false, - }, - { - "team observer, belongs to team", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, - true, - false, - }, - { - "team maintainer, DOES NOT belong to team", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleMaintainer}}}, - true, - true, - }, - { - "team observer, DOES NOT belong to team", - &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleObserver}}}, - true, - true, - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) - - _, err := svc.NewTeamPolicy(ctx, 1, 2) - checkAuthErr(t, tt.shouldFailWrite, err) - - _, err = svc.ListTeamPolicies(ctx, 1) - checkAuthErr(t, tt.shouldFailRead, err) - - _, err = svc.GetTeamPolicyByIDQueries(ctx, 1, 1) - checkAuthErr(t, tt.shouldFailRead, err) - - _, err = svc.DeleteTeamPolicies(ctx, 1, []uint{1}) - checkAuthErr(t, tt.shouldFailWrite, err) - }) - } -} - -func checkAuthErr(t *testing.T, shouldFail bool, err error) { - if shouldFail { - require.Error(t, err) - require.Equal(t, (&authz.Forbidden{}).Error(), err.Error()) - } else { - require.NoError(t, err) - } -} diff --git a/server/service/team_schedule.go b/server/service/team_schedule.go index 9561cd2595..f1578d966f 100644 --- a/server/service/team_schedule.go +++ b/server/service/team_schedule.go @@ -39,9 +39,6 @@ func (svc Service) GetTeamScheduledQueries(ctx context.Context, teamID uint, opt if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { - return nil, err - } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -105,9 +102,6 @@ func (svc Service) TeamScheduleQuery(ctx context.Context, teamID uint, q *fleet. if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { - return nil, err - } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -149,9 +143,6 @@ func (svc Service) ModifyTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return nil, err } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { - return nil, err - } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -192,8 +183,6 @@ func (svc Service) DeleteTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return err } - if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { - return err - } + _ = teamID return svc.DeleteScheduledQuery(ctx, scheduledQueryID) } diff --git a/server/service/testing_client.go b/server/service/testing_client.go deleted file mode 100644 index 89cbaf28eb..0000000000 --- a/server/service/testing_client.go +++ /dev/null @@ -1,148 +0,0 @@ -package service - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - - "github.com/fleetdm/fleet/v4/server/datastore/mysql" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/test" - "github.com/ghodss/yaml" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -type withDS struct { - s *suite.Suite - ds *mysql.Datastore -} - -func (ts *withDS) SetupSuite(dbName string) { - ts.ds = mysql.CreateNamedMySQLDS(ts.s.T(), dbName) - test.AddAllHostsLabel(ts.s.T(), ts.ds) -} - -func (ts *withDS) TearDownSuite() { - ts.ds.Close() -} - -type withServer struct { - withDS - - server *httptest.Server - users map[string]fleet.User - token string -} - -func (ts *withServer) SetupSuite(dbName string) { - ts.withDS.SetupSuite(dbName) - - users, server := RunServerForTestsWithDS(ts.s.T(), ts.ds) - ts.server = server - ts.users = users - ts.token = ts.getTestAdminToken() -} - -func (ts *withServer) TearDownSuite() { - ts.withDS.TearDownSuite() -} - -func (ts *withServer) Do(verb, path string, params interface{}, expectedStatusCode int) *http.Response { - t := ts.s.T() - - j, err := json.Marshal(params) - require.NoError(t, err) - - resp := ts.DoRaw(verb, path, j, expectedStatusCode) - - t.Cleanup(func() { - resp.Body.Close() - }) - return resp -} - -func (ts *withServer) DoRawWithHeaders(verb string, path string, rawBytes []byte, expectedStatusCode int, headers map[string]string) *http.Response { - t := ts.s.T() - - requestBody := io.NopCloser(bytes.NewBuffer(rawBytes)) - req, _ := http.NewRequest(verb, ts.server.URL+path, requestBody) - for key, val := range headers { - req.Header.Add(key, val) - } - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(t, err) - require.Equal(t, expectedStatusCode, resp.StatusCode) - - return resp -} - -func (ts *withServer) DoRaw(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { - return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, map[string]string{ - "Authorization": fmt.Sprintf("Bearer %s", ts.token), - }) -} - -func (ts *withServer) DoRawNoAuth(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { - return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, nil) -} - -func (ts *withServer) DoJSON(verb, path string, params interface{}, expectedStatusCode int, v interface{}) { - resp := ts.Do(verb, path, params, expectedStatusCode) - err := json.NewDecoder(resp.Body).Decode(v) - require.NoError(ts.s.T(), err) - if e, ok := v.(errorer); ok { - require.NoError(ts.s.T(), e.error()) - } -} - -func (ts *withServer) getTestAdminToken() string { - testUser := testUsers["admin1"] - - return ts.getTestToken(testUser.Email, testUser.PlaintextPassword) -} - -func (ts *withServer) getTestToken(email string, password string) string { - params := loginRequest{ - Email: email, - Password: password, - } - j, err := json.Marshal(¶ms) - require.NoError(ts.s.T(), err) - - requestBody := io.NopCloser(bytes.NewBuffer(j)) - resp, err := http.Post(ts.server.URL+"/api/v1/fleet/login", "application/json", requestBody) - require.NoError(ts.s.T(), err) - defer resp.Body.Close() - assert.Equal(ts.s.T(), http.StatusOK, resp.StatusCode) - - var jsn = struct { - User *fleet.User `json:"user"` - Token string `json:"token"` - Err []map[string]string `json:"errors,omitempty"` - }{} - err = json.NewDecoder(resp.Body).Decode(&jsn) - require.NoError(ts.s.T(), err) - require.Len(ts.s.T(), jsn.Err, 0) - - return jsn.Token -} - -func (ts *withServer) applyConfig(spec []byte) { - var appConfigSpec interface{} - err := yaml.Unmarshal(spec, &appConfigSpec) - require.NoError(ts.s.T(), err) - - ts.Do("PATCH", "/api/v1/fleet/config", appConfigSpec, http.StatusOK) -} - -func (ts *withServer) getConfig() *appConfigResponse { - var responseBody *appConfigResponse - ts.DoJSON("GET", "/api/v1/fleet/config", nil, http.StatusOK, &responseBody) - return responseBody -} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 8d57e4f344..3b2ab6316a 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -2,9 +2,6 @@ package service import ( "context" - "encoding/json" - "io/ioutil" - "net/http" "net/http/httptest" "os" "strings" @@ -12,7 +9,6 @@ import ( eeservice "github.com/fleetdm/fleet/v4/ee/server/service" "github.com/fleetdm/fleet/v4/server/logging" - "github.com/stretchr/testify/assert" "github.com/WatchBeam/clock" "github.com/fleetdm/fleet/v4/server/config" @@ -167,9 +163,6 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...TestServe limitStore, _ := memstore.New(0) r := MakeHandler(svc, config.FleetConfig{}, logger, limitStore) server := httptest.NewServer(r) - t.Cleanup(func() { - server.Close() - }) return users, server } @@ -242,21 +235,3 @@ func testStdoutPluginConfig() config.FleetConfig { c.Osquery.StatusLogPlugin = "stdout" return c } - -func assertBodyContains(t *testing.T, resp *http.Response, expected string) { - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err) - bodyString := string(bodyBytes) - assert.Contains(t, bodyString, expected) -} - -func getJSON(r *http.Response, target interface{}) error { - return json.NewDecoder(r.Body).Decode(target) -} - -func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { - err := &fleet.Error{} - require.Nil(t, getJSON(resp, err)) - assert.Equal(t, code, err.Code) - assert.Equal(t, message, err.Message) -} diff --git a/server/sso/session_store.go b/server/sso/session_store.go index 7b0bb36fdc..b2277c5619 100644 --- a/server/sso/session_store.go +++ b/server/sso/session_store.go @@ -46,7 +46,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin if len(requestID) < 8 { return errors.New("request id must be 8 or more characters in length") } - conn := s.pool.ConfigureDoer(s.pool.Get()) + conn := s.pool.Get() defer conn.Close() sess := Session{OriginalURL: originalURL, Metadata: metadata} var writer bytes.Buffer @@ -59,7 +59,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin } func (s *store) Get(requestID string) (*Session, error) { - conn := s.pool.ConfigureDoer(s.pool.Get()) + conn := s.pool.Get() defer conn.Close() val, err := redis.String(conn.Do("GET", requestID)) if err != nil { @@ -81,7 +81,7 @@ func (s *store) Get(requestID string) (*Session, error) { var ErrSessionNotFound = errors.New("session not found") func (s *store) Expire(requestID string) error { - conn := s.pool.ConfigureDoer(s.pool.Get()) + conn := s.pool.Get() defer conn.Close() _, err := conn.Do("DEL", requestID) return err diff --git a/server/sso/session_store_test.go b/server/sso/session_store_test.go index 318ba12acd..45b46af866 100644 --- a/server/sso/session_store_test.go +++ b/server/sso/session_store_test.go @@ -2,7 +2,6 @@ package sso import ( "os" - "runtime" "testing" "time" @@ -13,10 +12,6 @@ import ( ) func newPool(t *testing.T, cluster bool) fleet.RedisPool { - if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { - t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) - } - if _, ok := os.LookupEnv("REDIS_TEST"); ok { var ( addr = "127.0.0.1:" @@ -30,14 +25,7 @@ func newPool(t *testing.T, cluster bool) fleet.RedisPool { } addr += port - pool, err := redis.NewRedisPool(redis.PoolConfig{ - Server: addr, - Password: password, - Database: database, - UseTLS: useTLS, - ConnTimeout: 5 * time.Second, - KeepAlive: 10 * time.Second, - }) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) conn := pool.Get() defer conn.Close() diff --git a/server/test/new_objects.go b/server/test/new_objects.go index 171239ad79..aaa09a7f11 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -105,7 +105,6 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t UUID: uuid, DetailUpdatedAt: now, LabelUpdatedAt: now, - PolicyUpdatedAt: now, SeenTime: now, OsqueryHostID: osqueryHostID, }) diff --git a/server/vulnerabilities/cpe_test.go b/server/vulnerabilities/cpe_test.go index ed004902bf..e0e35e9170 100644 --- a/server/vulnerabilities/cpe_test.go +++ b/server/vulnerabilities/cpe_test.go @@ -53,10 +53,6 @@ func TestCpeFromSoftware(t *testing.T) { } func TestSyncCPEDatabase(t *testing.T) { - if os.Getenv("NETWORK_TEST") == "" { - t.Skip("set environment variable NETWORK_TEST=1 to run") - } - // Disabling vcr because the resulting file exceeds the 100mb limit for github r, err := recorder.NewAsMode("fixtures/nvd-cpe-release", recorder.ModeDisabled, http.DefaultTransport) require.NoError(t, err) @@ -148,10 +144,6 @@ func (f *fakeSoftwareIterator) Err() error { return nil } func (f *fakeSoftwareIterator) Close() error { f.closed = true; return nil } func TestTranslateSoftwareToCPE(t *testing.T) { - if os.Getenv("NETWORK_TEST") == "" { - t.Skip("set environment variable NETWORK_TEST=1 to run") - } - tempDir, err := os.MkdirTemp(os.TempDir(), "TestTranslateSoftwareToCPE-*") require.NoError(t, err) defer os.RemoveAll(tempDir) diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index cbf984b600..adc83e8082 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -26,10 +26,6 @@ var cvetests = []struct { } func TestTranslateCPEToCVE(t *testing.T) { - if os.Getenv("NETWORK_TEST") == "" { - t.Skip("set environment variable NETWORK_TEST=1 to run") - } - tempDir := t.TempDir() ds := new(mock.Store) diff --git a/tools/backup_db/README.md b/tools/backup_db/README.md index 11d7ad61a8..39f52ea042 100644 --- a/tools/backup_db/README.md +++ b/tools/backup_db/README.md @@ -1,3 +1,3 @@ These scripts are for backing up and restore the Docker development MySQL database. -Usage is documented [here](../../docs/03-Contributing/02-Testing.md#database-backuprestore). +Usage is documented [here](../../docs/3-Contributing/2-Testing.md#database-backuprestore). diff --git a/tools/fleetctl-npm/README.md b/tools/fleetctl-npm/README.md index e303e10c50..52ff50a021 100644 --- a/tools/fleetctl-npm/README.md +++ b/tools/fleetctl-npm/README.md @@ -10,4 +10,4 @@ Simply install `fleetctl` with `npm install -g fleetctl`. ## Usage -See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. +See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json index 8634b01ca6..3d03c517c5 100644 --- a/tools/fleetctl-npm/package.json +++ b/tools/fleetctl-npm/package.json @@ -1,6 +1,6 @@ { "name": "fleetctl", - "version": "v4.3.1", + "version": "v4.3.0", "description": "Installer for the fleetctl CLI tool", "bin": { "fleetctl": "./run.js" diff --git a/tools/terraform/.terraform-version b/tools/terraform/.terraform-version deleted file mode 100644 index a6a3a43c3a..0000000000 --- a/tools/terraform/.terraform-version +++ /dev/null @@ -1 +0,0 @@ -1.0.4 \ No newline at end of file diff --git a/tools/terraform/ecs-iam.tf b/tools/terraform/ecs-iam.tf deleted file mode 100644 index c726f85398..0000000000 --- a/tools/terraform/ecs-iam.tf +++ /dev/null @@ -1,54 +0,0 @@ -data "aws_iam_policy_document" "fleet" { - statement { - effect = "Allow" - actions = ["cloudwatch:PutMetricData"] - resources = ["*"] - } - - statement { - effect = "Allow" - actions = ["secretsmanager:GetSecretValue"] - resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn] - } - - statement { - effect = "Allow" - actions = [ - "firehose:DescribeDeliveryStream", - "firehose:PutRecord", - "firehose:PutRecordBatch", - ] - resources = [aws_kinesis_firehose_delivery_stream.osquery_logs.arn] - } -} - -data "aws_iam_policy_document" "assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - principals { - identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] - type = "Service" - } - } -} - -resource "aws_iam_role" "main" { - name = "fleetdm-role" - assume_role_policy = data.aws_iam_policy_document.assume_role.json -} - -resource "aws_iam_role_policy_attachment" "role_attachment" { - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" - role = aws_iam_role.main.name -} - -resource "aws_iam_policy" "main" { - name = "fleet-iam-policy" - policy = data.aws_iam_policy_document.fleet.json -} - -resource "aws_iam_role_policy_attachment" "attachment" { - policy_arn = aws_iam_policy.main.arn - role = aws_iam_role.main.name -} \ No newline at end of file diff --git a/tools/terraform/ecs-sgs.tf b/tools/terraform/ecs-sgs.tf deleted file mode 100644 index c5d3222dbd..0000000000 --- a/tools/terraform/ecs-sgs.tf +++ /dev/null @@ -1,78 +0,0 @@ -# Security group for the public internet facing load balancer -resource "aws_security_group" "lb" { - name = "${var.prefix} load balancer" - description = "${var.prefix} Load balancer security group" - vpc_id = module.vpc.vpc_id -} - -# Allow traffic from public internet -resource "aws_security_group_rule" "lb-ingress" { - description = "${var.prefix}: allow traffic from public internet" - type = "ingress" - - from_port = "443" - to_port = "443" - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - - security_group_id = aws_security_group.lb.id -} - -resource "aws_security_group_rule" "lb-http-ingress" { - description = "${var.prefix}: allow traffic from public internet" - type = "ingress" - - from_port = "80" - to_port = "80" - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - - security_group_id = aws_security_group.lb.id -} - -# Allow outbound traffic -resource "aws_security_group_rule" "lb-egress" { - description = "${var.prefix}: allow all outbound traffic" - type = "egress" - - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - - security_group_id = aws_security_group.lb.id -} - -# Security group for the backends that run the application. -# Allows traffic from the load balancer -resource "aws_security_group" "backend" { - name = "${var.prefix} backend" - description = "${var.prefix} Backend security group" - vpc_id = module.vpc.vpc_id - -} - -# Allow traffic from the load balancer to the backends -resource "aws_security_group_rule" "backend-ingress" { - description = "${var.prefix}: allow traffic from load balancer" - type = "ingress" - - from_port = "8080" - to_port = "8080" - protocol = "tcp" - source_security_group_id = aws_security_group.lb.id - security_group_id = aws_security_group.backend.id -} - -# Allow outbound traffic from the backends -resource "aws_security_group_rule" "backend-egress" { - description = "${var.prefix}: allow all outbound traffic" - type = "egress" - - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - - security_group_id = aws_security_group.backend.id -} \ No newline at end of file diff --git a/tools/terraform/ecs.tf b/tools/terraform/ecs.tf deleted file mode 100644 index 090cc6a2da..0000000000 --- a/tools/terraform/ecs.tf +++ /dev/null @@ -1,328 +0,0 @@ -//resource "aws_route53_record" "record" { -// name = "fleetdm" -// type = "A" -// zone_id = "Z046188311R47QSK245X" -// alias { -// evaluate_target_health = false -// name = aws_alb.main.dns_name -// zone_id = aws_alb.main.zone_id -// } -//} - -resource "aws_alb" "main" { - name = "fleetdm" - internal = false - security_groups = [aws_security_group.lb.id, aws_security_group.backend.id] - subnets = module.vpc.public_subnets -} - -resource "aws_alb_target_group" "main" { - name = "fleetdm" - protocol = "HTTP" - target_type = "ip" - port = "8080" - vpc_id = module.vpc.vpc_id - deregistration_delay = 30 - - load_balancing_algorithm_type = "least_outstanding_requests" - - health_check { - path = "/healthz" - matcher = "200" - timeout = 10 - interval = 15 - healthy_threshold = 5 - unhealthy_threshold = 5 - } - - depends_on = [aws_alb.main] -} - -resource "aws_alb_listener" "https-fleetdm" { - load_balancer_arn = aws_alb.main.arn - port = 443 - protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" - certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn - - default_action { - target_group_arn = aws_alb_target_group.main.arn - type = "forward" - } -} - -resource "aws_alb_listener" "http" { - load_balancer_arn = aws_alb.main.arn - port = "80" - protocol = "HTTP" - - default_action { - type = "redirect" - - redirect { - port = "443" - protocol = "HTTPS" - status_code = "HTTP_301" - } - } -} - -resource "aws_ecs_cluster" "fleet" { - name = "${var.prefix}-backend" - - setting { - name = "containerInsights" - value = "enabled" - } -} - -resource "aws_ecs_service" "fleet" { - name = "fleet" - launch_type = "FARGATE" - cluster = aws_ecs_cluster.fleet.id - task_definition = aws_ecs_task_definition.backend.arn - desired_count = 1 - deployment_minimum_healthy_percent = 100 - deployment_maximum_percent = 200 - health_check_grace_period_seconds = 30 - - load_balancer { - target_group_arn = aws_alb_target_group.main.arn - container_name = "fleet" - container_port = 8080 - } - - network_configuration { - subnets = module.vpc.private_subnets - security_groups = [aws_security_group.backend.id] - } - - depends_on = [aws_alb_listener.http, aws_alb_listener.https-fleetdm] -} - -resource "aws_cloudwatch_log_group" "backend" { - name = "fleetdm" - retention_in_days = 1 -} - -data "aws_region" "current" {} - -data "aws_secretsmanager_secret" "license" { - name = "/fleet/license" -} - -resource "aws_ecs_task_definition" "backend" { - family = "fleet" - network_mode = "awsvpc" - requires_compatibilities = ["FARGATE"] - execution_role_arn = aws_iam_role.main.arn - task_role_arn = aws_iam_role.main.arn - cpu = 512 - memory = 4096 - container_definitions = jsonencode( - [ - { - name = "fleet" - image = "fleetdm/fleet" - cpu = 512 - memory = 4096 - mountPoints = [] - volumesFrom = [] - essential = true - portMappings = [ - { - # This port is the same that the contained application also uses - containerPort = 8080 - protocol = "tcp" - } - ] - networkMode = "awsvpc" - logConfiguration = { - logDriver = "awslogs" - options = { - awslogs-group = aws_cloudwatch_log_group.backend.name - awslogs-region = data.aws_region.current.name - awslogs-stream-prefix = "fleet" - } - }, - secrets = [ - { - name = "FLEET_MYSQL_PASSWORD" - valueFrom = aws_secretsmanager_secret.database_password_secret.arn - }, - { - name = "FLEET_MYSQL_READ_REPLICA_PASSWORD" - valueFrom = aws_secretsmanager_secret.database_password_secret.arn - }, - { - name = "FLEET_LICENSE_KEY" - valueFrom = data.aws_secretsmanager_secret.license.arn - } - ] - environment = [ - { - name = "FLEET_MYSQL_USERNAME" - value = "fleet" - }, - { - name = "FLEET_MYSQL_DATABASE" - value = "fleet" - }, - { - name = "FLEET_MYSQL_ADDRESS" - value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" - }, - { - name = "FLEET_MYSQL_READ_REPLICA_USERNAME" - value = "fleet" - }, - { - name = "FLEET_MYSQL_READ_REPLICA_DATABASE" - value = "fleet" - }, - { - name = "FLEET_MYSQL_READ_REPLICA_ADDRESS" - value = "${module.aurora_mysql.rds_cluster_reader_endpoint}:3306" - }, - { - name = "FLEET_REDIS_ADDRESS" - value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" - }, - { - name = "FLEET_FIREHOSE_STATUS_STREAM" - value = aws_kinesis_firehose_delivery_stream.osquery_logs.name - }, - { - name = "FLEET_FIREHOSE_RESULT_STREAM" - value = aws_kinesis_firehose_delivery_stream.osquery_logs.name - }, - { - name = "FLEET_FIREHOSE_REGION" - value = data.aws_region.current.name - }, - { - name = "FLEET_OSQUERY_STATUS_LOG_PLUGIN" - value = "firehose" - }, - { - name = "FLEET_OSQUERY_RESULT_LOG_PLUGIN" - value = "firehose" - }, - { - name = "FLEET_SERVER_TLS" - value = "false" - }, - { - name = "FLEET_BETA_SOFTWARE_INVENTORY" - value = "1" - }, - { - name = "FLEET_VULNERABILITIES_DATABASES_PATH" - value = "/home/fleet" - } - ] - } - ]) -} - -resource "aws_ecs_task_definition" "migration" { - family = "fleet-migrate" - network_mode = "awsvpc" - requires_compatibilities = ["FARGATE"] - execution_role_arn = aws_iam_role.main.arn - task_role_arn = aws_iam_role.main.arn - cpu = 256 - memory = 512 - container_definitions = jsonencode( - [ - { - name = "fleet-prepare-db" - image = "fleetdm/fleet" - cpu = 256 - memory = 512 - mountPoints = [] - volumesFrom = [] - essential = true - portMappings = [ - { - # This port is the same that the contained application also uses - containerPort = 8080 - protocol = "tcp" - } - ] - networkMode = "awsvpc" - logConfiguration = { - logDriver = "awslogs" - options = { - awslogs-group = aws_cloudwatch_log_group.backend.name - awslogs-region = data.aws_region.current.name - awslogs-stream-prefix = "fleet" - } - }, - command = ["fleet", "prepare", "db"] - secrets = [ - { - name = "FLEET_MYSQL_PASSWORD" - valueFrom = aws_secretsmanager_secret.database_password_secret.arn - } - ] - environment = [ - { - name = "FLEET_MYSQL_USERNAME" - value = "fleet" - }, - { - name = "FLEET_MYSQL_DATABASE" - value = "fleet" - }, - { - name = "FLEET_MYSQL_ADDRESS" - value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" - }, - { - name = "FLEET_REDIS_ADDRESS" - value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" - } - ] - } - ]) -} - -resource "aws_appautoscaling_target" "ecs_target" { - max_capacity = 5 - min_capacity = 1 - resource_id = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}" - scalable_dimension = "ecs:service:DesiredCount" - service_namespace = "ecs" -} - -resource "aws_appautoscaling_policy" "ecs_policy_memory" { - name = "fleet-memory-autoscaling" - policy_type = "TargetTrackingScaling" - resource_id = aws_appautoscaling_target.ecs_target.resource_id - scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension - service_namespace = aws_appautoscaling_target.ecs_target.service_namespace - - target_tracking_scaling_policy_configuration { - predefined_metric_specification { - predefined_metric_type = "ECSServiceAverageMemoryUtilization" - } - target_value = 80 - } -} - -resource "aws_appautoscaling_policy" "ecs_policy_cpu" { - name = "fleet-cpu-autoscaling" - policy_type = "TargetTrackingScaling" - resource_id = aws_appautoscaling_target.ecs_target.resource_id - scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension - service_namespace = aws_appautoscaling_target.ecs_target.service_namespace - - target_tracking_scaling_policy_configuration { - predefined_metric_specification { - predefined_metric_type = "ECSServiceAverageCPUUtilization" - } - - target_value = 60 - } -} diff --git a/tools/terraform/firehose.tf b/tools/terraform/firehose.tf deleted file mode 100644 index ceed78ac42..0000000000 --- a/tools/terraform/firehose.tf +++ /dev/null @@ -1,69 +0,0 @@ -resource "aws_s3_bucket" "osquery" { - bucket = "fleet-osquery-logs-archive" - acl = "private" - - lifecycle_rule { - enabled = true - expiration { - days = 1 - } - } - - server_side_encryption_configuration { - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "aws:kms" - } - } - } -} - -// allow firehose to write to bucket -data "aws_iam_policy_document" "osquery_logs_policy_doc" { - statement { - effect = "Allow" - actions = [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject" - ] - resources = [aws_s3_bucket.osquery.arn, "${aws_s3_bucket.osquery.arn}/*"] - } -} - -resource "aws_iam_policy" "firehose" { - name = "osquery_logs_firehose_policy" - policy = data.aws_iam_policy_document.osquery_logs_policy_doc.json -} - -resource "aws_iam_role" "firehose" { - assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json -} - -resource "aws_iam_role_policy_attachment" "firehose" { - policy_arn = aws_iam_policy.firehose.arn - role = aws_iam_role.firehose.name -} - -data "aws_iam_policy_document" "osquery_firehose_assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - principals { - identifiers = ["firehose.amazonaws.com"] - type = "Service" - } - } -} - -resource "aws_kinesis_firehose_delivery_stream" "osquery_logs" { - name = "osquery_logs" - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.osquery.arn - } -} \ No newline at end of file diff --git a/tools/terraform/main.tf b/tools/terraform/main.tf deleted file mode 100644 index 95bb588b3e..0000000000 --- a/tools/terraform/main.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "region" { - default = "us-east-2" -} - -provider "aws" { - region = var.region -} - -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.54.0" - } - } -} - -data "aws_caller_identity" "current" {} - diff --git a/tools/terraform/outputs.tf b/tools/terraform/outputs.tf deleted file mode 100644 index caa6ef2314..0000000000 --- a/tools/terraform/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "nameservers_fleetctl" { - value = aws_route53_zone.dogfood_fleetctl_com.name_servers -} - -output "nameservers_fleetdm" { - value = aws_route53_zone.dogfood_fleetdm_com.name_servers -} diff --git a/tools/terraform/r53.tf b/tools/terraform/r53.tf deleted file mode 100644 index 619483a545..0000000000 --- a/tools/terraform/r53.tf +++ /dev/null @@ -1,93 +0,0 @@ -resource "aws_route53_zone" "dogfood_fleetctl_com" { - name = "dogfood.fleetctl.com" -} - -resource "aws_route53_zone" "dogfood_fleetdm_com" { - name = "dogfood.fleetdm.com" -} - -resource "aws_route53_record" "dogfood_fleetctl_com" { - zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id - name = "dogfood.fleetctl.com" - type = "A" - - alias { - name = aws_alb.main.dns_name - zone_id = aws_alb.main.zone_id - evaluate_target_health = false - } -} - -resource "aws_route53_record" "dogfood_fleetdm_com" { - zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id - name = "dogfood.fleetdm.com" - type = "A" - - alias { - name = aws_alb.main.dns_name - zone_id = aws_alb.main.zone_id - evaluate_target_health = false - } -} - -resource "aws_acm_certificate" "dogfood_fleetctl_com" { - domain_name = "dogfood.fleetctl.com" - validation_method = "DNS" - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_acm_certificate" "dogfood_fleetdm_com" { - domain_name = "dogfood.fleetdm.com" - validation_method = "DNS" - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_route53_record" "dogfood_fleetctl_com_validation" { - for_each = { - for dvo in aws_acm_certificate.dogfood_fleetctl_com.domain_validation_options : dvo.domain_name => { - name = dvo.resource_record_name - record = dvo.resource_record_value - type = dvo.resource_record_type - } - } - - allow_overwrite = true - name = each.value.name - records = [each.value.record] - ttl = 60 - type = each.value.type - zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id -} - -resource "aws_route53_record" "dogfood_fleetdm_com_validation" { - for_each = { - for dvo in aws_acm_certificate.dogfood_fleetdm_com.domain_validation_options : dvo.domain_name => { - name = dvo.resource_record_name - record = dvo.resource_record_value - type = dvo.resource_record_type - } - } - - allow_overwrite = true - name = each.value.name - records = [each.value.record] - ttl = 60 - type = each.value.type - zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id -} - -resource "aws_acm_certificate_validation" "dogfood_fleetctl_com" { - certificate_arn = aws_acm_certificate.dogfood_fleetctl_com.arn - validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetctl_com_validation : record.fqdn] -} - -resource "aws_acm_certificate_validation" "dogfood_fleetdm_com" { - certificate_arn = aws_acm_certificate.dogfood_fleetdm_com.arn - validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetdm_com_validation : record.fqdn] -} \ No newline at end of file diff --git a/tools/terraform/rds.tf b/tools/terraform/rds.tf deleted file mode 100644 index c4a13466d6..0000000000 --- a/tools/terraform/rds.tf +++ /dev/null @@ -1,113 +0,0 @@ -locals { - name = "fleetdm" -} - -resource "random_password" "database_password" { - length = 16 - special = false -} - -resource "aws_secretsmanager_secret" "database_password_secret" { - name = "/fleet/database/password/master" -} - -resource "aws_secretsmanager_secret_version" "database_password_secret_version" { - secret_id = aws_secretsmanager_secret.database_password_secret.id - secret_string = random_password.database_password.result -} - -// if you want to use RDS Serverless option prefer the following commented block -//module "aurora_mysql_serverless" { -// source = "terraform-aws-modules/rds-aurora/aws" -// version = "5.2.0" -// -// name = "${local.name}-mysql" -// engine = "aurora-mysql" -// engine_mode = "serverless" -// storage_encrypted = true -// username = "fleet" -// password = random_password.database_password.result -// create_random_password = false -// database_name = "fleet" -// enable_http_endpoint = true -// -// vpc_id = module.vpc.vpc_id -// subnets = module.vpc.database_subnets -// create_security_group = true -// allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks -// -// replica_scale_enabled = false -// replica_count = 0 -// -// monitoring_interval = 60 -// -// apply_immediately = true -// skip_final_snapshot = true -// -// db_parameter_group_name = aws_db_parameter_group.example_mysql.id -// db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id -// -// scaling_configuration = { -// auto_pause = true -// min_capacity = 2 -// max_capacity = 16 -// seconds_until_auto_pause = 300 -// timeout_action = "ForceApplyCapacityChange" -// } -//} - -module "aurora_mysql" { - source = "terraform-aws-modules/rds-aurora/aws" - version = "5.2.0" - - name = "${local.name}-mysql-iam" - engine = "aurora-mysql" - engine_version = "5.7.mysql_aurora.2.10.0" - instance_type = "db.t4g.medium" - instance_type_replica = "db.t4g.medium" - - iam_database_authentication_enabled = true - storage_encrypted = true - username = "fleet" - password = random_password.database_password.result - create_random_password = false - database_name = "fleet" - enable_http_endpoint = false - #performance_insights_enabled = true - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.database_subnets - create_security_group = true - allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks - - replica_count = 1 - replica_scale_enabled = true - replica_scale_min = 1 - replica_scale_max = 3 - - monitoring_interval = 60 - iam_role_name = "${local.name}-rds-enhanced-monitoring" - iam_role_use_name_prefix = true - iam_role_description = "${local.name} RDS enhanced monitoring IAM role" - iam_role_path = "/autoscaling/" - iam_role_max_session_duration = 7200 - - apply_immediately = true - skip_final_snapshot = true - - db_parameter_group_name = aws_db_parameter_group.example_mysql.id - db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id - -} - -resource "aws_db_parameter_group" "example_mysql" { - name = "${local.name}-aurora-db-mysql-parameter-group" - family = "aurora-mysql5.7" - description = "${local.name}-aurora-db-mysql-parameter-group" -} - -resource "aws_rds_cluster_parameter_group" "example_mysql" { - name = "${local.name}-aurora-mysql-cluster-parameter-group" - family = "aurora-mysql5.7" - description = "${local.name}-aurora-mysql-cluster-parameter-group" -} \ No newline at end of file diff --git a/tools/terraform/readme.md b/tools/terraform/readme.md deleted file mode 100644 index 5c6cb4baae..0000000000 --- a/tools/terraform/readme.md +++ /dev/null @@ -1,44 +0,0 @@ -## Terraform - -`terraform init && terraform workspace new dev` - -`terraform plan` - -`terraform apply` - -### Configuration - -Typical settings to override in an existing environment: - -`module.vpc.vpc_id` -- the VPC ID output from VPC module. If you are introducing fleet to an existing VPC, you could replace all instances with your VPC ID. - -In this reference architecture we are placing ECS, RDS MySQL, and Redis (ElastiCache) in separate subnets, each associated to a route table, allowing communication between. -This is not required, as long as Fleet can resolve the MySQL and Redis hosts, that should be adequate. - -#### HTTPS - -The ALB is in the public subnet with an ENI to bridge into the private subnet. SSL is terminated at the ALB and `fleet serve` is launched with `FLEET_SERVER_TLS=false` as an -environment variable. - -Replace `cert_arn` with the **certificate ARN** that applies to your environment. This is the **certificate ARN** used in the **ALB HTTPS Listener**. - -### Migrating the DB - -After applying terraform run the following to migrate the database: -``` -aws ecs run-task --cluster fleet-backend --task-definition fleet-migrate: --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets=[],securityGroups=[]}" -``` - -### Connecting a Host - -Build orbit: - -``` - fleetctl package --type=msi --fleet-url= --enroll-secret= -``` - -Run orbit: - -``` - "C:\Program Files\Orbit\bin\orbit\orbit.exe" --root-dir "C:\Program Files\Orbit\." --log-file "C:\Program Files\Orbit\orbit-log.txt" --fleet-url "http://" --enroll-secret-path "C:\Program Files\Orbit\secret.txt" --update-url "https://tuf.fleetctl.com" --orbit-channel "stable" --osqueryd-channel "stable" -``` \ No newline at end of file diff --git a/tools/terraform/redis.tf b/tools/terraform/redis.tf deleted file mode 100644 index d289472318..0000000000 --- a/tools/terraform/redis.tf +++ /dev/null @@ -1,65 +0,0 @@ -variable "maintenance_window" { - default = "" -} -variable "engine_version" { - default = "5.0.6" -} -variable "node_type" { - default = "cache.t2.micro" -} -variable "number_cache_clusters" { - default = 3 -} -resource "aws_elasticache_replication_group" "default" { - availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] - engine = "redis" - parameter_group_name = aws_elasticache_parameter_group.default.name - subnet_group_name = module.vpc.elasticache_subnet_group_name - security_group_ids = [aws_security_group.redis.id] - replication_group_id = "fleetdm-redis" - number_cache_clusters = var.number_cache_clusters - node_type = var.node_type - engine_version = var.engine_version - port = "6379" - maintenance_window = var.maintenance_window - snapshot_retention_limit = 0 - automatic_failover_enabled = false - at_rest_encryption_enabled = false - transit_encryption_enabled = false - apply_immediately = true - replication_group_description = "fleetdm-redis" -} - -resource "aws_elasticache_parameter_group" "default" { - name = "fleetdm-redis" - family = "redis5.0" - description = "for fleet" -} - -resource "aws_security_group" "redis" { - name = local.security_group_name - vpc_id = module.vpc.vpc_id -} - -locals { - security_group_name = "${var.prefix}-elasticache-redis" -} - -resource "aws_security_group_rule" "ingress" { - type = "ingress" - from_port = "6379" - to_port = "6379" - protocol = "tcp" - cidr_blocks = module.vpc.private_subnets_cidr_blocks - security_group_id = aws_security_group.redis.id -} - -resource "aws_security_group_rule" "egress" { - type = "egress" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = aws_security_group.redis.id -} - diff --git a/tools/terraform/variables.tf b/tools/terraform/variables.tf deleted file mode 100644 index 40d60b7d4d..0000000000 --- a/tools/terraform/variables.tf +++ /dev/null @@ -1,3 +0,0 @@ -variable "prefix" { - default = "fleet" -} \ No newline at end of file diff --git a/tools/terraform/vpc.tf b/tools/terraform/vpc.tf deleted file mode 100644 index ed9fa8b4a7..0000000000 --- a/tools/terraform/vpc.tf +++ /dev/null @@ -1,24 +0,0 @@ -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - - name = "fleet-vpc" - cidr = "10.10.0.0/16" - - azs = ["us-east-2a", "us-east-2b", "us-east-2c"] - private_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] - public_subnets = ["10.10.11.0/24", "10.10.12.0/24", "10.10.13.0/24"] - database_subnets = ["10.10.21.0/24", "10.10.22.0/24", "10.10.23.0/24"] - elasticache_subnets = ["10.10.31.0/24", "10.10.32.0/24", "10.10.33.0/24"] - - create_database_subnet_group = true - create_database_subnet_route_table = true - - create_elasticache_subnet_group = true - create_elasticache_subnet_route_table = true - - enable_vpn_gateway = false - one_nat_gateway_per_az = false - - single_nat_gateway = true - enable_nat_gateway = true -} \ No newline at end of file diff --git a/website/.sailsrc b/website/.sailsrc index 7499bf8d2f..9f58d730b0 100644 --- a/website/.sailsrc +++ b/website/.sailsrc @@ -12,264 +12,224 @@ "url": "/docs", "title": "Readme.md", "lastModifiedAt": 1624049901000, - "htmlId": "docs--readme--27004f4448", - "sectionRelativeRepoPath": "README.md", - "meta": {} - }, - { - "url": "/docs/deploying/installation", - "title": "Installation", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--01-installation--c1f7b7262d", - "sectionRelativeRepoPath": "02-Deploying/01-Installation.md", - "meta": {} - }, - { - "url": "/docs/deploying/configuration", - "title": "Configuration", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--02-configuration--25bb47a163", - "sectionRelativeRepoPath": "02-Deploying/02-Configuration.md", - "meta": {} - }, - { - "url": "/docs/deploying/example-deployment-scenarios", - "title": "Example deployment scenarios", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--03-example-deploymen--1d32b988ab", - "sectionRelativeRepoPath": "02-Deploying/03-Example-deployment-scenarios.md", - "meta": {} - }, - { - "url": "/docs/deploying/fleetctl-agent-updates", - "title": "Fleetctl agent updates", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--04-fleetctl-agent-up--92c6890fa9", - "sectionRelativeRepoPath": "02-Deploying/04-fleetctl-agent-updates.md", - "meta": {} - }, - { - "url": "/docs/deploying/faq", - "title": "FAQ", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--faq--3ad91393ce", - "sectionRelativeRepoPath": "02-Deploying/FAQ.md", - "meta": {} - }, - { - "url": "/docs/deploying", - "title": "Deploying", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--readme--fb635b427f", - "sectionRelativeRepoPath": "02-Deploying/README.md", - "meta": {} - }, - { - "url": "/docs/contributing/building-fleet", - "title": "Building Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--01-building-fleet--abcea456d8", - "sectionRelativeRepoPath": "03-Contributing/01-Building-Fleet.md", - "meta": {} - }, - { - "url": "/docs/contributing/testing", - "title": "Testing", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--02-testing--2f307719a6", - "sectionRelativeRepoPath": "03-Contributing/02-Testing.md", - "meta": {} - }, - { - "url": "/docs/contributing/migrations", - "title": "Migrations", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--03-migrations--b553b6254f", - "sectionRelativeRepoPath": "03-Contributing/03-Migrations.md", - "meta": {} - }, - { - "url": "/docs/contributing/committing-changes", - "title": "Committing changes", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--04-committing-change--9b92fdc560", - "sectionRelativeRepoPath": "03-Contributing/04-Committing-Changes.md", - "meta": {} - }, - { - "url": "/docs/contributing/releasing-fleet", - "title": "Releasing Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--05-releasing-fleet--1f39f77c64", - "sectionRelativeRepoPath": "03-Contributing/05-Releasing-Fleet.md", - "meta": {} - }, - { - "url": "/docs/contributing/seeding-data", - "title": "Seeding data", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--06-seeding-data--af5ac86a99", - "sectionRelativeRepoPath": "03-Contributing/06-Seeding-Data.md", - "meta": {} - }, - { - "url": "/docs/contributing/faq", - "title": "FAQ", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--faq--1b33e57806", - "sectionRelativeRepoPath": "03-Contributing/FAQ.md", - "meta": {} - }, - { - "url": "/docs/contributing", - "title": "Contributing", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--readme--6de1bc799d", - "sectionRelativeRepoPath": "03-Contributing/README.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleet-ui", - "title": "Fleet UI", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--01-fleet-ui--4b5755ee58", - "sectionRelativeRepoPath": "01-Using-Fleet/01-Fleet-UI.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleetctl-cli", - "title": "Fleetctl CLI", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--02-fleetctl-cli--2a521b49d6", - "sectionRelativeRepoPath": "01-Using-Fleet/02-fleetctl-CLI.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/rest-api", - "title": "REST API", - "lastModifiedAt": 1632174198000, - "htmlId": "docs--03-rest-api--f0b4e26bae", - "sectionRelativeRepoPath": "01-Using-Fleet/03-REST-API.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/adding-hosts", - "title": "Adding hosts", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--04-adding-hosts--9ffccb2221", - "sectionRelativeRepoPath": "01-Using-Fleet/04-Adding-hosts.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/osquery-logs", - "title": "Osquery logs", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--05-osquery-logs--7fbf2c5c5a", - "sectionRelativeRepoPath": "01-Using-Fleet/05-Osquery-logs.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/monitoring-fleet", - "title": "Monitoring Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--06-monitoring-fleet--83f7cca9f9", - "sectionRelativeRepoPath": "01-Using-Fleet/06-Monitoring-Fleet.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/security-best-practices", - "title": "Security best practices", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--07-security-best-pra--7ba1af6048", - "sectionRelativeRepoPath": "01-Using-Fleet/07-Security-best-practices.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/updating-fleet", - "title": "Updating Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--08-updating-fleet--3b4e821ee3", - "sectionRelativeRepoPath": "01-Using-Fleet/08-Updating-Fleet.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/permissions", - "title": "Permissions", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--09-permissions--eb9ac05ff5", - "sectionRelativeRepoPath": "01-Using-Fleet/09-Permissions.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/teams", - "title": "Teams", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--10-teams--bd0bdf9444", - "sectionRelativeRepoPath": "01-Using-Fleet/10-Teams.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/usage-statistics", - "title": "Usage statistics", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--11-usage-statistics--ccd73f532c", - "sectionRelativeRepoPath": "01-Using-Fleet/11-Usage-statistics.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/supported-browsers", - "title": "Supported browsers", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--12-supported-browser--c3a9c18d40", - "sectionRelativeRepoPath": "01-Using-Fleet/12-Supported-browsers.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/vulnerability-processing", - "title": "Vulnerability processing", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--13-vulnerability-pro--7a9b62b621", - "sectionRelativeRepoPath": "01-Using-Fleet/13-Vulnerability-Processing.md", - "meta": {} - }, - { - "url": "/docs/using-fleet/faq", - "title": "FAQ", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--faq--75e099695e", - "sectionRelativeRepoPath": "01-Using-Fleet/FAQ.md", - "meta": {} - }, - { - "url": "/docs/using-fleet", - "title": "Using Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--readme--0b226f5257", - "sectionRelativeRepoPath": "01-Using-Fleet/README.md", + "htmlId": "docs--readme--9f534d32b2", "meta": {} }, { "url": "/docs/using-fleet/learn-how-to-use-fleet", "title": "Learn how to use Fleet", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--00-learn-how-to-use---95b515dfd1", - "sectionRelativeRepoPath": "01-Using-Fleet/00-Learn-how-to-use-Fleet.md", + "lastModifiedAt": 1631573400000, + "htmlId": "docs--0-learn-how-to-use-f--1b80658ae8", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleet-ui", + "title": "Fleet UI", + "lastModifiedAt": 1631640519000, + "htmlId": "docs--1-fleet-ui--ed954948be", + "meta": {} + }, + { + "url": "/docs/using-fleet/teams", + "title": "Teams", + "lastModifiedAt": 1629395421000, + "htmlId": "docs--10-teams--782f2af710", + "meta": {} + }, + { + "url": "/docs/using-fleet/usage-statistics", + "title": "Usage statistics", + "lastModifiedAt": 1624989594000, + "htmlId": "docs--11-usage-statistics--3ed9f3101b", + "meta": {} + }, + { + "url": "/docs/using-fleet/supported-browsers", + "title": "Supported browsers", + "lastModifiedAt": 1630452786000, + "htmlId": "docs--12-supported-browser--6f8b591603", + "meta": {} + }, + { + "url": "/docs/using-fleet/vulnerability-processing", + "title": "Vulnerability processing", + "lastModifiedAt": 1629761820000, + "htmlId": "docs--13-vulnerability-pro--edb754352c", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleetctl-cli", + "title": "Fleetctl CLI", + "lastModifiedAt": 1631040955000, + "htmlId": "docs--2-fleetctl-cli--b4a4f6b08c", + "meta": {} + }, + { + "url": "/docs/using-fleet/rest-api", + "title": "REST API", + "lastModifiedAt": 1631555916000, + "htmlId": "docs--3-rest-api--0370e3eaff", + "meta": {} + }, + { + "url": "/docs/using-fleet/adding-hosts", + "title": "Adding hosts", + "lastModifiedAt": 1625588060000, + "htmlId": "docs--4-adding-hosts--f25bc11364", + "meta": {} + }, + { + "url": "/docs/using-fleet/osquery-logs", + "title": "Osquery logs", + "lastModifiedAt": 1624631015000, + "htmlId": "docs--5-osquery-logs--b2e649cc1f", + "meta": {} + }, + { + "url": "/docs/using-fleet/monitoring-fleet", + "title": "Monitoring Fleet", + "lastModifiedAt": 1630357234000, + "htmlId": "docs--6-monitoring-fleet--b1fa6e4a69", + "meta": {} + }, + { + "url": "/docs/using-fleet/security-best-practices", + "title": "Security best practices", + "lastModifiedAt": 1624893322000, + "htmlId": "docs--7-security-best-prac--ad931bb00b", + "meta": {} + }, + { + "url": "/docs/using-fleet/updating-fleet", + "title": "Updating Fleet", + "lastModifiedAt": 1630641746000, + "htmlId": "docs--8-updating-fleet--1887128e93", + "meta": {} + }, + { + "url": "/docs/using-fleet/permissions", + "title": "Permissions", + "lastModifiedAt": 1630415095000, + "htmlId": "docs--9-permissions--905e9c08da", + "meta": {} + }, + { + "url": "/docs/using-fleet/faq", + "title": "FAQ", + "lastModifiedAt": 1627511403000, + "htmlId": "docs--faq--f96c7228ae", + "meta": {} + }, + { + "url": "/docs/using-fleet", + "title": "Using Fleet", + "lastModifiedAt": 1626938622000, + "htmlId": "docs--readme--b097d08746", + "meta": {} + }, + { + "url": "/docs/deploying/installation", + "title": "Installation", + "lastModifiedAt": 1625173055000, + "htmlId": "docs--1-installation--fe7d4e2e74", + "meta": {} + }, + { + "url": "/docs/deploying/example-deployment-scenarios", + "title": "Example deployment scenarios", + "lastModifiedAt": 1625588060000, + "htmlId": "docs--3-example-deployment--b850738ae0", + "meta": {} + }, + { + "url": "/docs/deploying/configuration", + "title": "Configuration", + "lastModifiedAt": 1631134512000, + "htmlId": "docs--2-configuration--a242085fa7", + "meta": {} + }, + { + "url": "/docs/deploying/fleetctl-agent-updates", + "title": "Fleetctl agent updates", + "lastModifiedAt": 1631165652000, + "htmlId": "docs--4-fleetctl-agent-upd--f6d6a601d4", + "meta": {} + }, + { + "url": "/docs/deploying/faq", + "title": "FAQ", + "lastModifiedAt": 1627511403000, + "htmlId": "docs--faq--7abb678d36", + "meta": {} + }, + { + "url": "/docs/deploying", + "title": "Deploying", + "lastModifiedAt": 1624893322000, + "htmlId": "docs--readme--a0a26f55e2", + "meta": {} + }, + { + "url": "/docs/contributing/building-fleet", + "title": "Building Fleet", + "lastModifiedAt": 1629140343000, + "htmlId": "docs--1-building-fleet--a0d05ce171", + "meta": {} + }, + { + "url": "/docs/contributing/testing", + "title": "Testing", + "lastModifiedAt": 1630685123000, + "htmlId": "docs--2-testing--20bd58879c", + "meta": {} + }, + { + "url": "/docs/contributing/migrations", + "title": "Migrations", + "lastModifiedAt": 1629743194000, + "htmlId": "docs--3-migrations--ee672f0676", + "meta": {} + }, + { + "url": "/docs/contributing/committing-changes", + "title": "Committing changes", + "lastModifiedAt": 1629140343000, + "htmlId": "docs--4-committing-changes--62d8075df1", + "meta": {} + }, + { + "url": "/docs/contributing/releasing-fleet", + "title": "Releasing Fleet", + "lastModifiedAt": 1629929664000, + "htmlId": "docs--5-releasing-fleet--2b2a696ea0", + "meta": {} + }, + { + "url": "/docs/contributing/faq", + "title": "FAQ", + "lastModifiedAt": 1624893322000, + "htmlId": "docs--faq--92e0006bf2", + "meta": {} + }, + { + "url": "/docs/contributing", + "title": "Contributing", + "lastModifiedAt": 1624893322000, + "htmlId": "docs--readme--d5e4f68946", "meta": {} }, { "url": "/docs/using-fleet/configuration-files", "title": "Configuration files", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--readme--dc5df431cb", - "sectionRelativeRepoPath": "01-Using-Fleet/configuration-files/README.md", + "lastModifiedAt": 1631296113000, + "htmlId": "docs--readme--7908cef8a3", "meta": {} }, { "url": "/docs/using-fleet/standard-query-library", "title": "Standard query library", - "lastModifiedAt": 1632163704000, - "htmlId": "docs--readme--db16aa6f37", - "sectionRelativeRepoPath": "01-Using-Fleet/standard-query-library/README.md", + "lastModifiedAt": 1624049901000, + "htmlId": "docs--readme--d3c7d96146", "meta": {} } ], @@ -304,11 +264,11 @@ "remediation": "N/A" }, { - "name": "Get OpenSSL versions", + "name": "Detect Linux hosts with high severity vulnerable versions of OpenSSL", "platforms": "Linux", "description": "Retrieves the OpenSSL version.", "query": "SELECT name AS name, version AS version, 'deb_packages' AS source FROM deb_packages WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'apt_sources' AS source FROM apt_sources WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'rpm_packages' AS source FROM rpm_packages WHERE name LIKE 'openssl%';", - "purpose": "Informational", + "purpose": "Detection", "contributors": [ { "name": "Zach Wasserman", @@ -317,15 +277,15 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "get-open-ssl-versions", + "slug": "detect-linux-hosts-with-high-severity-vulnerable-versions-of-open-ssl", "remediation": "N/A" }, { - "name": "Get whether Gatekeeper is disabled", + "name": "Detect machines with Gatekeeper disabled", "platforms": "macOS", "description": "Gatekeeper tries to ensure only trusted software is run on a mac machine.", "query": "SELECT * FROM gatekeeper WHERE assessments_enabled = 0;", - "purpose": "Informational", + "purpose": "Detection", "contributors": [ { "name": "Zach Wasserman", @@ -334,16 +294,16 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "get-whether-gatekeeper-is-disabled", + "slug": "detect-machines-with-gatekeeper-disabled", "remediation": "N/A" }, { - "name": "Get authorized SSH keys", + "name": "Detect presence of authorized SSH keys", "platforms": "macOS, Linux", "description": "Presence of authorized SSH keys may be unusual on laptops. Could be completely normal on servers, but may be worth auditing for unusual keys and/or changes.", "query": "SELECT username, authorized_keys. * FROM users CROSS JOIN authorized_keys USING (uid);", - "purpose": "Informational", - "remediation": "N/A", + "purpose": "Detection", + "remediation": "Check out the linked table (https://github.com/fleetdm/fleet/blob/32b4d53e7f1428ce43b0f9fa52838cbe7b413eed/handbook/queries/detect-hosts-with-high-severity-vulnerable-versions-of-openssl.md#table-of-vulnerable-openssl-versions) to determine if the installed version is a high severity vulnerability and view the corresponding CVE(s)", "contributors": [ { "name": "Mike Thomas", @@ -352,7 +312,7 @@ "htmlUrl": "https://github.com/mike-j-thomas" } ], - "slug": "get-authorized-ssh-keys" + "slug": "detect-presence-of-authorized-ssh-keys" }, { "name": "Get authorized keys for Local Accounts", @@ -746,12 +706,12 @@ "remediation": "N/A" }, { - "name": "Get unencrypted SSH keys for local accounts", + "name": "Detect unencrypted SSH keys for local accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0;", - "purpose": "Informational", - "remediation": "N/A", + "purpose": "Detection", + "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", "contributors": [ { "name": "Ahmed Elshaer", @@ -760,15 +720,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-unencrypted-ssh-keys-for-local-accounts" + "slug": "detect-unencrypted-ssh-keys-for-local-accounts" }, { - "name": "Get unencrypted SSH keys for domain joined accounts", + "name": "Detect unencrypted SSH keys for domain joined accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0 and username in (SELECT distinct(username) FROM last);", - "purpose": "Informational", - "remediation": "N/A", + "purpose": "Detection", + "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", "contributors": [ { "name": "Ahmed Elshaer", @@ -777,7 +737,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-unencrypted-ssh-keys-for-domain-joined-accounts" + "slug": "detect-unencrypted-ssh-keys-for-domain-joined-accounts" }, { "name": "Get crontab jobs", @@ -814,12 +774,12 @@ "remediation": "N/A" }, { - "name": "Get dynamic linker hijacking on Linux (MITRE. T1574.006)", + "name": "Detect dynamic linker hijacking on Linux (MITRE. T1574.006)", "platforms": "Linux", "description": "Detect any processes that run with LD_PRELOAD environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='LD_PRELOAD';", - "purpose": "Informational", - "remediation": "N/A", + "purpose": "Detection", + "remediation": "Identify the process/binary detected and confirm with the system's owner.", "contributors": [ { "name": "Ahmed Elshaer", @@ -828,15 +788,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" + "slug": "detect-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" }, { - "name": "Get dynamic linker hijacking on macOS (MITRE. T1574.006)", + "name": "Detect dynamic linker hijacking on macOS (MITRE. T1574.006)", "platforms": "macOS", "description": "Detect any processes that run with DYLD_INSERT_LIBRARIES environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='DYLD_INSERT_LIBRARIES';", - "purpose": "Informational", - "remediation": "N/A", + "purpose": "Detection", + "remediation": "Identify the process/binary detected and confirm with the system's owner.", "contributors": [ { "name": "Ahmed Elshaer", @@ -845,7 +805,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" + "slug": "detect-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" }, { "name": "Get etc hosts entries", @@ -899,11 +859,11 @@ "remediation": "N/A" }, { - "name": "Get active user accounts on servers", + "name": "Detect active user accounts on servers", "platforms": "Linux", "description": "Domain Joined environment normally have root or other service account only and users are SSH-ing using their Domain Accounts.", "query": "SELECT * FROM shadow WHERE password_status='active' and username!='root';", - "purpose": "Informational", + "purpose": "Detection", "contributors": [ { "name": "Ahmed Elshaer", @@ -912,15 +872,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-active-user-accounts-on-servers", + "slug": "detect-active-user-accounts-on-servers", "remediation": "N/A" }, { - "name": "Get Nmap scanner", + "name": "Detect Nmap scanner", "platforms": "macOS, Linux, Windows, FreeBSD", - "description": "Get Nmap scanner process, as well as its user, parent, and process details.", + "description": "Detect Nmap scanner process, identify the user, parent, process details.", "query": "SELECT p.pid, name, p.path, cmdline, cwd, start_time, parent, (SELECT name FROM processes WHERE pid=p.parent) AS parent_name, (SELECT username FROM users WHERE uid=p.uid) AS username FROM processes as p WHERE cmdline like 'nmap%';", - "purpose": "Informational", + "purpose": "Detection", "contributors": [ { "name": "Ahmed Elshaer", @@ -929,7 +889,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "get-nmap-scanner", + "slug": "detect-nmap-scanner", "remediation": "N/A" }, { @@ -984,11 +944,11 @@ "remediation": "N/A" }, { - "name": "Get Windows print spooler remote code execution vulnerability", + "name": "Detect Windows print spooler remote code execution vulnerability", "platforms": "Windows", "description": "Detects devices that are potentially vulnerable to CVE-2021-1675 because the print spooler service is not disabled.", "query": "SELECT CASE cnt WHEN 2 THEN \"TRUE\" ELSE \"FALSE\" END \"Vulnerable\" FROM (SELECT name start_type, COUNT(name) AS cnt FROM services WHERE name = 'NTDS' or (name = 'Spooler' and start_type <> 'DISABLED')) WHERE cnt = 2;", - "purpose": "Informational", + "purpose": "Detection", "contributors": [ { "name": null, @@ -997,7 +957,7 @@ "htmlUrl": "https://github.com/maravedi" } ], - "slug": "get-windows-print-spooler-remote-code-execution-vulnerability", + "slug": "detect-windows-print-spooler-remote-code-execution-vulnerability", "remediation": "N/A" }, { @@ -1018,7 +978,7 @@ "remediation": "N/A" }, { - "name": "Get processes that no longer exist on disk", + "name": "Find deleted files from disk", "platforms": "Linux, macOS, Windows", "description": "Lists all processes of which the binary which launched them no longer exists on disk. Attackers often delete files from disk after launching process to mask presence.", "query": "SELECT name, path, pid FROM processes WHERE on_disk = 0;", @@ -1031,96 +991,11 @@ "htmlUrl": "https://github.com/alphabrevity" } ], - "slug": "get-processes-that-no-longer-exist-on-disk", - "remediation": "N/A" - }, - { - "name": "Get user files matching a specific hash", - "platforms": "macOS, Linux", - "description": "Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.)", - "query": "SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE \"\"/Users/%/Documents/%%\"\") AND sha256 = \"\"16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75\"\";", - "purpose": "Informational", - "contributors": [ - { - "name": "AndrewB", - "handle": "alphabrevity", - "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", - "htmlUrl": "https://github.com/alphabrevity" - } - ], - "slug": "get-user-files-matching-a-specific-hash", - "remediation": "N/A" - }, - { - "name": "Get local administrator accounts on macOS", - "platforms": "macOS", - "description": "The query allows you to check macOS systems for local administrator accounts.", - "query": "SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid;", - "purpose": "Informational", - "contributors": [ - { - "name": "AndrewB", - "handle": "alphabrevity", - "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", - "htmlUrl": "https://github.com/alphabrevity" - } - ], - "slug": "get-local-administrator-accounts-on-mac-os", - "remediation": "N/A" - }, - { - "name": "Get all listening ports, by process", - "platforms": "Linux, macOS, Windows", - "description": "List ports that are listening on all interfaces, along with the process to which they are attached.", - "query": "SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = \"0.0.0.0\";", - "purpose": "Informational", - "contributors": [ - { - "name": "AndrewB", - "handle": "alphabrevity", - "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", - "htmlUrl": "https://github.com/alphabrevity" - } - ], - "slug": "get-all-listening-ports-by-process", - "remediation": "N/A" - }, - { - "name": "Get whether TeamViewer is installed/running", - "platforms": "Windows", - "description": "Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine.", - "query": "SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE \"%teamviewer%\";", - "purpose": "Informational", - "contributors": [ - { - "name": "AndrewB", - "handle": "alphabrevity", - "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", - "htmlUrl": "https://github.com/alphabrevity" - } - ], - "slug": "get-whether-team-viewer-is-installed-running", - "remediation": "N/A" - }, - { - "name": "Get malicious Python backdoors", - "platforms": "macOS, Linux, Windows", - "description": "Watches for the backdoored Python packages installed on system. See (http://www.nbu.gov.sk/skcsirt-sa-20170909-pypi/index.html)", - "query": "select case cnt when 0 then \"NONE_INSTALLED\" else \"INSTALLED\" end as \"Malicious Python Packages\",package_name,package_version from (select count(name) as cnt,nameas package_name,version as package_version,path as package_pathfrom python_packages where package_name in ('acqusition','apidev-coop','bzip','crypt','django-server','pwd','setup-tools','telnet','urlib3','urllib'));", - "purpose": "Informational", - "contributors": [ - { - "name": "AndrewB", - "handle": "alphabrevity", - "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", - "htmlUrl": "https://github.com/alphabrevity" - } - ], - "slug": "get-malicious-python-backdoors", + "slug": "find-deleted-files-from-disk", "remediation": "N/A" } ], - "queryLibraryYmlRepoPath": "docs/01-Using-Fleet/standard-query-library/standard-query-library.yml", + "queryLibraryYmlRepoPath": "docs/1-Using-Fleet/standard-query-library/standard-query-library.yml", "compiledPagePartialsAppPath": "views/partials/built-from-markdown" } } diff --git a/website/api/controllers/docs/view-basic-documentation.js b/website/api/controllers/docs/view-basic-documentation.js index 463d134991..769474f0ad 100644 --- a/website/api/controllers/docs/view-basic-documentation.js +++ b/website/api/controllers/docs/view-basic-documentation.js @@ -44,26 +44,14 @@ module.exports = { // console.log('pageUrlSuffix:',pageUrlSuffix); // console.log('SECTION_URL_PREFIX + "/" + _.trim(pageUrlSuffix, "/"):',SECTION_URL_PREFIX + '/' + _.trim(pageUrlSuffix, '/')); // console.log('thisPage:',thisPage); + if (!thisPage) { + throw 'notFound'; + } - // Setting a flag if the pageUrlSuffix doesn't match any existing page, or if the page it matches doesn't exactly match the pageUrlSuffix provided - // Note: because this also handles the docs landing page and a pageUrlSuffix might not have provided, we set this flag to false if the url is just '/docs' - let needsRedirectMaybe = (!thisPage || (thisPage.url !== '/docs/'+pageUrlSuffix && thisPage.url !== '/docs')); - - if (needsRedirectMaybe) { - // Creating a lower case, repeating-slashless pageUrlSuffix - let multipleSlashesRegex = /\/{2,}/g; - let modifiedPageUrlSuffix = pageUrlSuffix.toLowerCase().replace(multipleSlashesRegex, '/'); - // Finding the appropriate page content using the modified pageUrlSuffix. - let revisedPage = _.find(sails.config.builtStaticContent.markdownPages, { - url: _.trimRight(SECTION_URL_PREFIX + '/' + _.trim(modifiedPageUrlSuffix, '/'), '/') - }); - if(revisedPage) { - // If we matched a page with the modified pageUrlSuffix, then redirect to that. - throw {redirect: revisedPage.url}; - } else { - // If no page was found, throw a 404 error. - throw 'notFound'; - } + if (false) { + // TODO: add "redirect" exit and handle mismatched capitalization / extra slashes by redirecting to the correct URL. e.g. "http://localhost:2024/docs//usiNG-fleet///" Partial example of this: https://github.com/sailshq/sailsjs.com/blob/b53c6e6a90c9afdf89e5cae00b9c9dd3f391b0e7/api/controllers/documentation/view-documentation.js#L161-L166 + let revisedUrl = 'todo'; + throw {redirect: revisedUrl}; } // Respond with view. diff --git a/website/api/controllers/webhooks/receive-from-github.js b/website/api/controllers/webhooks/receive-from-github.js index 20e5abd03e..190b899f88 100644 --- a/website/api/controllers/webhooks/receive-from-github.js +++ b/website/api/controllers/webhooks/receive-from-github.js @@ -25,24 +25,8 @@ module.exports = { let GitHub = require('machinepack-github'); let GREEN_LABEL_COLOR = 'C2E0C6';// « Used in multiple places below. - let GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS = [// « Used in multiple places below. - 'sailsbot', - 'fleet-release', - 'noahtalerman', - 'mike-j-thomas', - 'mikermcneil', - 'lukeheath', - 'zwass', - 'rlynnj11', - 'martavis', - 'rachelelysia', - 'gillespi314', - 'chiiph', - 'mna', - 'edwardsb', - 'alphabrevity', - 'eashaw', - 'drewbakerfdm' + let GITHUB_USERNAMES_OF_BOTS = [// « Used in multiple places below. + 'sailsbot' ]; let GITHUB_USERNAME_OF_DRI_FOR_LABELS = 'noahtalerman';// « Used below @@ -110,7 +94,7 @@ module.exports = { // `For help with questions about Sails, [click here](http://sailsjs.com/support).\n`; // } // } else { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); // if (wasReopenedByBot) { // newBotComment = '';// « checked below // } else { @@ -162,7 +146,7 @@ module.exports = { // if (action === 'edited' && pr.state !== 'open') { // // If this is an edit to an already-closed pull request, then do nothing. // } else if (action === 'reopened') { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); // if (!wasReopenedByBot) { // let newBotComment = // `Oh hey again, @${issueOrPr.user.login}. Now that this pull request is reopened, it's on our radar. Please let us know if there's any new information we should be aware of!\n`+ @@ -225,7 +209,7 @@ module.exports = { let repo = repository.name; let issueNumber = issueOrPr.number; - let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); + let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); if (!wasPostedByBot) { let greenLabels = _.filter(issueOrPr.labels, ({color}) => color === GREEN_LABEL_COLOR); await sails.helpers.flow.simultaneouslyForEach(greenLabels, async(greenLabel)=>{ @@ -233,10 +217,10 @@ module.exports = { });//∞ß }//fi } else if ( - (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(comment.user.login))|| + (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS.includes(comment.user.login))|| (ghNoun === 'commit_comment' && ['created'].includes(action))|| (ghNoun === 'label' && ['created','edited','deleted'].includes(action) && GITHUB_USERNAME_OF_DRI_FOR_LABELS !== sender.login)||//« exempt label changes made by the directly responsible individual for labels, because otherwise when process changes/fiddlings happen, they can otherwise end up making too much noise in Slack - (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 7*24*60*60*1000 ) ) + (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 24*60*60*1000 ) ) ) { // ██╗███╗ ██╗███████╗ ██████╗ ██████╗ ███╗ ███╗ ██╗ ██╗███████╗ // ██║████╗ ██║██╔════╝██╔═══██╗██╔══██╗████╗ ████║ ██║ ██║██╔════╝ @@ -270,7 +254,7 @@ module.exports = { : (ghNoun === 'label') ? `@${sender.login} just ${action} a GitHub label "*${label.name}*" (#${label.color}) in ${repository.owner.login}/${repository.name}.\n\n> To manage labels in ${repository.owner.login}/${repository.name}, visit https://github.com/${encodeURIComponent(repository.owner.login)}/${encodeURIComponent(repository.name)}/labels` : - `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >7 days (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` + `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >24 hours (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` )+`\n` }, {'Content-Type':'application/json'} diff --git a/website/api/controllers/webhooks/receive-usage-analytics.js b/website/api/controllers/webhooks/receive-usage-analytics.js index 7895e1ca3e..5b3fc6bcb9 100644 --- a/website/api/controllers/webhooks/receive-usage-analytics.js +++ b/website/api/controllers/webhooks/receive-usage-analytics.js @@ -4,7 +4,7 @@ module.exports = { friendlyName: 'Receive usage analytics', - description: 'Receive anonymous usage analytics from deployments of Fleet running in production. (Not fleetctl preview or dev-mode deployments.)', + description: '', inputs: { diff --git a/website/api/helpers/strings/to-html.js b/website/api/helpers/strings/to-html.js index 99f0dc8f96..09bfabc6df 100644 --- a/website/api/helpers/strings/to-html.js +++ b/website/api/helpers/strings/to-html.js @@ -73,14 +73,8 @@ module.exports = { smartLists: true, smartypants: false, }; - if (inputs.addIdsToHeadings === true) { - var headingRenderer = new marked.Renderer(); - headingRenderer.heading = function (text, level) { - var headingID = _.kebabCase(text); - return ''+text+'\n'; - }; - markedOpts.renderer = headingRenderer; - } else { + + if (inputs.addIdsToHeadings === false) { var renderer = new marked.Renderer(); renderer.heading = function (text, level) { return ''+text+''; diff --git a/website/assets/images/icon-link-16x16@2x.png b/website/assets/images/icon-link-16x16@2x.png deleted file mode 100644 index 41120a4844..0000000000 Binary files a/website/assets/images/icon-link-16x16@2x.png and /dev/null differ diff --git a/website/assets/js/pages/docs/basic-documentation.page.js b/website/assets/js/pages/docs/basic-documentation.page.js index c5b6e046fc..e7fcdfe35d 100644 --- a/website/assets/js/pages/docs/basic-documentation.page.js +++ b/website/assets/js/pages/docs/basic-documentation.page.js @@ -104,13 +104,11 @@ parasails.registerPage('basic-documentation', { // console.log(subtopics); this.subtopics = (() => { - let subtopics = $('#body-content').find('h2').map((_, el) => el.innerText); + let subtopics = $('#body-content').find('h2').map((_, el) => el.innerHTML); subtopics = $.makeArray(subtopics).map((title) => { - // Removing all apostrophes from the title to keep _.kebabCase() from turning words like 'user’s' into 'user-s' - let kebabCaseFriendlyTitle = title.replace(/[\’]/g, ''); return { title, - url: '#' + _.kebabCase(kebabCaseFriendlyTitle), + url: '#' + _.kebabCase(title), }; }); return subtopics; diff --git a/website/assets/styles/pages/docs/basic-documentation.less b/website/assets/styles/pages/docs/basic-documentation.less index 4d9956c5dd..6f83ce2d41 100644 --- a/website/assets/styles/pages/docs/basic-documentation.less +++ b/website/assets/styles/pages/docs/basic-documentation.less @@ -38,17 +38,6 @@ } } - .markdown-heading:hover { - - .markdown-link { - height: 16px; - vertical-align: middle; - margin-left: 8px; - content: url('/images/icon-link-16x16@2x.png'); - } - - } - [purpose='search'] { @@ -198,56 +187,6 @@ color: @core-fleet-black; } - } - .edit-button-container { - position: relative; - - .edit-button { - color: @core-vibrant-blue; - opacity: 0; - font-size: 12px; - position: absolute; - right: 4px; - top: 8px; - cursor: pointer; - border: 1px solid @core-vibrant-blue; - border-radius: 4px; - padding: 4px 8px; - text-decoration: none; - - .edit-link { - color: @core-vibrant-blue; - - } - .edit-pencil { - height: 16px; - padding-right: 5px; - } - - } - - .edit-button:hover { - - background: @core-vibrant-blue; - - a { - text-decoration: none; - } - .edit-link { - color: @accent-white; - - } - } - - } - - &:hover { - - .edit-button { - opacity: 1; - - } - } [purpose='search'] { diff --git a/website/assets/styles/pages/docs/code-blocks.less b/website/assets/styles/pages/docs/code-blocks.less index d7caa9932c..4a01c5a023 100644 --- a/website/assets/styles/pages/docs/code-blocks.less +++ b/website/assets/styles/pages/docs/code-blocks.less @@ -1,6 +1,6 @@ // lesshint-disable spaceAroundComma, trailingWhitespace - code:not(.hljs):not(.nohighlight) { + code:not(.nohighlight, .hljs) { background-color: @ui-off-white; border: 1px solid @border-lt-gray; color: @core-fleet-black-75; @@ -16,7 +16,7 @@ } - pre:not(.algolia-autocomplete):not(.ds-dropdown-menu):not(.json) { + pre:not(.algolia-autocomplete, .ds-dropdown-menu) { padding: 24px; border: 1px solid @border-lt-gray; border-radius: 6px; diff --git a/website/config/routes.js b/website/config/routes.js index c107ee8700..6a1b6ffdeb 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -13,77 +13,19 @@ module.exports.routes = { // ╦ ╦╔═╗╔╗ ╔═╗╔═╗╔═╗╔═╗╔═╗ // ║║║║╣ ╠╩╗╠═╝╠═╣║ ╦║╣ ╚═╗ // ╚╩╝╚═╝╚═╝╩ ╩ ╩╚═╝╚═╝╚═╝ - 'GET /': { - action: 'view-homepage-or-redirect', - locals: { isHomepage: true } - }, + 'GET /': { action: 'view-homepage-or-redirect', locals: { isHomepage: true } }, + 'GET /company/contact': { action: 'view-contact' }, + 'GET /get-started': { action: 'view-get-started' }, + 'GET /pricing': { action: 'view-pricing' }, + 'GET /press-kit': { action: 'view-press-kit' }, - 'GET /company/contact': { - action: 'view-contact', - locals:{ - title: 'Contact us | Fleet for osquery', - description: 'Get in touch with our team.' - } - }, + 'GET /queries': { action: 'view-query-library' }, + 'GET /queries/:slug': { action: 'view-query-detail' }, - 'GET /get-started': { - action: 'view-get-started' , - locals:{ - title: 'Get Started | Fleet for osquery', - description: 'Learn about getting started with Fleet.' - } - }, + 'GET /docs/?*': { skipAssets: false, action: 'docs/view-basic-documentation' },// handles /docs and /docs/foo/bar + // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar - 'GET /pricing': { - action: 'view-pricing', - locals:{ - title: 'Pricing | Fleet for osquery', - description: 'View Fleet plans and pricing details.' - } - }, - - 'GET /press-kit': { - action: 'view-press-kit', - locals:{ - title: 'Press kit | Fleet for osquery', - description: 'Download Fleet logos, wallpapers, and screenshots.' - } - }, - - 'GET /queries': { - action: 'view-query-library', - locals:{ - title: 'Queries | Fleet for osquery', - description: 'A growing collection of useful queries for organizations deploying Fleet and osquery.' - } - }, - - 'GET /queries/:slug': { - action: 'view-query-detail', - locals:{ - title: 'Query details | Fleet for osquery', - description: 'View more information about a query in Fleet\'s standard query library', - } - }, - - 'GET /docs/?*': { - skipAssets: false, - action: 'docs/view-basic-documentation', - locals:{ - title: 'Documentation | Fleet for osquery', - description: 'Documentation for Fleet for osquery.', - } - },// handles /docs and /docs/foo/bar - - // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar - - 'GET /transparency': { - action: 'view-transparency', - locals:{ - title: 'Transparency | Fleet for osquery', - description: 'Learn what data osquery can see.', - } - }, + 'GET /transparency': { action: 'view-transparency' }, diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index f52a6044bf..2d9b24a15e 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -25,8 +25,8 @@ module.exports = { await sails.helpers.flow.simultaneously([ async()=>{// Parse query library from YAML and prepare to bake them into the Sails app's configuration. - let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml'; - let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)).intercept('doesNotExist', (err)=>new Error(`Could not find standard query library YAML file at "${RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO}". Was it accidentally moved? Raw error: `+err.message)); + let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/1-Using-Fleet/standard-query-library/standard-query-library.yml'; + let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)); let queriesWithProblematicRemediations = []; let queriesWithProblematicContributors = []; @@ -226,19 +226,13 @@ module.exports = { // to some page on the destination site where this will be hosted, like `(*.)?fleetdm.com`. // If external, add target="_blank" so the link will open in a new tab. let isExternal = ! hrefString.match(/^href=\"https?:\/\/([^\.]+\.)*fleetdm\.com/g);// « FUTURE: make this smarter with sails.config.baseUrl + _.escapeRegExp() - // Check if this link is to fleetdm.com or www.fleetdm.com. - let isBaseUrl = hrefString.match(/^(href="https?:\/\/)([^\.]+\.)*fleetdm\.com"$/g); if (isExternal) { return hrefString.replace(/(href="https?:\/\/([^"]+)")/g, '$1 target="_blank"'); } else { // Otherwise, change the link to be web root relative. // (e.g. 'href="http://sailsjs.com/documentation/concepts"'' becomes simply 'href="/documentation/concepts"'') // > Note: See the Git version history of "compile-markdown-content.js" in the sailsjs.com website repo for examples of ways this can work across versioned subdomains. - if (isBaseUrl) { - return hrefString.replace(/href="https?:\/\//, '').replace(/([^\.]+\.)*fleetdm\.com/, 'href="/'); - } else { - return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); - } + return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); } });//∞ @@ -299,17 +293,12 @@ module.exports = { await sails.helpers.fs.write(htmlOutputPath, htmlString); } - // Determine the path of the file in the fleet repo so we can link to - // the file on github from fleetdm.com (e.g. 1-Using-Fleet/2-fleetctl-CLI.md) - let sectionRelativeRepoPath = path.relative(path.join(topLvlRepoPath, sectionRepoPath), path.resolve(pageSourcePath)); - // Append to what will become configuration for the Sails app. builtStaticContent.markdownPages.push({ url: rootRelativeUrlPath, title: pageTitle, lastModifiedAt: lastModifiedAt, htmlId: htmlId, - sectionRelativeRepoPath: sectionRelativeRepoPath, meta: _.omit(embeddedMetadata, 'title') }); } diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index 4c3854afb4..2f956ccbbd 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -7,15 +7,14 @@ %> - <%= typeof title !== 'undefined' ? title : 'Fleet for osquery | Open source device management' %> - + Fleet for osquery | Open source device management <% /* Viewport tag for sensible mobile support */ %> - - + + <% /* Script tags should normally be included further down the page- but any scripts that load fonts (e.g. Fontawesome ≥v5) are special exceptions to the @@ -135,7 +134,7 @@ Blog Jobs Contribute - Logos/artwork + Press Kit Hall of fame
diff --git a/website/views/pages/docs/basic-documentation.ejs b/website/views/pages/docs/basic-documentation.ejs index 7cc744d5d3..c0b7434fa7 100644 --- a/website/views/pages/docs/basic-documentation.ejs +++ b/website/views/pages/docs/basic-documentation.ejs @@ -189,13 +189,6 @@
- -
-
- Edit page -
-
- <%- partial( path.relative( path.dirname(__filename), @@ -212,7 +205,7 @@

Is there something missing?

- If you notice something we’ve missed, or that could be improved, please click here to edit this page. + If you notice something we've missed or could be improved on, please follow this link and submit a pull request to the Fleet repo.

diff --git a/website/views/pages/homepage.ejs b/website/views/pages/homepage.ejs index 18792d2001..b877c20cc6 100644 --- a/website/views/pages/homepage.ejs +++ b/website/views/pages/homepage.ejs @@ -12,8 +12,8 @@
Quickly deploy osquery and scale your fleet to 100,000+ devices on top of a stable core technology.
-
- Try it out +
+ Get started with Fleet Watch video @@ -36,7 +36,7 @@

- Single source or truth + Inventory management

Track and segment your enrolled hosts. Search by important details, and zoom in on individual targets. @@ -67,7 +67,7 @@

- Deploy anywhere + Deployment

Fleet is self-hosted and self-managed, and can be run within your own data centers or in the cloud. @@ -136,7 +136,7 @@ class="btn btn-block btn-md btn-primary mx-4" href="/get-started" > - Try it out + Get started - Try it out + Get started

Contributors

Want to add your own query? Please submit a pull request - + over on GitHub .

diff --git a/yarn.lock b/yarn.lock index 4afebf1bc4..ae51c26324 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1122,10 +1122,10 @@ date-fns "^1.27.2" figures "^1.7.0" -"@cypress/request@^2.88.5", "@cypress/request@^2.88.6": - version "2.88.6" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.6.tgz#a970dd675befc6bdf8a8921576c01f51cc5798e9" - integrity sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ== +"@cypress/request@^2.88.5": + version "2.88.5" + resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.5.tgz#8d7ecd17b53a849cfd5ab06d5abe7d84976375d7" + integrity sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA== dependencies: aws-sign2 "~0.7.0" aws4 "^1.8.0" @@ -1140,12 +1140,13 @@ isstream "~0.1.2" json-stringify-safe "~5.0.1" mime-types "~2.1.19" + oauth-sign "~0.9.0" performance-now "^2.1.0" qs "~6.5.2" safe-buffer "^5.1.2" tough-cookie "~2.5.0" tunnel-agent "^0.6.0" - uuid "^8.3.2" + uuid "^3.3.2" "@cypress/xvfb@^1.2.4": version "1.2.4" @@ -1691,11 +1692,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.50.tgz#e9b2e85fafc15f2a8aa8fdd41091b983da5fd6ee" integrity sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w== -"@types/node@^14.14.31": - version "14.17.16" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.16.tgz#2b9252bd4fdf0393696190cd9550901dd967c777" - integrity sha512-WiFf2izl01P1CpeY8WqFAeKWwByMueBEkND38EcN8N68qb0aDG3oIS1P5MhAX5kUdr469qRyqsY/MjanLjsFbQ== - "@types/node@^14.14.37": version "14.14.37" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.37.tgz#a3dd8da4eb84a996c36e331df98d82abd76b516e" @@ -1788,15 +1784,15 @@ "@types/prop-types" "*" csstype "^3.0.2" -"@types/sinonjs__fake-timers@^6.0.1", "@types/sinonjs__fake-timers@^6.0.2": - version "6.0.3" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08" - integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g== +"@types/sinonjs__fake-timers@^6.0.1": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz#3a84cf5ec3249439015e14049bd3161419bf9eae" + integrity sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg== "@types/sizzle@^2.3.2": - version "2.3.3" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" - integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== + version "2.3.2" + resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.2.tgz#a811b8c18e2babab7d542b3365887ae2e4d9de47" + integrity sha512-7EJYyKTL7tFR8+gDbB6Wwz/arpGa0Mywk1TJbNzKzHtzbwVmY4HR9WqS5VV7dsBUKQmPNr192jHr/VpBluj/hg== "@types/sockjs-client@^1.5.1": version "1.5.1" @@ -1832,13 +1828,6 @@ dependencies: "@types/yargs-parser" "*" -"@types/yauzl@^2.9.1": - version "2.9.2" - resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" - integrity sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA== - dependencies: - "@types/node" "*" - "@typescript-eslint/eslint-plugin@^4.15.2": version "4.15.2" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.15.2.tgz#981b26b4076c62a5a55873fbef3fe98f83360c61" @@ -2121,14 +2110,6 @@ acorn@^8.1.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.1.1.tgz#fb0026885b9ac9f48bac1e185e4af472971149ff" integrity sha512-xYiIVjNuqtKXMxlRMDc6mZUhXehod4a3gbZ1qRlM7icK4EbxUFNLhWoPblCvFtB2Y9CIqHP3CF/rdxLItaQv8g== -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - airbnb-prop-types@^2.16.0: version "2.16.0" resolved "https://registry.yarnpkg.com/airbnb-prop-types/-/airbnb-prop-types-2.16.0.tgz#b96274cefa1abb14f623f804173ee97c13971dc2" @@ -2221,13 +2202,6 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.11.0" -ansi-escapes@^4.3.0: - version "4.3.2" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" - integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== - dependencies: - type-fest "^0.21.3" - ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" @@ -2249,9 +2223,9 @@ ansi-regex@^4.1.0: integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== ansi-regex@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + version "5.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" + integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== ansi-styles@^2.2.1: version "2.2.1" @@ -2303,7 +2277,7 @@ aproba@^1.0.3, aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -arch@^2.1.2, arch@^2.2.0: +arch@^2.1.2: version "2.2.0" resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== @@ -2513,9 +2487,9 @@ async@^2.5.0: lodash "^4.17.11" async@^3.2.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8" - integrity sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg== + version "3.2.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" + integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== asynckit@^0.4.0: version "0.4.0" @@ -2563,9 +2537,9 @@ aws-sign2@~0.7.0: integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= aws4@^1.8.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" - integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + version "1.8.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" + integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== axios@^0.21.1: version "0.21.1" @@ -3417,9 +3391,9 @@ balanced-match@^0.4.2: integrity sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg= balanced-match@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" - integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= base64-js@^1.0.2: version "1.3.0" @@ -3471,7 +3445,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -blob-util@2.0.2, blob-util@^2.0.2: +blob-util@2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== @@ -3693,9 +3667,9 @@ buffer-crc32@~0.2.3: integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= buffer-from@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" - integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== + version "1.1.1" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== buffer-xor@^1.0.3: version "1.0.3" @@ -3900,9 +3874,9 @@ chalk@^3.0.0: supports-color "^7.1.0" chalk@^4.0.0, chalk@^4.1.0: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + version "4.1.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" + integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" @@ -3991,11 +3965,6 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== -ci-info@^3.1.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.2.0.tgz#2876cb948a498797b5236f0095bc057d0dca38b6" - integrity sha512-dVqRX7fLUm8J6FgHJ418XuIgDLZDkYcDFTeL6TA2gt5WlIZUQrrH6EZrNClwT/H0FateUsZkGIOPRrLbP+PR9A== - cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -4043,11 +4012,6 @@ clean-css@4.2.x: dependencies: source-map "~0.6.0" -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - cli-cursor@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" @@ -4062,13 +4026,6 @@ cli-cursor@^2.0.0, cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - cli-table3@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.0.tgz#b7b1bc65ca8e7b5cef9124e13dc2b21e2ce4faee" @@ -4087,14 +4044,6 @@ cli-truncate@^0.2.1: slice-ansi "0.0.4" string-width "^1.0.1" -cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" - cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -4217,11 +4166,6 @@ colorette@^1.2.1: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== -colorette@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" - integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== - colormin@^1.0.5: version "1.1.2" resolved "https://registry.yarnpkg.com/colormin/-/colormin-1.1.2.tgz#ea2f7420a72b96881a38aae59ec124a6f7298133" @@ -4232,9 +4176,9 @@ colormin@^1.0.5: has "^1.0.1" colors@^1.1.2: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== + version "1.3.3" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d" + integrity sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg== colors@~1.1.2: version "1.1.2" @@ -4409,16 +4353,11 @@ core-js@^2.4.0, core-js@^2.5.0: resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.9.tgz#6b4b214620c834152e179323727fc19741b084f2" integrity sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A== -core-util-is@1.0.2: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= -core-util-is@~1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" - integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== - cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a" @@ -4737,48 +4676,47 @@ cypress@*: url "^0.11.0" yauzl "^2.10.0" -cypress@8.4.0: - version "8.4.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-8.4.0.tgz#09ec06a73f1cb10121c103cba15076e659e24876" - integrity sha512-RtVgGFR06ikyMaq/VqapeqOjGaIA42PpK7F0qe1MCiFArfUuJECsLmeYaOA+1TlmNUgJNMSF5fWKkZIJr5Uc7w== +cypress@^6.9.1: + version "6.9.1" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.9.1.tgz#ce1106bfdc47f8d76381dba63f943447883f864c" + integrity sha512-/RVx6sOhsyTR9sd9v0BHI4tnDZAhsH9rNat7CIKCUEr5VPWxyfGH0EzK4IHhAqAH8vjFcD4U14tPiJXshoUrmQ== dependencies: - "@cypress/request" "^2.88.6" + "@cypress/listr-verbose-renderer" "^0.4.1" + "@cypress/request" "^2.88.5" "@cypress/xvfb" "^1.2.4" - "@types/node" "^14.14.31" - "@types/sinonjs__fake-timers" "^6.0.2" + "@types/node" "12.12.50" + "@types/sinonjs__fake-timers" "^6.0.1" "@types/sizzle" "^2.3.2" - arch "^2.2.0" - blob-util "^2.0.2" + arch "^2.1.2" + blob-util "2.0.2" bluebird "^3.7.2" cachedir "^2.3.0" chalk "^4.1.0" check-more-types "^2.24.0" - cli-cursor "^3.1.0" cli-table3 "~0.6.0" commander "^5.1.0" common-tags "^1.8.0" - dayjs "^1.10.4" - debug "^4.3.2" - enquirer "^2.3.6" - eventemitter2 "^6.4.3" - execa "4.1.0" + dayjs "^1.9.3" + debug "4.3.2" + eventemitter2 "^6.4.2" + execa "^4.0.2" executable "^4.1.1" - extract-zip "2.0.1" - figures "^3.2.0" - fs-extra "^9.1.0" + extract-zip "^1.7.0" + fs-extra "^9.0.1" getos "^3.2.1" - is-ci "^3.0.0" - is-installed-globally "~0.4.0" + is-ci "^2.0.0" + is-installed-globally "^0.3.2" lazy-ass "^1.6.0" - listr2 "^3.8.3" - lodash "^4.17.21" + listr "^0.14.3" + lodash "^4.17.19" log-symbols "^4.0.0" minimist "^1.2.5" + moment "^2.29.1" ospath "^1.2.2" - pretty-bytes "^5.6.0" + pretty-bytes "^5.4.1" ramda "~0.27.1" request-progress "^3.0.0" - supports-color "^8.1.1" + supports-color "^7.2.0" tmp "~0.2.1" untildify "^4.0.0" url "^0.11.0" @@ -4815,10 +4753,10 @@ date-now@^0.1.4: resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" integrity sha1-6vQ5/U1ISK105cx9vvIAZyueNFs= -dayjs@^1.10.4, dayjs@^1.9.3: - version "1.10.7" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.7.tgz#2cf5f91add28116748440866a0a1d26f3a6ce468" - integrity sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig== +dayjs@^1.9.3: + version "1.10.4" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.4.tgz#8e544a9b8683f61783f570980a8a80eaf54ab1e2" + integrity sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw== debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: version "2.6.9" @@ -4827,7 +4765,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4.3.2, debug@^4.3.2: +debug@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -5238,20 +5176,13 @@ encoding@^0.1.11: dependencies: iconv-lite "^0.6.2" -end-of-stream@^1.0.0: +end-of-stream@^1.0.0, end-of-stream@^1.1.0: version "1.4.1" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" integrity sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q== dependencies: once "^1.4.0" -end-of-stream@^1.1.0: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - enhanced-resolve@4.1.0, enhanced-resolve@^4.0.0, enhanced-resolve@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" @@ -5270,7 +5201,7 @@ enhanced-resolve@~0.9.0: memory-fs "^0.2.0" tapable "^0.1.8" -enquirer@^2.3.5, enquirer@^2.3.6: +enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -5786,7 +5717,7 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -eventemitter2@^6.4.2, eventemitter2@^6.4.3: +eventemitter2@^6.4.2: version "6.4.4" resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== @@ -5816,7 +5747,20 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" integrity sha512-sEFIkc61v75sWeOe72qyrqg2Qg0OuLESziUDk/O/z2qgS15y2gWVFrI6f2Qn/qw/0/NCfCEsmNA4zOjkwEZT1A== -execa@4.1.0, execa@^4.0.0, execa@^4.0.2: +execa@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" + integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== + dependencies: + cross-spawn "^6.0.0" + get-stream "^4.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + +execa@^4.0.0, execa@^4.0.2: version "4.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== @@ -5831,19 +5775,6 @@ execa@4.1.0, execa@^4.0.0, execa@^4.0.2: signal-exit "^3.0.2" strip-final-newline "^2.0.0" -execa@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" - integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== - dependencies: - cross-spawn "^6.0.0" - get-stream "^4.0.0" - is-stream "^1.1.0" - npm-run-path "^2.0.0" - p-finally "^1.0.0" - signal-exit "^3.0.0" - strip-eof "^1.0.0" - executable@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" @@ -6002,17 +5933,6 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" -extract-zip@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - extract-zip@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -6061,9 +5981,9 @@ fast-glob@^3.1.1: picomatch "^2.2.1" fast-json-stable-stringify@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" - integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + version "2.0.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" + integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I= fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.4: version "2.0.6" @@ -6136,13 +6056,6 @@ figures@^2.0.0: dependencies: escape-string-regexp "^1.0.5" -figures@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - file-entry-cache@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -6374,7 +6287,7 @@ from2@^2.1.0: inherits "^2.0.1" readable-stream "^2.0.0" -fs-extra@^9.0.1, fs-extra@^9.1.0: +fs-extra@^9.0.1: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== @@ -6526,7 +6439,7 @@ get-stream@^4.0.0: dependencies: pump "^3.0.0" -get-stream@^5.0.0, get-stream@^5.1.0: +get-stream@^5.0.0: version "5.2.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== @@ -6593,7 +6506,7 @@ glob@^5.0.13: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.4, glob@~7.1.1: +glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@~7.1.1: version "7.1.4" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255" integrity sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A== @@ -6605,18 +6518,6 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.4, glob@~7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.3: - version "7.1.7" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" - integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - global-dirs@^2.0.1: version "2.1.0" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.1.0.tgz#e9046a49c806ff04d6c1825e196c8f0091e8df4d" @@ -6624,13 +6525,6 @@ global-dirs@^2.0.1: dependencies: ini "1.3.7" -global-dirs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" - integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== - dependencies: - ini "2.0.0" - global-modules@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -6705,15 +6599,15 @@ globule@^1.0.0: lodash "~4.17.10" minimatch "~3.0.2" -graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2: +graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6: version "4.2.0" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b" integrity sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg== -graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.8" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" - integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== +graceful-fs@^4.2.0: + version "4.2.6" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" + integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== graceful-fs@^4.2.4: version "4.2.4" @@ -7256,11 +7150,6 @@ ini@1.3.7: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.7.tgz#a09363e1911972ea16d7a8851005d84cf09a9a84" integrity sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" @@ -7396,13 +7285,6 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-ci@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.0.tgz#c7e7be3c9d8eef7d0fa144390bd1e4b88dc4c994" - integrity sha512-kDXyttuLeslKAHYL/K28F2YkM3x5jvFPEw3yXbRptXydjD9rpLEz+C5K5iutY9ZiUu6AP41JdvRQwF4Iqs4ZCQ== - dependencies: - ci-info "^3.1.1" - is-core-module@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946" @@ -7571,14 +7453,6 @@ is-installed-globally@^0.3.2: global-dirs "^2.0.1" is-path-inside "^3.0.1" -is-installed-globally@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" - is-negative-zero@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" @@ -7625,10 +7499,10 @@ is-observable@^1.1.0: dependencies: symbol-observable "^1.1.0" -is-path-inside@^3.0.1, is-path-inside@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== +is-path-inside@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" + integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== is-plain-obj@^1.0.0: version "1.1.0" @@ -7690,9 +7564,9 @@ is-stream@^1.0.1, is-stream@^1.1.0: integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= is-stream@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" - integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" + integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== is-string@^1.0.4: version "1.0.4" @@ -7735,11 +7609,6 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= -is-unicode-supported@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" - integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== - is-utf8@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" @@ -8608,19 +8477,6 @@ listr-verbose-renderer@^0.5.0: date-fns "^1.27.2" figures "^2.0.0" -listr2@^3.8.3: - version "3.12.1" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.12.1.tgz#75e515b86c66b60baf253542cc0dced6b60fedaf" - integrity sha512-oB1DlXlCzGPbvWhqYBZUQEPJKqsmebQWofXG6Mpbe3uIvoNl8mctBEojyF13ZyqwQ91clCWXpwsWp+t98K4FOQ== - dependencies: - cli-truncate "^2.1.0" - colorette "^1.4.0" - log-update "^4.0.0" - p-map "^4.0.0" - rxjs "^6.6.7" - through "^2.3.8" - wrap-ansi "^7.0.0" - listr@^0.14.3: version "0.14.3" resolved "https://registry.yarnpkg.com/listr/-/listr-0.14.3.tgz#2fea909604e434be464c50bddba0d496928fa586" @@ -8783,7 +8639,7 @@ lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw== -lodash@^4.15.0, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: +lodash@^4.15.0, lodash@^4.17.20, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -8793,6 +8649,11 @@ lodash@^4.17.15: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== +lodash@^4.17.19: + version "4.17.20" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" + integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== + log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -8801,12 +8662,11 @@ log-symbols@^1.0.2: chalk "^1.0.0" log-symbols@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" - integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== + version "4.0.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.0.0.tgz#69b3cc46d20f448eccdb75ea1fa733d9e821c920" + integrity sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA== dependencies: - chalk "^4.1.0" - is-unicode-supported "^0.1.0" + chalk "^4.0.0" log-update@^2.3.0: version "2.3.0" @@ -8817,16 +8677,6 @@ log-update@^2.3.0: cli-cursor "^2.0.0" wrap-ansi "^3.0.1" -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" - lolex@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/lolex/-/lolex-5.1.2.tgz#953694d098ce7c07bc5ed6d0e42bc6c0c6d5a367" @@ -9108,19 +8958,7 @@ mime-db@1.40.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== -mime-db@1.49.0: - version "1.49.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed" - integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA== - -mime-types@^2.1.12, mime-types@~2.1.19: - version "2.1.32" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5" - integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A== - dependencies: - mime-db "1.49.0" - -mime-types@~2.1.24: +mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.24" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== @@ -9287,16 +9125,11 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@2.1.2: +ms@2.1.2, ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - nan@^2.12.1, nan@^2.13.2: version "2.14.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" @@ -9994,13 +9827,6 @@ p-map@^2.0.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -10637,7 +10463,7 @@ prettier@2.2.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== -pretty-bytes@^5.4.1, pretty-bytes@^5.6.0: +pretty-bytes@^5.4.1: version "5.6.0" resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== @@ -10746,12 +10572,12 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= -psl@^1.1.24: +psl@^1.1.24, psl@^1.1.28: version "1.2.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.2.0.tgz#df12b5b1b3a30f51c329eacbdef98f3a6e136dc6" integrity sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -11175,7 +11001,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== @@ -11188,19 +11014,6 @@ read-pkg@^5.2.0: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^2.2.2: - version "2.3.7" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" - integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - readable-stream@^3.1.1: version "3.4.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" @@ -11644,14 +11457,6 @@ restore-cursor@^2.0.0: onetime "^2.0.0" signal-exit "^3.0.2" -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -11718,10 +11523,10 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.3.3, rxjs@^6.6.7: - version "6.6.7" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" - integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== +rxjs@^6.3.3: + version "6.6.6" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.6.tgz#14d8417aa5a07c5e633995b525e1e3c0dec03b70" + integrity sha512-/oTwee4N4iWzAMAL9xdGKjkEHmIwupR3oXbQjCKywF1BeFohswF3vZdogbmEF6pZkOsXTzWkrZszrWpQTByYVg== dependencies: tslib "^1.9.0" @@ -11730,16 +11535,11 @@ safe-buffer@5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" integrity sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg== -safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@^5.1.1: +safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2: version "5.2.0" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== -safe-buffer@^5.0.1, safe-buffer@^5.1.2: - version "5.2.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== - safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -12021,9 +11821,9 @@ signal-exit@^3.0.0: integrity sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0= signal-exit@^3.0.2: - version "3.0.4" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.4.tgz#366a4684d175b9cab2081e3681fda3747b6c51d7" - integrity sha512-rqYhcAnZ6d/vTPGghdrw7iumdcbXpsk1b8IG/rz+VWV51DM0p7XCtMoJ3qhPLIbp3tvyt3pKRbaaEMZYpHto8Q== + version "3.0.3" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" + integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== sisteransi@^1.0.5: version "1.0.5" @@ -12050,15 +11850,6 @@ slice-ansi@0.0.4: resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" integrity sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU= -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -12406,7 +12197,7 @@ string-width@^3.0.0, string-width@^3.1.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" -string-width@^4.1.0: +string-width@^4.1.0, string-width@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== @@ -12415,15 +12206,6 @@ string-width@^4.1.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" -string-width@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" - integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" - string.prototype.matchall@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz#48bb510326fb9fdeb6a33ceaa81a6ea04ef7648e" @@ -12611,13 +12393,6 @@ supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" -supports-color@^8.1.1: - version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-hyperlinks@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47" @@ -12767,11 +12542,6 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= - timers-browserify@^2.0.4: version "2.0.10" resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-2.0.10.tgz#1d28e3d2aadf1d5a5996c4e9f95601cd053480ae" @@ -12967,16 +12737,11 @@ tslib@1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" integrity sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ== -tslib@^1.8.0, tslib@^1.8.1: +tslib@^1.8.0, tslib@^1.8.1, tslib@^1.9.0: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== -tslib@^1.9.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - tslint-eslint-rules@^5.4.0: version "5.4.0" resolved "https://registry.yarnpkg.com/tslint-eslint-rules/-/tslint-eslint-rules-5.4.0.tgz#e488cc9181bf193fe5cd7bfca213a7695f1737b5" @@ -13073,11 +12838,6 @@ type-fest@^0.11.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== -type-fest@^0.21.3: - version "0.21.3" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" - integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== - type-fest@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" @@ -13245,9 +13005,9 @@ upper-case@^1.1.1: integrity sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg= uri-js@^4.2.2: - version "4.4.1" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" - integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + version "4.2.2" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" + integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== dependencies: punycode "^2.1.0" @@ -13329,9 +13089,9 @@ utils-merge@1.0.1: integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= uuid@^3.3.2: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== uuid@^7.0.3: version "7.0.3" @@ -13343,11 +13103,6 @@ uuid@^8.3.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31" integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg== -uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - v8-compile-cache@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" @@ -13715,15 +13470,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"