Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-23 17:08:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Implement manual labels

Browse source 
"Manual" labels can be specified by hostname, allowing users to specify
the membership of a label without having to use a dynamic query. See the
included documentation.
This commit is contained in:
Zachary Wasserman 2020-04-07 15:12:32 -07:00 committed by Zachary Wasserman
parent 608772917c
commit 42bea2a144
12 changed files with 332 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
docs/cli/file-format.md
View file

@ -155,6 +155,22 @@ spec:
);
```
Labels can also be "manually managed". When defining the label, reference hosts
by hostname:
```yaml
apiVersion: v1
kind: label
spec:
name: Manually Managed Example
label_membership_type: manual
hosts:
- hostname1
- hostname2
- hostname3
```
## Osquery Configuration Options
The following file describes options returned to osqueryd when it checks for configuration. See the [osquery documentation](https://osquery.readthedocs.io/en/stable/deployment/configuration/#options) for the available options. Existing options will be over-written by the application of this file.

25
frontend/components/forms/LabelForm/LabelForm.jsx
View file

@ -22,6 +22,9 @@ class LabelForm extends Component {
platform: formFieldInterface.isRequired,
query: formFieldInterface.isRequired,
}).isRequired,
formData: PropTypes.shape({
label_membership_type: PropTypes.string,
}),
handleSubmit: PropTypes.func.isRequired,
isEdit: PropTypes.bool,
onCancel: PropTypes.func.isRequired,
@ -50,24 +53,35 @@ class LabelForm extends Component {
}
render () {
const { baseError, fields, handleSubmit, isEdit, onCancel } = this.props;
const { baseError, fields, handleSubmit, isEdit, onCancel, formData } = this.props;
const { onLoad } = this;
const isBuiltin = formData && (formData.label_type === 'builtin' || formData.type === 'status');
const isManual = formData && formData.label_membership_type === 'manual';
const headerText = isEdit ? 'Edit Label' : 'New Label';
const saveBtnText = isEdit ? 'Update Label' : 'Save Label';
const aceHintText = isEdit ? 'Label queries are immutable. To change the query, delete this label and create a new one.' : '';
if (isBuiltin) {
return (
<form className={`${baseClass}__wrapper`} onSubmit={handleSubmit}>
<h1>Built in labels cannot be edited</h1>
</form>
);
}
return (
<form className={`${baseClass}__wrapper`} onSubmit={handleSubmit}>
<h1>{headerText}</h1>
<KolideAce
{!isManual && (<KolideAce
{...fields.query}
label="SQL"
onLoad={onLoad}
readOnly={isEdit}
wrapperClassName={`${baseClass}__text-editor-wrapper`}
hint={<span>{aceHintText}</span>}
hint={aceHintText}
handleSubmit={noop}
/>
)}
{baseError && <div className="form__base-error">{baseError}</div>}
<InputField
@ -81,13 +95,14 @@ class LabelForm extends Component {
label="Description"
type="textarea"
/>
<div className="form-field form-field--dropdown">
{!isManual &&
(<div className="form-field form-field--dropdown">
<label className="form-field__label" htmlFor="platform">Platform</label>
<Dropdown
{...fields.platform}
options={helpers.platformOptions}
/>
</div>
</div>)}
<div className={`${baseClass}__button-wrap`}>
<Button
className={`${baseClass}__cancel-btn`}

4
frontend/components/forms/LabelForm/validate.js
View file

@ -1,11 +1,11 @@
import { size } from 'lodash';
import validateQuery from 'components/forms/validators/validate_query';
export default ({ name, query }) => {
export default ({ name, query, label_membership_type: membershipType }) => {
const errors = {};
const { error: queryError, valid: queryValid } = validateQuery(query);
if (!queryValid) {
if (membershipType !== 'manual' && !queryValid) {
errors.query = queryError;
}

10
frontend/pages/hosts/ManageHostsPage/ManageHostsPage.jsx
View file

@ -402,9 +402,15 @@ export class ManageHostsPage extends PureComponent {
renderQuery = () => {
const { selectedLabel } = this.props;
const { label_type: labelType, query } = selectedLabel;
const { slug, label_type: labelType, label_membership_type: membershipType, query } = selectedLabel;
if (!query || labelType === 1) {
if (membershipType === 'manual' && labelType !== 'builtin') {
return (
<h4 title="Manage manual labels with fleetctl">Manually managed</h4>
);
}
if (!query || slug === 'all-hosts') {
return false;
}

36
server/datastore/datastore_labels_test.go
View file

@ -28,10 +28,10 @@ func testLabels(t *testing.T, db kolide.Datastore) {
baseTime := time.Now()
// Only 'All Hosts' query should be returned
// No labels to check
queries, err := db.LabelQueriesForHost(host, baseTime)
assert.Nil(t, err)
assert.Len(t, queries, 1)
assert.Len(t, queries, 0)
// Only 'All Hosts' label should be returned
labels, err := db.ListLabelsForHost(host.ID)
@ -64,7 +64,6 @@ func testLabels(t *testing.T, db kolide.Datastore) {
require.Nil(t, err)
expectQueries := map[string]string{
"1": "select 1",
"2": "query3",
"3": "query1",
"4": "query2",
@ -125,7 +124,7 @@ func testLabels(t *testing.T, db kolide.Datastore) {
expectQueries["7"] = "query6"
queries, err = db.LabelQueriesForHost(host, baseTime.Add(-1*time.Minute))
assert.Nil(t, err)
assert.Len(t, queries, 6)
assert.Len(t, queries, 5)
// After expiration, all queries should be returned
queries, err = db.LabelQueriesForHost(host, baseTime.Add((2 * time.Minute)))
@ -147,7 +146,7 @@ func testLabels(t *testing.T, db kolide.Datastore) {
hosts[0].Platform = "darwin"
queries, err = db.LabelQueriesForHost(&hosts[0], time.Now())
assert.Nil(t, err)
assert.Len(t, queries, 6)
assert.Len(t, queries, 5)
// Only the 'All Hosts' label should apply for a host with no labels
// executed.
@ -434,6 +433,19 @@ func testChangeLabelDetails(t *testing.T, db kolide.Datastore) {
}
func setupLabelSpecsTest(t *testing.T, ds kolide.Datastore) []*kolide.LabelSpec {
for i := 0; i < 1000; i++ {
_, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
LabelUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: strconv.Itoa(i),
NodeKey: strconv.Itoa(i),
UUID: strconv.Itoa(i),
HostName: strconv.Itoa(i),
})
require.Nil(t, err)
}
expectedSpecs := []*kolide.LabelSpec{
&kolide.LabelSpec{
Name: "foo",
@ -450,9 +462,17 @@ func setupLabelSpecsTest(t *testing.T, ds kolide.Datastore) []*kolide.LabelSpec
Query: "select * from bing",
},
&kolide.LabelSpec{
Name: "All Hosts",
Query: "SELECT 1",
LabelType: kolide.LabelTypeBuiltIn,
Name: "All Hosts",
Query: "SELECT 1",
LabelType: kolide.LabelTypeBuiltIn,
LabelMembershipType: kolide.LabelMembershipTypeManual,
},
&kolide.LabelSpec{
Name: "Manual Label",
LabelMembershipType: kolide.LabelMembershipTypeManual,
Hosts: []string{
"1", "2", "3", "4",
},
},
}
err := ds.ApplyLabelSpecs(expectedSpecs)

2
server/datastore/mysql/hosts.go
View file

@ -265,7 +265,7 @@ func (d *Datastore) EnrollHost(osqueryHostID, nodeKey, secretName string) (*koli
return nil, errors.Wrap(err, "getting the host to return")
}
_, err = d.db.Exec(`INSERT INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, id)
_, err = d.db.Exec(`INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, id)
if err != nil {
return nil, errors.Wrap(err, "insert new host into all hosts label")
}

148
server/datastore/mysql/labels.go
View file

@ -18,14 +18,16 @@ func (d *Datastore) ApplyLabelSpecs(specs []*kolide.LabelSpec) (err error) {
description,
query,
platform,
label_type
) VALUES ( ?, ?, ?, ?, ? )
label_type,
label_membership_type
) VALUES ( ?, ?, ?, ?, ? , ?)
ON DUPLICATE KEY UPDATE
name = VALUES(name),
description = VALUES(description),
query = VALUES(query),
platform = VALUES(platform),
label_type = VALUES(label_type),
label_membership_type = VALUES(label_membership_type),
deleted = false
`
stmt, err := tx.Prepare(sql)
@ -37,10 +39,54 @@ func (d *Datastore) ApplyLabelSpecs(specs []*kolide.LabelSpec) (err error) {
if s.Name == "" {
return errors.New("label name must not be empty")
}
_, err := stmt.Exec(s.Name, s.Description, s.Query, s.Platform, s.LabelType)
_, err := stmt.Exec(s.Name, s.Description, s.Query, s.Platform, s.LabelType, s.LabelMembershipType)
if err != nil {
return errors.Wrap(err, "exec ApplyLabelSpecs insert")
}
if s.LabelType == kolide.LabelTypeBuiltIn ||
s.LabelMembershipType != kolide.LabelMembershipTypeManual {
// No need to update membership
continue
}
var labelID uint
sql = `
SELECT id from labels WHERE name = ?
`
err = tx.Get(&labelID, sql, s.Name)
if err != nil {
return errors.Wrap(err, "get label ID")
}
sql = `
DELETE FROM label_membership WHERE label_id = ?
`
_, err = tx.Exec(sql, labelID)
if err != nil {
return errors.Wrap(err, "clear membership for ID")
}
if len(s.Hosts) == 0 {
continue
}
// Split hostnames into batches to avoid parameter limit in MySQL.
for _, hostnames := range batchHostnames(s.Hosts) {
// Use ignore because duplicate hostnames could appear in
// different batches and would result in duplicate key errors.
sql = `
INSERT IGNORE INTO label_membership (label_id, host_id) (SELECT ?, id FROM hosts where host_name IN (?))
`
sql, args, err := sqlx.In(sql, labelID, hostnames)
if err != nil {
return errors.Wrap(err, "build membership IN statement")
}
_, err = tx.Exec(sql, args...)
if err != nil {
return errors.Wrap(err, "execute membership INSERT")
}
}
}
return nil
@ -49,21 +95,46 @@ func (d *Datastore) ApplyLabelSpecs(specs []*kolide.LabelSpec) (err error) {
return errors.Wrap(err, "ApplyLabelSpecs transaction")
}
func batchHostnames(hostnames []string) [][]string {
// Split hostnames into batches so that they can all be inserted without
// overflowing the MySQL max number of parameters (somewhere around 65,000
// but not well documented). Algorithm from
// https://github.com/golang/go/wiki/SliceTricks#batching-with-minimal-allocation
const batchSize = 50000 // Large, but well under the undocumented limit
batches := make([][]string, 0, (len(hostnames)+batchSize-1)/batchSize)
for batchSize < len(hostnames) {
hostnames, batches = hostnames[batchSize:], append(batches, hostnames[0:batchSize:batchSize])
}
batches = append(batches, hostnames)
return batches
}
func (d *Datastore) GetLabelSpecs() ([]*kolide.LabelSpec, error) {
var specs []*kolide.LabelSpec
// Get basic specs
query := "SELECT name, description, query, platform, label_type FROM labels"
query := "SELECT name, description, query, platform, label_type, label_membership_type FROM labels"
if err := d.db.Select(&specs, query); err != nil {
return nil, errors.Wrap(err, "get labels")
}
for _, spec := range specs {
if spec.LabelType != kolide.LabelTypeBuiltIn &&
spec.LabelMembershipType == kolide.LabelMembershipTypeManual {
err := d.getLabelHostnames(spec)
if err != nil {
return nil, err
}
}
}
return specs, nil
}
func (d *Datastore) GetLabelSpec(name string) (*kolide.LabelSpec, error) {
var specs []*kolide.LabelSpec
query := `
SELECT name, description, query, platform, label_type
SELECT name, description, query, platform, label_type, label_membership_type
FROM labels
WHERE name = ?
`
@ -77,7 +148,34 @@ WHERE name = ?
return nil, errors.Errorf("expected 1 label row, got %d", len(specs))
}
return specs[0], nil
spec := specs[0]
if spec.LabelType != kolide.LabelTypeBuiltIn &&
spec.LabelMembershipType == kolide.LabelMembershipTypeManual {
err := d.getLabelHostnames(spec)
if err != nil {
return nil, err
}
}
return spec, nil
}
func (d *Datastore) getLabelHostnames(label *kolide.LabelSpec) error {
sql := `
SELECT host_name
FROM hosts
WHERE id IN
(
SELECT host_id
FROM label_membership
WHERE label_id = (SELECT id FROM labels WHERE name = ?)
)
`
err := d.db.Select(&label.Hosts, sql, label.Name)
if err != nil {
return errors.Wrap(err, "get hostnames for label")
}
return nil
}
// NewLabel creates a new kolide.Label
@ -97,8 +195,9 @@ func (d *Datastore) NewLabel(label *kolide.Label, opts ...kolide.OptionalArg) (*
description,
query,
platform,
label_type
) VALUES ( ?, ?, ?, ?, ?)
label_type,
label_membership_type
) VALUES ( ?, ?, ?, ?, ?, ?)
`
case sql.ErrNoRows:
query = `
@ -107,13 +206,22 @@ func (d *Datastore) NewLabel(label *kolide.Label, opts ...kolide.OptionalArg) (*
description,
query,
platform,
label_type
) VALUES ( ?, ?, ?, ?, ?)
label_type,
label_membership_type
) VALUES ( ?, ?, ?, ?, ?, ?)
`
default:
return nil, errors.Wrap(err, "check for existing label")
}
result, err := db.Exec(query, label.Name, label.Description, label.Query, label.Platform, label.LabelType)
result, err := db.Exec(
query,
label.Name,
label.Description,
label.Query,
label.Platform,
label.LabelType,
label.LabelMembershipType,
)
if err != nil {
return nil, errors.Wrap(err, "inserting label")
}
@ -185,8 +293,10 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma
sql := `
SELECT id, query
FROM labels
WHERE platform = ? OR platform = ''`
rows, err = d.db.Query(sql, host.Platform)
WHERE platform = ? OR platform = ''
AND label_membership_type = ?
`
rows, err = d.db.Query(sql, host.Platform, kolide.LabelMembershipTypeDynamic)
} else {
// Retrieve all labels (with matching platform) iff there is a label
// that has been created since this host last reported label query
@ -195,8 +305,16 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma
SELECT id, query
FROM labels
WHERE ((SELECT max(created_at) FROM labels WHERE platform = ? OR platform = '') > ?)
AND (platform = ? OR platform = '')`
rows, err = d.db.Query(sql, host.Platform, host.LabelUpdateTime, host.Platform)
AND (platform = ? OR platform = '')
AND label_membership_type = ?
`
rows, err = d.db.Query(
sql,
host.Platform,
host.LabelUpdateTime,
host.Platform,
kolide.LabelMembershipTypeDynamic,
)
}
if err != nil && err != sql.ErrNoRows {

35
server/datastore/mysql/migrations/tables/20200407120000_AddLabelMembershipType.go Normal file
View file

@ -0,0 +1,35 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up20200407120000, Down20200407120000)
}
func Up20200407120000(tx *sql.Tx) error {
if _, err := tx.Exec(
"ALTER TABLE `labels` " +
"ADD COLUMN `label_membership_type` int(10) unsigned NOT NULL default '0'",
); err != nil {
return errors.Wrap(err, "add label_membership_type column ")
}
// All hosts should now be the only "manual" label
if _, err := tx.Exec(
"UPDATE `labels` " +
"SET `label_membership_type` = 1 " +
"WHERE `name` = 'All Hosts' AND `label_type` = 1",
); err != nil {
return errors.Wrap(err, "drop label_query_executions")
}
return nil
}
func Down20200407120000(tx *sql.Tx) error {
return nil
}

2
server/datastore/mysql/mysql.go
View file

@ -23,7 +23,7 @@ import (
)
const (
defaultSelectLimit = 100000
defaultSelectLimit = 1000000
)
var (

86
server/kolide/labels.go
View file

@ -3,6 +3,8 @@ package kolide
import (
"context"
"time"
"github.com/pkg/errors"
)
type LabelStore interface {
@ -103,15 +105,73 @@ const (
LabelTypeBuiltIn
)
func (t LabelType) MarshalJSON() ([]byte, error) {
switch t {
case LabelTypeRegular:
return []byte(`"regular"`), nil
case LabelTypeBuiltIn:
return []byte(`"builtin"`), nil
default:
return nil, errors.Errorf("invalid LabelType: %d", t)
}
}
func (t *LabelType) UnmarshalJSON(b []byte) error {
switch string(b) {
case `"regular"`:
*t = LabelTypeRegular
case `"builtin"`:
*t = LabelTypeBuiltIn
default:
return errors.Errorf("invalid LabelType: %s", string(b))
}
return nil
}
// LabelMembershipType sets how the membership of the label is determined.
type LabelMembershipType uint
const (
// LabelTypeDynamic indicates that the label is populated dynamically (by
// the execution of a label query).
LabelMembershipTypeDynamic LabelMembershipType = iota
// LabelTypeManual indicates that the label is populated manually.
LabelMembershipTypeManual
)
func (t LabelMembershipType) MarshalJSON() ([]byte, error) {
switch t {
case LabelMembershipTypeDynamic:
return []byte(`"dynamic"`), nil
case LabelMembershipTypeManual:
return []byte(`"manual"`), nil
default:
return nil, errors.Errorf("invalid LabelMembershipType: %d", t)
}
}
func (t *LabelMembershipType) UnmarshalJSON(b []byte) error {
switch string(b) {
case `"dynamic"`:
*t = LabelMembershipTypeDynamic
case `"manual"`:
*t = LabelMembershipTypeManual
default:
return errors.Errorf("invalid LabelMembershipType: %s", string(b))
}
return nil
}
type Label struct {
UpdateCreateTimestamps
DeleteFields
ID uint `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Query string `json:"query"`
Platform string `json:"platform"`
LabelType LabelType `json:"label_type" db:"label_type"`
ID uint `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Query string `json:"query"`
Platform string `json:"platform"`
LabelType LabelType `json:"label_type" db:"label_type"`
LabelMembershipType LabelMembershipType `json:"label_membership_type" db:"label_membership_type"`
}
type LabelQueryExecution struct {
@ -123,10 +183,12 @@ type LabelQueryExecution struct {
}
type LabelSpec struct {
ID uint
Name string `json:"name"`
Description string `json:"description"`
Query string `json:"query"`
Platform string `json:"platform,omitempty"`
LabelType LabelType `json:"label_type" db:"label_type"`
ID uint
Name string `json:"name"`
Description string `json:"description"`
Query string `json:"query"`
Platform string `json:"platform,omitempty"`
LabelType LabelType `json:"label_type,omitempty" db:"label_type"`
LabelMembershipType LabelMembershipType `json:"label_membership_type" db:"label_membership_type"`
Hosts []string `json:"hosts,omitempty"`
}

10
server/service/service_labels.go
View file

@ -4,9 +4,19 @@ import (
"context"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (svc service) ApplyLabelSpecs(ctx context.Context, specs []*kolide.LabelSpec) error {
for _, spec := range specs {
if spec.LabelMembershipType == kolide.LabelMembershipTypeDynamic && len(spec.Hosts) > 0 {
return errors.Errorf("label %s is declared as dynamic but contains `hosts` key", spec.Name)
}
if spec.LabelMembershipType == kolide.LabelMembershipTypeManual && spec.Hosts == nil {
// Hosts list doesn't need to contain anything, but it should at least not be nil.
return errors.Errorf("label %s is declared as manual but contains not `hosts key`", spec.Name)
}
}
return svc.ds.ApplyLabelSpecs(specs)
}

7
server/test/new_objects.go
View file

@ -78,9 +78,10 @@ func AddLabelToCampaign(t *testing.T, ds kolide.Datastore, campaignID, labelID u
func AddAllHostsLabel(t *testing.T, ds kolide.Datastore) {
_, err := ds.NewLabel(
&kolide.Label{
Name: "All Hosts",
Query: "select 1",
LabelType: kolide.LabelTypeBuiltIn,
Name: "All Hosts",
Query: "select 1",
LabelType: kolide.LabelTypeBuiltIn,
LabelMembershipType: kolide.LabelMembershipTypeManual,
},
)
require.Nil(t, err)