firehose addon module updates (#15439)

This commit is contained in:
Benjamin Edwards 2023-12-07 19:24:03 -05:00 committed by GitHub
parent 685353be61
commit eb7f838125
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 112 additions and 151 deletions

View file

@ -0,0 +1 @@
* Update firehose delivery addon to use latest module version, this includes breaking changes to previous configurations as the default prefixes have been changed to natively support time-partitioned Athena table creation

View file

@ -0,0 +1,9 @@
# Firehose Logging Destination Setup
In this Terraform code, we are defining an IAM Role named `fleet_role` in our AWS Account, that will be assumed by the Fleet application we are hosting. We are only allowing this specific IAM Role (identified by its ARN) to perform certain actions on the Firehose service, such as `DescribeDeliveryStream`, `PutRecord`, and `PutRecordBatch`.
The reason we need a local IAM role in your account is so that we can assume role into it, and you have full control over the permissions it has. The associated IAM policy in the same file specifies the minimum allowed permissions.
The Firehose service is KMS encrypted, so the IAM Role we assume into needs permission to the KMS key that is being used to encrypt the data going into Firehose. Additionally, if the data is being delivered to S3, it will also be encrypted with KMS using the AWS S3 KMS key that is managed by AWS. This is because only customer managed keys can be shared across accounts, and the Firehose delivery stream is actually the one writing to S3.
This code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account.

View file

@ -0,0 +1 @@
header-from: .header.md

View file

@ -1,4 +1,4 @@
## Introduction
# Firehose Logging Destination Setup
In this Terraform code, we are defining an IAM Role named `fleet_role` in our AWS Account, that will be assumed by the Fleet application we are hosting. We are only allowing this specific IAM Role (identified by its ARN) to perform certain actions on the Firehose service, such as `DescribeDeliveryStream`, `PutRecord`, and `PutRecordBatch`.
@ -6,20 +6,20 @@ The reason we need a local IAM role in your account is so that we can assume rol
The Firehose service is KMS encrypted, so the IAM Role we assume into needs permission to the KMS key that is being used to encrypt the data going into Firehose. Additionally, if the data is being delivered to S3, it will also be encrypted with KMS using the AWS S3 KMS key that is managed by AWS. This is because only customer managed keys can be shared across accounts, and the Firehose delivery stream is actually the one writing to S3.
Overall, this code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account.
<!-- BEGIN_TF_DOCS -->
This code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account.
## Requirements
| Name | Version |
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.7 |
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.52.0 |
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.29.0 |
## Providers
| Name | Version |
|------|---------|
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.52.0 |
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.29.0 |
## Modules
@ -30,16 +30,14 @@ No modules.
| Name | Type |
|------|------|
| [aws_iam_policy.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.fleet-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy_attachment.fleet-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
| [aws_iam_policy.fleet_firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_role.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.fleet_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy_attachment.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_kinesis_firehose_delivery_stream.osquery_results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kinesis_firehose_delivery_stream.osquery_status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kms_key.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
| [aws_iam_role_policy_attachment.fleet_firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_kinesis_firehose_delivery_stream.fleet_log_destinations](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kms_key.firehose_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
| [aws_s3_bucket.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket_acl.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
| [aws_s3_bucket_public_access_block.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@ -54,19 +52,16 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_firehose_results_name"></a> [firehose\_results\_name](#input\_firehose\_results\_name) | firehose delivery stream name for osquery results logs | `string` | `"osquery_results"` | no |
| <a name="input_firehose_status_name"></a> [firehose\_status\_name](#input\_firehose\_status\_name) | firehose delivery stream name for osquery status logs | `string` | `"osquery_status"` | no |
| <a name="input_fleet_iam_role_arn"></a> [fleet\_iam\_role\_arn](#input\_fleet\_iam\_role\_arn) | the arn of the fleet role that firehose will assume to write data to your bucket | `string` | n/a | yes |
| <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created. | `string` | `""` | no |
| <a name="input_log_destinations"></a> [log\_destinations](#input\_log\_destinations) | A map of configurations for Firehose delivery streams. | <pre>map(object({<br> name = string<br> prefix = string<br> error_output_prefix = string<br> buffering_size = number<br> buffering_interval = number<br> compression_format = string<br> }))</pre> | <pre>{<br> "audit": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "fleet_audit",<br> "prefix": "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> },<br> "results": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "osquery_results",<br> "prefix": "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> },<br> "status": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "osquery_status",<br> "prefix": "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> }<br>}</pre> | no |
| <a name="input_osquery_logging_destination_bucket_name"></a> [osquery\_logging\_destination\_bucket\_name](#input\_osquery\_logging\_destination\_bucket\_name) | name of the bucket to store osquery results & status logs | `string` | n/a | yes |
| <a name="input_results_prefix"></a> [results\_prefix](#input\_results\_prefix) | s3 object prefix to give to results logs | `string` | `"results/"` | no |
| <a name="input_status_prefix"></a> [status\_prefix](#input\_status\_prefix) | s3 object prefix to give status logs | `string` | `"status/"` | no |
| <a name="input_server_side_encryption_enabled"></a> [server\_side\_encryption\_enabled](#input\_server\_side\_encryption\_enabled) | A boolean flag to enable/disable server-side encryption. Defaults to true (enabled). | `bool` | `true` | no |
## Outputs
| Name | Description |
|------|-------------|
| <a name="output_firehose_iam_role"></a> [firehose\_iam\_role](#output\_firehose\_iam\_role) | n/a |
| <a name="output_firehose_results"></a> [firehose\_results](#output\_firehose\_results) | n/a |
| <a name="output_firehose_status"></a> [firehose\_status](#output\_firehose\_status) | n/a |
| <a name="output_log_destinations"></a> [log\_destinations](#output\_log\_destinations) | Map of Firehose delivery streams' names. |
| <a name="output_s3_destination"></a> [s3\_destination](#output\_s3\_destination) | n/a |
<!-- END_TF_DOCS -->

View file

@ -30,13 +30,9 @@ data "aws_iam_policy_document" "firehose_policy" {
statement {
effect = "Allow"
actions = ["logs:PutLogEvents"]
resources = concat([
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_results_name}:*",
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_status_name}:*",
],
var.firehose_status_name == "" ? [] : [
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_audit_name}:*"
])
resources = [
for name in keys(var.log_destinations) : "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.log_destinations[name].name}:*"
]
}
statement {
@ -63,52 +59,32 @@ resource "aws_iam_role_policy_attachment" "firehose" {
role = aws_iam_role.firehose.name
}
resource "aws_kms_key" "firehose" {
enable_key_rotation = true
resource "aws_kms_key" "firehose_key" {
count = var.server_side_encryption_enabled && length(var.kms_key_arn) == 0 ? 1 : 0
description = "KMS key for encrypting Firehose data."
}
resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
name = var.firehose_results_name
destination = "s3"
resource "aws_kinesis_firehose_delivery_stream" "fleet_log_destinations" {
for_each = var.log_destinations
name = each.value.name
destination = "extended_s3"
server_side_encryption {
key_arn = aws_kms_key.firehose.arn
dynamic "server_side_encryption" {
for_each = var.server_side_encryption_enabled ? [1] : []
content {
enabled = var.server_side_encryption_enabled
key_arn = length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn
key_type = "CUSTOMER_MANAGED_CMK"
}
}
s3_configuration {
prefix = var.results_prefix
role_arn = aws_iam_role.firehose.arn
bucket_arn = aws_s3_bucket.destination.arn
}
}
resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
name = var.firehose_status_name
destination = "s3"
server_side_encryption {
key_arn = aws_kms_key.firehose.arn
}
s3_configuration {
prefix = var.status_prefix
role_arn = aws_iam_role.firehose.arn
bucket_arn = aws_s3_bucket.destination.arn
}
}
resource "aws_kinesis_firehose_delivery_stream" "fleet_audit" {
count = length(var.firehose_audit_name) > 0 ? 1 : 0
name = var.firehose_audit_name
destination = "s3"
server_side_encryption {
key_arn = aws_kms_key.firehose.arn
}
s3_configuration {
prefix = var.audit_prefix
role_arn = aws_iam_role.firehose.arn
bucket_arn = aws_s3_bucket.destination.arn
extended_s3_configuration {
bucket_arn = aws_s3_bucket.destination.arn
role_arn = aws_iam_role.firehose.arn
prefix = each.value.prefix
error_output_prefix = each.value.error_output_prefix
buffering_size = each.value.buffering_size
buffering_interval = each.value.buffering_interval
compression_format = each.value.compression_format
}
}

View file

@ -22,18 +22,23 @@ data "aws_iam_policy_document" "firehose" {
"firehose:PutRecordBatch",
]
resources = [
aws_kinesis_firehose_delivery_stream.osquery_results.arn,
aws_kinesis_firehose_delivery_stream.osquery_status.arn
for stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : stream.arn
]
}
statement {
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:GenerateDataKey"
]
resources = [aws_kms_key.firehose.arn]
dynamic "statement" {
for_each = var.server_side_encryption_enabled ? [1] : []
content {
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:GenerateDataKey",
]
resources = [
length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn
]
}
}
}
@ -45,39 +50,4 @@ resource "aws_iam_policy" "fleet_firehose" {
resource "aws_iam_role_policy_attachment" "fleet_firehose" {
policy_arn = aws_iam_policy.fleet_firehose.arn
role = aws_iam_role.fleet_role.name
}
data "aws_iam_policy_document" "firehose_audit" {
count = length(var.firehose_audit_name) > 0 ? 1 : 0
statement {
effect = "Allow"
actions = [
"firehose:DescribeDeliveryStream",
"firehose:PutRecord",
"firehose:PutRecordBatch",
]
resources = [
aws_kinesis_firehose_delivery_stream.fleet_audit.*.arn
]
}
statement {
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:GenerateDataKey"
]
resources = [aws_kms_key.firehose.arn]
}
}
resource "aws_iam_policy" "fleet_firehose_audit" {
count = length(var.firehose_audit_name) > 0 ? 1 : 0
policy = data.aws_iam_policy_document.firehose_audit.*.json
}
resource "aws_iam_role_policy_attachment" "fleet_firehose_audit" {
count = length(var.firehose_audit_name) > 0 ? 1 : 0
policy_arn = aws_iam_policy.fleet_firehose_audit.*.arn
role = aws_iam_role.fleet_role.name
}

View file

@ -6,10 +6,7 @@ output "s3_destination" {
value = aws_s3_bucket.destination.arn
}
output "firehose_results" {
value = aws_kinesis_firehose_delivery_stream.osquery_results.name
output "log_destinations" {
description = "Map of Firehose delivery streams' names."
value = { for key, stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : key => stream.name }
}
output "firehose_status" {
value = aws_kinesis_firehose_delivery_stream.osquery_status.name
}

View file

@ -1,7 +1,7 @@
data "aws_region" "current" {}
data "aws_caller_identity" "current" {}
data "aws_kms_alias" "s3" {
name = "aws/s3"
name = "alias/aws/s3"
}
resource "aws_s3_bucket" "destination" {
@ -16,11 +16,6 @@ resource "aws_s3_bucket_public_access_block" "destination" {
restrict_public_buckets = true
}
resource "aws_s3_bucket_acl" "destination" {
bucket = aws_s3_bucket.destination.id
acl = "private"
}
// Objects in S3 are now encrypted by default https://aws.amazon.com/blogs/aws/amazon-s3-encrypts-new-objects-by-default/
// If you need more granular control, use a customer managed KMS Key
resource "aws_s3_bucket_server_side_encryption_configuration" "destination" {

View file

@ -3,40 +3,57 @@ variable "osquery_logging_destination_bucket_name" {
description = "name of the bucket to store osquery results & status logs"
}
variable "firehose_results_name" {
type = string
description = "firehose delivery stream name for osquery results logs"
default = "osquery_results"
}
variable "firehose_status_name" {
type = string
description = "firehose delivery stream name for osquery status logs"
default = "osquery_status"
}
variable "firehose_audit_name" {
type = string
description = "firehose delivery stream name for Fleet audit logs"
default = ""
}
variable "fleet_iam_role_arn" {
type = string
description = "the arn of the fleet role that firehose will assume to write data to your bucket"
}
variable "results_prefix" {
default = "results/"
description = "s3 object prefix to give to results logs"
variable "log_destinations" {
description = "A map of configurations for Firehose delivery streams."
type = map(object({
name = string
prefix = string
error_output_prefix = string
buffering_size = number
buffering_interval = number
compression_format = string
}))
default = {
results = {
name = "osquery_results"
prefix = "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
},
status = {
name = "osquery_status"
prefix = "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
},
audit = {
name = "fleet_audit"
prefix = "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
}
}
}
variable "status_prefix" {
default = "status/"
description = "s3 object prefix to give status logs"
variable "server_side_encryption_enabled" {
description = "A boolean flag to enable/disable server-side encryption. Defaults to true (enabled)."
type = bool
default = true
}
variable "audit_prefix" {
default = "audit/"
description = "s3 object prefix to give Fleet audit logs"
variable "kms_key_arn" {
description = "An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created."
type = string
default = ""
}

View file

@ -4,7 +4,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 4.52.0"
version = ">= 5.29.0"
}
}
}