Enabling s3 software-installers backups with AWS Backup for Dogfood (#29358)

- Creating AWS Backup Plan for S3
- Creating AWS Backup Selection for S3
- Creating AWS Backup Vault for S3 (Source)
- Creating AWS Backup Vault for S3 (Destination)
- Creating KMS Key for AWS Backup (Source)
- Creating KMS Key for AWS Backup (Destination)
- Added `tags = { backup = "true" }` for
fleet_config.software_installers
- Updating tf-mod-root-v1.15.0 -> tf-mod-root-v1.15.1
- Updating IAM Permissions for AWS Backup Role

infrastructure/dogfood/terraform/aws-tf-module/aws-backup.tf

@@ -3,6 +3,451 @@ provider "aws" {
   alias  = "replica"
 }
 
+#####################
+#### PERMISSIONS ####
+#####################
+data "aws_iam_policy_document" "aws_backup_assume_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["backup.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "aws_backup_policy" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:DescribeTable",
+      "dynamodb:CreateBackup"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:DescribeBackup",
+      "dynamodb:DeleteBackup"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*/backup/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:AddTagsToResource",
+      "rds:ListTagsForResource",
+      "rds:DescribeDBSnapshots",
+      "rds:CreateDBSnapshot",
+      "rds:CopyDBSnapshot",
+      "rds:DescribeDBInstances",
+      "rds:CreateDBClusterSnapshot",
+      "rds:DescribeDBClusters",
+      "rds:DescribeDBClusterSnapshots",
+      "rds:CopyDBClusterSnapshot",
+      "rds:DescribeDBClusterAutomatedBackups"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:DeleteDBInstanceAutomatedBackup"
+    ]
+    resources = ["arn:aws:rds:*:*:auto-backup:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:ModifyDBCluster"
+    ]
+    resources = ["arn:aws:rds:*:*:cluster:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:DeleteDBClusterAutomatedBackup"
+    ]
+    resources = ["arn:aws:rds:*:*:cluster-auto-backup:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:ModifyDBInstance"
+    ]
+    resources = ["arn:aws:rds:*:*:db:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:DeleteDBSnapshot",
+      "rds:ModifyDBSnapshotAttribute"
+    ]
+    resources = ["arn:aws:rds:*:*:snapshot:awsbackup:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:DeleteDBClusterSnapshot",
+      "rds:ModifyDBClusterSnapshotAttribute"
+    ]
+    resources = ["arn:aws:rds:*:*:cluster-snapshot:awsbackup:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:StringLike"
+      variable = "kms:ViaService"
+      values = [
+        "dynamodb.*.amazonaws.com",
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:DescribeKey"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:CreateGrant",
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:Bool"
+      variable = "kms:GrantIsForAWSResource"
+      values = [
+        "true"
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "tag:GetResources"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:StartAwsBackupJob",
+      "dynamodb:ListTagsOfResource"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "backup:TagResource"
+    ]
+    resources = ["arn:aws:backup:*:*:recovery-point:*"]
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "aws:PrincipalAccount"
+      values = [
+        "&{aws:ResourceAccount}"
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "cloudwatch:GetMetricData"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "events:DeleteRule",
+      "events:PutTargets",
+      "events:DescribeRule",
+      "events:EnableRule",
+      "events:PutRule",
+      "events:RemoveTargets",
+      "events:ListTargetsByRule",
+      "events:DisableRule"
+    ]
+    resources = ["arn:aws:events:*:*:rule/AwsBackupManagedRule*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "events:ListRules"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:StringLike"
+      variable = "kms:ViaService"
+      values = [
+        "s3.*.amazonaws.com",
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:GetBucketTagging",
+      "s3:GetInventoryConfiguration",
+      "s3:ListBucketVersions",
+      "s3:ListBucket",
+      "s3:GetBucketVersioning",
+      "s3:GetBucketLocation",
+      "s3:GetBucketAcl",
+      "s3:PutInventoryConfiguration",
+      "s3:GetBucketNotification",
+      "s3:PutBucketNotification"
+    ]
+    resources = ["arn:aws:s3:::*software-installers*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:GetObjectAcl",
+      "s3:GetObject",
+      "s3:GetObjectVersionTagging",
+      "s3:GetObjectVersionAcl",
+      "s3:GetObjectTagging",
+      "s3:GetObjectVersion"
+    ]
+    resources = ["arn:aws:s3:::*software-installers*/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:ListAllMyBuckets"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "backup:TagResource"
+    ]
+    resources = ["arn:aws:backup:*:*:recovery-point:*"]
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "aws:PrincipalAccount"
+      values = [
+        "&{aws:ResourceAccount}"
+      ]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "aws_restore_policy" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:Scan",
+      "dynamodb:Query",
+      "dynamodb:UpdateItem",
+      "dynamodb:PutItem",
+      "dynamodb:GetItem",
+      "dynamodb:DeleteItem",
+      "dynamodb:BatchWriteItem",
+      "dynamodb:DescribeTable"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:RestoreTableFromBackup"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*/backup/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:DescribeDBInstances",
+      "rds:DescribeDBSnapshots",
+      "rds:ListTagsForResource",
+      "rds:RestoreDBInstanceFromDBSnapshot",
+      "rds:DeleteDBInstance",
+      "rds:AddTagsToResource",
+      "rds:DescribeDBClusters",
+      "rds:RestoreDBClusterFromSnapshot",
+      "rds:DeleteDBCluster",
+      "rds:RestoreDBInstanceToPointInTime",
+      "rds:DescribeDBClusterSnapshots",
+      "rds:RestoreDBClusterToPointInTime",
+      "rds:CreateTenantDatabase",
+      "rds:DeleteTenantDatabase"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:DescribeKey"
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:Encrypt",
+      "kms:GenerateDataKey",
+      "kms:ReEncryptTo",
+      "kms:ReEncryptFrom",
+      "kms:GenerateDataKeyWithoutPlaintext"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:StringLike"
+      variable = "kms:ViaService"
+      values = [
+        "dynamodb.*.amazonaws.com",
+        "rds.*.amazonaws.com"
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:CreateGrant"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:Bool"
+      variable = "kms:GrantIsForAWSResource"
+      values = [
+        "true"
+      ]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "rds:CreateDBInstance"
+    ]
+    resources = ["arn:aws:rds:*:*:db:*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:RestoreTableFromAwsBackup"
+    ]
+    resources = ["arn:aws:dynamodb:*:*:table/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:CreateBucket",
+      "s3:ListBucketVersions",
+      "s3:ListBucket",
+      "s3:GetBucketVersioning",
+      "s3:GetBucketLocation",
+      "s3:PutBucketVersioning",
+      "s3:PutBucketOwnershipControls",
+      "s3:GetBucketOwnershipControls"
+    ]
+    resources = ["arn:aws:s3:::*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:DeleteObject",
+      "s3:PutObjectVersionAcl",
+      "s3:GetObjectVersionAcl",
+      "s3:GetObjectTagging",
+      "s3:PutObjectTagging",
+      "s3:GetObjectAcl",
+      "s3:PutObjectAcl",
+      "s3:ListMultipartUploadParts",
+      "s3:PutObject"
+    ]
+    resources = ["arn:aws:s3:::*/*"]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:DescribeKey",
+      "kms:GenerateDataKey",
+      "kms:Decrypt"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "ForAnyValue:StringLike"
+      variable = "kms:ViaService"
+      values   = ["s3.*.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "aws_backup" {
+  name               = "aws_backup_role"
+  assume_role_policy = data.aws_iam_policy_document.aws_backup_assume_role.json
+
+  inline_policy {
+    name   = "aws-backup-restore-policy"
+    policy = data.aws_iam_policy_document.aws_backup_policy.json
+  }
+
+  inline_policy {
+    name   = "aws-backup-backup-policy"
+    policy = data.aws_iam_policy_document.aws_restore_policy.json
+  }
+}
+
+##############
+### AURORA ###
+##############
+
 ###
 ## Source Key and backup vault
 ###
@@ -29,29 +474,6 @@ resource "aws_backup_vault" "aws_backup_aurora_destination" {
   kms_key_arn = resource.aws_kms_key.aws_backup_aurora_destination.arn
 }
 
-data "aws_iam_policy_document" "aws_backup_aurora_assume_role" {
-  statement {
-    actions = ["sts:AssumeRole"]
-    principals {
-      type        = "Service"
-      identifiers = ["backup.amazonaws.com"]
-    }
-  }
-}
-
-resource "aws_iam_role" "aws_backup_aurora" {
-  name               = "aws_backup_aurora_role"
-  assume_role_policy = data.aws_iam_policy_document.aws_backup_aurora_assume_role.json
-}
-
-###
-## TODO: MAKE PERMISSIONS MORE GRANULAR
-###
-resource "aws_iam_role_policy_attachment" "aws_backup_aurora_policy" {
-  role       = resource.aws_iam_role.aws_backup_aurora.name
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup"
-}
-
 ###
 ## Starts snapshot copy within 1 hour of scheduled plan start time
 ## Completes backup within 2 hours of start time
@@ -81,7 +503,7 @@ resource "aws_backup_plan" "snapshot_backup_plan" {
 ###
 resource "aws_backup_selection" "snapshot_selection" {
   name         = "aurora_snapshot_backup_selection"
-  iam_role_arn = resource.aws_iam_role.aws_backup_aurora.arn
+  iam_role_arn = resource.aws_iam_role.aws_backup.arn
   plan_id      = resource.aws_backup_plan.snapshot_backup_plan.id
   resources = [
     "arn:aws:rds:us-east-2:160035666661:cluster:*"
@@ -93,3 +515,75 @@ resource "aws_backup_selection" "snapshot_selection" {
     }
   }
 }
+
+##############
+##### S3 #####
+##############
+
+###
+## Source Key and backup vault
+###
+resource "aws_kms_key" "aws_backup_s3_source" {
+  description = "Source CMEK for s3 - AWS Backups"
+}
+
+resource "aws_backup_vault" "aws_backup_s3_source" {
+  name        = "backup_s3_vault_source"
+  kms_key_arn = resource.aws_kms_key.aws_backup_s3_source.arn
+}
+
+###
+## Destination Key and backup vault
+###
+resource "aws_kms_key" "aws_backup_s3_destination" {
+  provider    = aws.replica
+  description = "Destination CMEK for s3 - AWS Backups"
+}
+
+resource "aws_backup_vault" "aws_backup_s3_destination" {
+  provider    = aws.replica
+  name        = "backup_s3_vault_destination"
+  kms_key_arn = resource.aws_kms_key.aws_backup_s3_destination.arn
+}
+
+###
+## Starts snapshot copy within 1 hour of scheduled plan start time
+## Completes backup within 2 hours of start time
+###
+resource "aws_backup_plan" "s3_backup_plan" {
+  name = "s3_backup_plan"
+  rule {
+    rule_name         = "daily_s3_backup"
+    target_vault_name = resource.aws_backup_vault.aws_backup_s3_source.name
+    schedule          = "cron(0 5 * * ? *)"
+    start_window      = 60
+    completion_window = 120
+
+    lifecycle {
+      delete_after = 7
+    }
+
+    copy_action {
+      destination_vault_arn = resource.aws_backup_vault.aws_backup_s3_destination.arn
+    }
+  }
+}
+
+###
+## Backups will occur on:
+## S3 buckets that are tagged with backup = true
+###
+resource "aws_backup_selection" "s3_selection" {
+  name         = "s3_backup_selection"
+  iam_role_arn = resource.aws_iam_role.aws_backup.arn
+  plan_id      = resource.aws_backup_plan.s3_backup_plan.id
+  resources = [
+    "arn:aws:s3:::*-software-installers-*"
+  ]
+  condition {
+    string_equals {
+      key   = "aws:ResourceTag/backup"
+      value = "true"
+    }
+  }
+}

infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf

@@ -39,7 +39,7 @@ resource "null_resource" "build_osquery" {
     osquery_tags_changed    = sha256(jsonencode(var.osquery_tags))
   }
   provisioner "local-exec" {
-    working_dir = "${path.module}"
+    working_dir = path.module
     command     = <<-EOT
       mkdir -p osquery
       cd osquery

infrastructure/dogfood/terraform/aws-tf-module/free.tf

@@ -36,7 +36,7 @@ module "free" {
     subnets                 = module.main.vpc.database_subnets
     backup_retention_period = 30
     cluster_tags = {
-      backup                = "true"
+      backup = "true"
     }
   }
   redis_config = {

infrastructure/dogfood/terraform/aws-tf-module/main.tf

@@ -52,9 +52,9 @@ locals {
   fleet_image    = var.fleet_image # Set this to the version of fleet to be deployed
   geolite2_image = "${aws_ecr_repository.fleet.repository_url}:${split(":", var.fleet_image)[1]}-geolite2-${formatdate("YYYYMMDDhhmm", timestamp())}"
   extra_environment_variables = {
-    FLEET_LICENSE_KEY                          = var.fleet_license
-    FLEET_LOGGING_DEBUG                        = "true"
-    FLEET_LOGGING_JSON                         = "true"
+    FLEET_LICENSE_KEY   = var.fleet_license
+    FLEET_LOGGING_DEBUG = "true"
+    FLEET_LOGGING_JSON  = "true"
     # FLEET_LOGGING_TRACING_ENABLED              = "true"
     # FLEET_LOGGING_TRACING_TYPE                 = "elasticapm"
     FLEET_MYSQL_MAX_OPEN_CONNS                 = "10"
@@ -64,9 +64,9 @@ locals {
     # ELASTIC_APM_SERVER_URL                     = var.elastic_url
     # ELASTIC_APM_SECRET_TOKEN                   = var.elastic_token
     # ELASTIC_APM_SERVICE_NAME                   = "dogfood"
-    FLEET_CALENDAR_PERIODICITY                 = var.fleet_calendar_periodicity
-    FLEET_DEV_ANDROID_ENABLED                  = "1"
-    FLEET_DEV_ANDROID_SERVICE_CREDENTIALS      = var.android_service_credentials
+    FLEET_CALENDAR_PERIODICITY            = var.fleet_calendar_periodicity
+    FLEET_DEV_ANDROID_ENABLED             = "1"
+    FLEET_DEV_ANDROID_SERVICE_CREDENTIALS = var.android_service_credentials
   }
   sentry_secrets = {
     FLEET_SENTRY_DSN = "${aws_secretsmanager_secret.sentry.arn}:FLEET_SENTRY_DSN::"
@@ -75,7 +75,7 @@ locals {
 }
 
 module "main" {
-  source          = "github.com/fleetdm/fleet-terraform?ref=tf-mod-root-v1.13.0"
+  source          = "github.com/fleetdm/fleet-terraform?ref=tf-mod-root-v1.15.1"
   certificate_arn = module.acm.acm_certificate_arn
   vpc = {
     name = local.customer
@@ -92,7 +92,7 @@ module "main" {
     allowed_cidr_blocks     = ["10.255.1.0/24", "10.255.2.0/24", "10.255.3.0/24"]
     backup_retention_period = 30
     cluster_tags = {
-      backup                = "true"
+      backup = "true"
     }
   }
   redis_config = {
@@ -133,7 +133,7 @@ module "main" {
         policy_name = "${local.customer}-iam-policy-execution"
       }
     }
-    extra_iam_policies           = concat(module.firehose-logging.fleet_extra_iam_policies, module.osquery-carve.fleet_extra_iam_policies, module.ses.fleet_extra_iam_policies)
+    extra_iam_policies = concat(module.firehose-logging.fleet_extra_iam_policies, module.osquery-carve.fleet_extra_iam_policies, module.ses.fleet_extra_iam_policies)
     extra_environment_variables = merge(
       module.firehose-logging.fleet_extra_environment_variables,
       module.osquery-carve.fleet_extra_environment_variables,
@@ -147,7 +147,7 @@ module "main" {
       [aws_iam_policy.sentry.arn, aws_iam_policy.osquery_sidecar.arn],
       module.cloudfront-software-installers.extra_execution_iam_policies,
     ) #, module.saml_auth_proxy.fleet_extra_execution_policies)
-    extra_secrets           = merge(
+    extra_secrets = merge(
       module.mdm.extra_secrets,
       local.sentry_secrets,
       module.cloudfront-software-installers.extra_secrets
@@ -159,9 +159,12 @@ module "main" {
     #   container_port   = 8080
     # }]
     software_installers = {
-      bucket_prefix = "${local.customer}-software-installers-"
+      bucket_prefix  = "${local.customer}-software-installers-"
       create_kms_key = true
       kms_alias      = "${local.customer}-software-installers"
+      tags = {
+        backup = "true"
+      }
     }
     # sidecars = [
     #   {