diff --git a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/deployment.yaml b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/deployment.yaml index 2d973d3c51..bb52223706 100644 --- a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/deployment.yaml +++ b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/deployment.yaml @@ -37,7 +37,7 @@ spec: imagePullPolicy: Always command: [/usr/bin/fleet] args: ["serve"] - image: fleetdm/fleet:{{ .Values.imageTag }} + image: {{ .Values.imageRepo }}:{{ .Values.imageTag }} ports: - name: {{ .Values.fleetName }} containerPort: {{ .Values.fleet.listenPort }} diff --git a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/job-migration.yaml b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/job-migration.yaml index 4334b8ab68..8986e1513f 100644 --- a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/job-migration.yaml +++ b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/templates/job-migration.yaml @@ -27,7 +27,7 @@ spec: - name: {{ .Values.fleetName }}-migration command: [/usr/bin/fleet] args: ["prepare","db","--no-prompt"] - image: fleetdm/fleet:{{ .Values.imageTag }} + image: {{ .Values.imageRepo }}:{{ .Values.imageTag }} imagePullPolicy: Always resources: limits: diff --git a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/values.yaml b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/values.yaml index 2b94243cf0..df65d39fb9 100644 --- a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/values.yaml +++ b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/fleet/values.yaml @@ -6,6 +6,7 @@ fleetName: fleet hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy imageTag: v4.12.0 # Version of Fleet to deploy +imageRepo: fleetdm/fleet createNamespace: false # Whether or not to automatically create the Namespace createIngress: true # Whether or not to automatically create an Ingress ingressAnnotations: {} # Additional annotation to add to the Ingress diff --git a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/main.tf b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/main.tf index 03f8a3f0df..db956c858a 100644 --- a/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/main.tf +++ b/infrastructure/sandbox/PreProvisioner/lambda/deploy_terraform/main.tf @@ -54,6 +54,7 @@ variable "installer_bucket_arn" {} variable "oidc_provider_arn" {} variable "oidc_provider" {} variable "kms_key_arn" {} +variable "ecr_url" {} resource "mysql_user" "main" { user = terraform.workspace @@ -159,6 +160,11 @@ resource "helm_release" "main" { value = "main" } + set { + name = "imageRepo" + value = var.ecr_url + } + set { name = "packaging.enrollSecret" value = var.enroll_secret diff --git a/infrastructure/sandbox/PreProvisioner/main.tf b/infrastructure/sandbox/PreProvisioner/main.tf index f61e86ce5c..5a9b58cde2 100644 --- a/infrastructure/sandbox/PreProvisioner/main.tf +++ b/infrastructure/sandbox/PreProvisioner/main.tf @@ -289,6 +289,10 @@ resource "aws_ecs_task_definition" "main" { name = "TF_VAR_kms_key_arn" value = var.kms_key.arn }, + { + name = "TF_VAR_ecr_url" + value = var.ecr.repository_url + }, ]), secrets = concat([ { diff --git a/infrastructure/sandbox/PreProvisioner/variables.tf b/infrastructure/sandbox/PreProvisioner/variables.tf index 77f3f96655..b38a9ea695 100644 --- a/infrastructure/sandbox/PreProvisioner/variables.tf +++ b/infrastructure/sandbox/PreProvisioner/variables.tf @@ -11,3 +11,4 @@ variable "kms_key" {} variable "installer_bucket" {} variable "oidc_provider_arn" {} variable "oidc_provider" {} +variable "ecr" {} diff --git a/infrastructure/sandbox/SharedInfrastructure/eks.tf b/infrastructure/sandbox/SharedInfrastructure/eks.tf index 3511c187f6..c49cde8601 100644 --- a/infrastructure/sandbox/SharedInfrastructure/eks.tf +++ b/infrastructure/sandbox/SharedInfrastructure/eks.tf @@ -80,7 +80,8 @@ module "aws-eks-accelerator-for-terraform" { fargate_profiles = { default = { - fargate_profile_name = "default" + additional_iam_policies = [aws_iam_policy.ecr.arn] + fargate_profile_name = "default" fargate_profile_namespaces = [ { namespace = "default" @@ -160,6 +161,11 @@ resource "helm_release" "haproxy_ingress" { name = "controller.service.type" value = "NodePort" } + + set { + name = "controller.defaultBackendService" + value = "kube-system/default-redirect" + } } resource "aws_lb_target_group" "eks" { @@ -168,7 +174,7 @@ resource "aws_lb_target_group" "eks" { protocol = "HTTP" vpc_id = var.vpc.vpc_id health_check { - matcher = "404" + matcher = "302" } } @@ -202,3 +208,177 @@ resource "kubernetes_manifest" "targetgroupbinding" { } } } + +resource "kubernetes_service" "redirect" { + metadata { + name = "default-redirect" + namespace = "kube-system" + } + + spec { + selector = { + app = kubernetes_deployment.redirect.metadata.0.labels.app + } + port { + port = 80 + name = "http" + } + } +} + +resource "kubernetes_deployment" "redirect" { + metadata { + name = "default-redirect" + namespace = "kube-system" + labels = { + app = "default-redirect" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "default-redirect" + } + } + + template { + metadata { + labels = { + app = "default-redirect" + } + } + + spec { + container { + image = "nginx:1.23.1" + name = "nginx" + + port { + name = "http" + container_port = 80 + } + + resources { + limits = { + cpu = "0.5" + memory = "512Mi" + } + requests = { + cpu = "250m" + memory = "50Mi" + } + } + + volume_mount { + mount_path = "/etc/nginx" + read_only = true + name = "nginx-conf" + } + } + volume { + name = "nginx-conf" + config_map { + name = "default-redirect-config" + items { + key = "nginx.conf" + path = "nginx.conf" + } + } + } + } + } + } +} + +resource "kubernetes_config_map" "redirect" { + metadata { + name = "default-redirect-config" + namespace = "kube-system" + } + + data = { + "nginx.conf" = <<-EOT + user nginx; + worker_processes 1; + error_log /dev/stderr; + events { + worker_connections 10240; + } + http { + log_format main + 'remote_addr:$remote_addr\t' + 'time_local:$time_local\t' + 'method:$request_method\t' + 'uri:$request_uri\t' + 'host:$host\t' + 'status:$status\t' + 'bytes_sent:$body_bytes_sent\t' + 'referer:$http_referer\t' + 'useragent:$http_user_agent\t' + 'forwardedfor:$http_x_forwarded_for\t' + 'request_time:$request_time'; + access_log /dev/stderr main; + server { + listen 80; + server_name _; + location / { + return 302 https://fleetdm.com/try-fleet/sandbox-expired; + } + } + } + EOT + } +} + +resource "aws_iam_policy" "ecr" { + name = "${var.prefix}-ecr" + policy = data.aws_iam_policy_document.ecr.json +} + +data "aws_iam_policy_document" "ecr" { + statement { + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken" + ] + resources = ["*"] + } + statement { + actions = [ #tfsec:ignore:aws-iam-no-policy-wildcards + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ] + resources = [aws_kms_key.ecr.arn] + } +} + +resource "aws_ecr_repository" "main" { + name = "${var.prefix}-eks" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.ecr.arn + } +} + +output "ecr" { + value = aws_ecr_repository.main +} + +resource "aws_kms_key" "ecr" { + deletion_window_in_days = 10 + enable_key_rotation = true +} diff --git a/infrastructure/sandbox/main.tf b/infrastructure/sandbox/main.tf index 12d423811a..3b33e128dd 100644 --- a/infrastructure/sandbox/main.tf +++ b/infrastructure/sandbox/main.tf @@ -156,6 +156,7 @@ module "pre-provisioner" { installer_bucket = module.shared-infrastructure.installer_bucket oidc_provider_arn = module.shared-infrastructure.oidc_provider_arn oidc_provider = module.shared-infrastructure.oidc_provider + ecr = module.shared-infrastructure.ecr } module "jit-provisioner" { diff --git a/infrastructure/sandbox/readme.md b/infrastructure/sandbox/readme.md index c2b28e7639..6b0512183d 100644 --- a/infrastructure/sandbox/readme.md +++ b/infrastructure/sandbox/readme.md @@ -31,11 +31,3 @@ for i in $((aws dynamodb scan --table-name sandbox-prod-lifecycle | jq -r '.Item ```bash for i in $(aws dynamodb scan --table-name sandbox-prod-lifecycle | jq -r '.Items[] | select(.State.S == "provisioned") | .ID.S'); do helm uninstall $i; aws dynamodb delete-item --table-name sandbox-prod-lifecycle --key "{\"ID\": {\"S\": \"${i}\"}}"; done ``` - -### TODOs -1. JITProvisioner needs to return proper errors -1. Create and use a different kms key for installers -1. Sane scale levels for prod -1. Allow for parallel spinup of sandbox instances (preprovisioner) -1. https://redis.io/commands/flushdb/ during the teardown process -1. name state machines something random and track the new name in dynamodb