From d006b8ad2c175e1f1ee4705b7b17aef77b765517 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Mon, 7 Sep 2026 14:00:19 +1000 Subject: [PATCH 01/36] feat: grant wafv2:Get*, ec2:GetEbsEncryptionByDefault and the missing Kubernetes list kinds The AWS collectors and the managed Kubernetes scanner read resources the connector never granted, so those reads fail closed or record a permission gap. - Add `wafv2:Get*` and `ec2:GetEbsEncryptionByDefault` to all three AWS templates (CloudFormation, terraform, terraform-v6), keeping the Allow sets byte-identical. - Reconcile the ClusterRole in both k8s-resources terraform modules and the Helm chart to the exact kinds the collector Lists: core configmaps, secrets, resourcequotas, limitranges; discovery.k8s.io endpointslices; all four admissionregistration.k8s.io kinds; storage.k8s.io csidrivers; node.k8s.io runtimeclasses; scheduling.k8s.io priorityclasses; flowcontrol.apiserver.k8s.io flowschemas; networking.k8s.io ingressclasses; certificates.k8s.io certificatesigningrequests. - Drop core `endpoints`, which every manifest granted and the collector never lists (it reads discovery.k8s.io/endpointslices instead). Co-Authored-By: Claude Fable 5.1 --- .../nullify-cloudformation-template.json | 2 + .../modules/k8s-resources/main.tf | 82 +++++++++++++++---- .../modules/nullify-aws-integration/data.tf | 2 + .../terraform/modules/k8s-resources/main.tf | 82 +++++++++++++++---- .../modules/nullify-aws-integration/data.tf | 2 + .../templates/clusterrole.yaml | 33 +++++++- 6 files changed, 166 insertions(+), 37 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 0d7d8f8..8605d66 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -511,6 +511,7 @@ "lambda:List*", "lambda:Get*", "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ], @@ -744,6 +745,7 @@ "waf:List*", "wafv2:CheckCapacity", "wafv2:Describe*", + "wafv2:Get*", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 238b300..342fb61 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -40,68 +40,116 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { resources = [ "pods", "services", - "endpoints", "namespaces", "nodes", + "serviceaccounts", + "configmaps", + "secrets", + "resourcequotas", + "limitranges", "persistentvolumes", "persistentvolumeclaims", - "serviceaccounts", ] verbs = ["get", "list"] } - rule { - api_groups = ["networking.k8s.io"] - resources = ["ingresses", "networkpolicies"] - verbs = ["get", "list"] - } - rule { api_groups = ["apps"] resources = [ "deployments", "replicasets", "statefulsets", - "daemonsets" + "daemonsets", + ] + verbs = ["get", "list"] + } + + rule { + api_groups = ["batch"] + resources = ["jobs", "cronjobs"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["autoscaling"] + resources = ["horizontalpodautoscalers"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["policy"] + resources = ["poddisruptionbudgets"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["networking.k8s.io"] + resources = [ + "ingresses", + "ingressclasses", + "networkpolicies", ] verbs = ["get", "list"] } + rule { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + verbs = ["get", "list"] + } + rule { api_groups = ["rbac.authorization.k8s.io"] resources = [ "roles", "rolebindings", "clusterroles", - "clusterrolebindings" + "clusterrolebindings", ] verbs = ["get", "list"] } rule { api_groups = ["storage.k8s.io"] - resources = ["storageclasses"] + resources = ["storageclasses", "csidrivers"] verbs = ["get", "list"] } rule { - api_groups = ["batch"] - resources = ["jobs", "cronjobs"] + api_groups = ["node.k8s.io"] + resources = ["runtimeclasses"] verbs = ["get", "list"] } rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] + api_groups = ["scheduling.k8s.io"] + resources = ["priorityclasses"] verbs = ["get", "list"] } rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] + api_groups = ["flowcontrol.apiserver.k8s.io"] + resources = ["flowschemas"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["certificates.k8s.io"] + resources = ["certificatesigningrequests"] verbs = ["get", "list"] } + rule { + api_groups = ["admissionregistration.k8s.io"] + resources = [ + "validatingwebhookconfigurations", + "mutatingwebhookconfigurations", + "validatingadmissionpolicies", + "validatingadmissionpolicybindings", + ] + verbs = ["get", "list"] + } + rule { api_groups = ["apiextensions.k8s.io"] resources = ["customresourcedefinitions"] diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 46cbf2a..fb16a72 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -269,6 +269,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "lambda:Get*", "lambda:List*", "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ] @@ -490,6 +491,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "waf:List*", "wafv2:CheckCapacity", "wafv2:Describe*", + "wafv2:Get*", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 8f77aeb..df9b9d4 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -40,68 +40,116 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { resources = [ "pods", "services", - "endpoints", "namespaces", "nodes", + "serviceaccounts", + "configmaps", + "secrets", + "resourcequotas", + "limitranges", "persistentvolumes", "persistentvolumeclaims", - "serviceaccounts", ] verbs = ["get", "list"] } - rule { - api_groups = ["networking.k8s.io"] - resources = ["ingresses", "networkpolicies"] - verbs = ["get", "list"] - } - rule { api_groups = ["apps"] resources = [ "deployments", "replicasets", "statefulsets", - "daemonsets" + "daemonsets", + ] + verbs = ["get", "list"] + } + + rule { + api_groups = ["batch"] + resources = ["jobs", "cronjobs"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["autoscaling"] + resources = ["horizontalpodautoscalers"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["policy"] + resources = ["poddisruptionbudgets"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["networking.k8s.io"] + resources = [ + "ingresses", + "ingressclasses", + "networkpolicies", ] verbs = ["get", "list"] } + rule { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + verbs = ["get", "list"] + } + rule { api_groups = ["rbac.authorization.k8s.io"] resources = [ "roles", "rolebindings", "clusterroles", - "clusterrolebindings" + "clusterrolebindings", ] verbs = ["get", "list"] } rule { api_groups = ["storage.k8s.io"] - resources = ["storageclasses"] + resources = ["storageclasses", "csidrivers"] verbs = ["get", "list"] } rule { - api_groups = ["batch"] - resources = ["jobs", "cronjobs"] + api_groups = ["node.k8s.io"] + resources = ["runtimeclasses"] verbs = ["get", "list"] } rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] + api_groups = ["scheduling.k8s.io"] + resources = ["priorityclasses"] verbs = ["get", "list"] } rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] + api_groups = ["flowcontrol.apiserver.k8s.io"] + resources = ["flowschemas"] + verbs = ["get", "list"] + } + + rule { + api_groups = ["certificates.k8s.io"] + resources = ["certificatesigningrequests"] verbs = ["get", "list"] } + rule { + api_groups = ["admissionregistration.k8s.io"] + resources = [ + "validatingwebhookconfigurations", + "mutatingwebhookconfigurations", + "validatingadmissionpolicies", + "validatingadmissionpolicybindings", + ] + verbs = ["get", "list"] + } + rule { api_groups = ["apiextensions.k8s.io"] resources = ["customresourcedefinitions"] diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 71a6285..b14613f 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -269,6 +269,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "lambda:Get*", "lambda:List*", "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ] @@ -490,6 +491,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "waf:List*", "wafv2:CheckCapacity", "wafv2:Describe*", + "wafv2:Get*", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index 30c2b99..39af3df 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -8,20 +8,47 @@ metadata: rules: # Access for collecting cluster information - apiGroups: [""] - resources: ["nodes", "namespaces", "pods", "services", "endpoints", "persistentvolumes", "persistentvolumeclaims", "serviceaccounts"] + resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "secrets", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list"] - apiGroups: ["apps"] - resources: ["deployments", "daemonsets", "statefulsets", "replicasets"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] verbs: ["get", "list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "list"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list"] - apiGroups: ["networking.k8s.io"] - resources: ["ingresses", "networkpolicies"] + resources: ["ingresses", "ingressclasses", "networkpolicies"] + verbs: ["get", "list"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] verbs: ["get", "list"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csidrivers"] + verbs: ["get", "list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get", "list"] + - apiGroups: ["flowcontrol.apiserver.k8s.io"] + resources: ["flowschemas"] + verbs: ["get", "list"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get", "list"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "list"] From 4932cdfc316db00eb4515dc5d03d0ec6b82b2b23 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Mon, 7 Sep 2026 22:57:28 +1000 Subject: [PATCH 02/36] fix(aws): grant the 45 read actions the CSPM collectors call ![Claude](https://img.shields.io/badge/Claude-Anthropic-blueviolet?logo=anthropic) The connector role never granted 45 of the AWS read actions the collectors issue. Each call returns AccessDenied, the collector tolerates it and leaves the field nil, and the CSPM rule that reads the field fires on every resource: kms:GetKeyRotationStatus on every customer-managed key, ecr:GetLifecyclePolicy on every repository, sns:GetTopicAttributes on every topic. sqs, codebuild, config and opensearch drop their whole inventory instead. - part1: athena:GetWorkGroup, backup:Get*, codebuild:BatchGet*, codebuild:List*, config:Describe*, dynamodb:GetResourcePolicy, ec2:GetManagedPrefixListEntries, ec2:GetSnapshotBlockPublicAccessState, ecr:GetLifecyclePolicy, ecr:GetRepositoryPolicy, es:Describe*, es:List*, glue:Get*, iam:GenerateCredentialReport, kms:GetKeyPolicy, kms:GetKeyRotationStatus - part2: s3:GetAccountPublicAccessBlock, s3:GetBucket*, s3:GetEncryptionConfiguration, s3:GetLifecycleConfiguration, s3:GetReplicationConfiguration, sns:GetTopicAttributes, sqs:GetQueueAttributes, ssm:Describe*, ssm:GetDocument - s3:GetBucketLocation dropped as redundant under the new s3:GetBucket* - terraform-v6/data.tf gains the trailing newline it was missing - applied identically to the CloudFormation template and both Terraform modules; part1 is at 5921/6144 chars and part2 at 5593/6144, so neither needs a Part3 The 45 were derived from every *.Input literal in context/internal/cloudintegrations/awsintegration/processors, mapped through the IAM prefix and action renames (opensearch -> es, s3 GetBucketEncryption -> s3:GetEncryptionConfiguration, API Gateway -> apigateway:GET). A Go test in that package now re-derives the list on every run and asserts this template grants it, which is the third source check_cloud_connector_iam_drift.py cannot supply: it compares the three templates to each other, so a grant missing from all three was green. --- .../nullify-cloudformation-template.json | 26 ++++++++++++++++- .../modules/nullify-aws-integration/data.tf | 28 +++++++++++++++++-- .../modules/nullify-aws-integration/data.tf | 26 ++++++++++++++++- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 8605d66..39cfa8d 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -372,6 +372,7 @@ "artifact:ListAgreements", "artifact:ListCustomerAgreements", "artifact:ListReports", + "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", "auditmanager:ListTagsForResource", @@ -380,6 +381,7 @@ "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", + "backup:Get*", "backup:List*", "batch:Describe*", "batch:List*", @@ -410,7 +412,9 @@ "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", + "codebuild:BatchGet*", "codebuild:Describe*", + "codebuild:List*", "cognito-identity:Describe*", "cognito-identity:GetIdentityPoolAnalytics", "cognito-identity:GetIdentityPoolDailyAnalytics", @@ -438,6 +442,8 @@ "ecr-public:ListTagsForResource", "ecr:BatchCheck*", "ecr:Describe*", + "ecr:GetLifecyclePolicy", + "ecr:GetRepositoryPolicy", "ecr:List*", "ecs:Describe*", "ecs:List*", @@ -461,12 +467,16 @@ "emr-containers:List*", "elastictranscoder:List*", "elastictranscoder:Read*", + "es:Describe*", + "es:List*", "events:Describe*", "events:List*", "events:ListTagsForResource", "dynamodb:Describe*", + "dynamodb:GetResourcePolicy", "dynamodb:List*", "discovery:Describe*", + "config:Describe*", "config:SelectResourceConfig", "iam:SimulateCustomPolicy", "iam:SimulatePrincipalPolicy", @@ -493,9 +503,11 @@ "kinesis:Describe*", "kinesis:List*", "iam:Simulate*", + "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", "glue:Describe*", + "glue:Get*", "glue:List*", "kinesisanalytics:Describe*", "kinesisanalytics:Discover*", @@ -503,6 +515,8 @@ "kinesisvideo:Describe*", "kinesisvideo:List*", "kms:Describe*", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", "kms:List*", "lakeformation:Describe*", "lakeformation:Get*", @@ -512,6 +526,8 @@ "lambda:Get*", "ec2:Describe*", "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ], @@ -659,8 +675,12 @@ "s3-outposts:Get*", "s3-outposts:List*", "s3:Describe*", + "s3:GetAccountPublicAccessBlock", + "s3:GetBucket*", + "s3:GetEncryptionConfiguration", + "s3:GetLifecycleConfiguration", + "s3:GetReplicationConfiguration", "s3:List*", - "s3:GetBucketLocation", "sagemaker:Describe*", "sagemaker:List*", "scheduler:List*", @@ -702,11 +722,15 @@ "snowball:Describe*", "snowball:List*", "sns:Check*", + "sns:GetTopicAttributes", "sns:List*", + "sqs:GetQueueAttributes", "sqs:List*", "ssm-contacts:List*", "ssm-sap:List*", "ssm-quicksetup:List*", + "ssm:Describe*", + "ssm:GetDocument", "ssm:List*", "sso-directory:List*", "sso:List*", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index fb16a72..6316220 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -130,6 +130,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "artifact:ListAgreements", "artifact:ListCustomerAgreements", "artifact:ListReports", + "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", "auditmanager:ListTagsForResource", @@ -138,6 +139,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", + "backup:Get*", "backup:List*", "batch:Describe*", "batch:List*", @@ -168,7 +170,9 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", + "codebuild:BatchGet*", "codebuild:Describe*", + "codebuild:List*", "cognito-identity:Describe*", "cognito-identity:GetIdentityPoolAnalytics", "cognito-identity:GetIdentityPoolDailyAnalytics", @@ -182,11 +186,13 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cognito-sync:Describe*", "cognito-sync:List*", "cognito-sync:QueryRecords", + "config:Describe*", "config:SelectResourceConfig", "devops-guru:List*", "discovery:Describe*", "discovery:List*", "dynamodb:Describe*", + "dynamodb:GetResourcePolicy", "dynamodb:List*", "ecr-public:BatchCheckLayerAvailability", "ecr-public:DescribeImages", @@ -198,6 +204,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ecr-public:ListTagsForResource", "ecr:BatchCheck*", "ecr:Describe*", + "ecr:GetLifecyclePolicy", + "ecr:GetRepositoryPolicy", "ecr:List*", "ecs:Describe*", "ecs:List*", @@ -224,8 +232,12 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "events:ListTagsForResource", "emr-containers:Describe*", "emr-containers:List*", + "es:Describe*", + "es:List*", "glue:Describe*", + "glue:Get*", "glue:List*", + "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", "iam:Simulate*", @@ -261,6 +273,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "kinesisvideo:Describe*", "kinesisvideo:List*", "kms:Describe*", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", "kms:List*", "lakeformation:Describe*", "lakeformation:Get*", @@ -270,6 +284,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "lambda:List*", "ec2:Describe*", "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ] @@ -405,8 +421,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "s3-outposts:Get*", "s3-outposts:List*", "s3:Describe*", + "s3:GetAccountPublicAccessBlock", + "s3:GetBucket*", + "s3:GetEncryptionConfiguration", + "s3:GetLifecycleConfiguration", + "s3:GetReplicationConfiguration", "s3:List*", - "s3:GetBucketLocation", "sagemaker:Describe*", "sagemaker:List*", "scheduler:List*", @@ -448,11 +468,15 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "snowball:Describe*", "snowball:List*", "sns:Check*", + "sns:GetTopicAttributes", "sns:List*", + "sqs:GetQueueAttributes", "sqs:List*", "ssm-contacts:List*", "ssm-sap:List*", "ssm-quicksetup:List*", + "ssm:Describe*", + "ssm:GetDocument", "ssm:List*", "sso-directory:List*", "sso:List*", @@ -596,4 +620,4 @@ data "aws_iam_policy_document" "deny_actions_policy" { ] resources = ["*"] } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index b14613f..6316220 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -130,6 +130,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "artifact:ListAgreements", "artifact:ListCustomerAgreements", "artifact:ListReports", + "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", "auditmanager:ListTagsForResource", @@ -138,6 +139,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", + "backup:Get*", "backup:List*", "batch:Describe*", "batch:List*", @@ -168,7 +170,9 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", + "codebuild:BatchGet*", "codebuild:Describe*", + "codebuild:List*", "cognito-identity:Describe*", "cognito-identity:GetIdentityPoolAnalytics", "cognito-identity:GetIdentityPoolDailyAnalytics", @@ -182,11 +186,13 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cognito-sync:Describe*", "cognito-sync:List*", "cognito-sync:QueryRecords", + "config:Describe*", "config:SelectResourceConfig", "devops-guru:List*", "discovery:Describe*", "discovery:List*", "dynamodb:Describe*", + "dynamodb:GetResourcePolicy", "dynamodb:List*", "ecr-public:BatchCheckLayerAvailability", "ecr-public:DescribeImages", @@ -198,6 +204,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ecr-public:ListTagsForResource", "ecr:BatchCheck*", "ecr:Describe*", + "ecr:GetLifecyclePolicy", + "ecr:GetRepositoryPolicy", "ecr:List*", "ecs:Describe*", "ecs:List*", @@ -224,8 +232,12 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "events:ListTagsForResource", "emr-containers:Describe*", "emr-containers:List*", + "es:Describe*", + "es:List*", "glue:Describe*", + "glue:Get*", "glue:List*", + "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", "iam:Simulate*", @@ -261,6 +273,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "kinesisvideo:Describe*", "kinesisvideo:List*", "kms:Describe*", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", "kms:List*", "lakeformation:Describe*", "lakeformation:Get*", @@ -270,6 +284,8 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "lambda:List*", "ec2:Describe*", "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", "ec2:GetTransitGateway*", "ec2:SearchTransitGateway*" ] @@ -405,8 +421,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "s3-outposts:Get*", "s3-outposts:List*", "s3:Describe*", + "s3:GetAccountPublicAccessBlock", + "s3:GetBucket*", + "s3:GetEncryptionConfiguration", + "s3:GetLifecycleConfiguration", + "s3:GetReplicationConfiguration", "s3:List*", - "s3:GetBucketLocation", "sagemaker:Describe*", "sagemaker:List*", "scheduler:List*", @@ -448,11 +468,15 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "snowball:Describe*", "snowball:List*", "sns:Check*", + "sns:GetTopicAttributes", "sns:List*", + "sqs:GetQueueAttributes", "sqs:List*", "ssm-contacts:List*", "ssm-sap:List*", "ssm-quicksetup:List*", + "ssm:Describe*", + "ssm:GetDocument", "ssm:List*", "sso-directory:List*", "sso:List*", From 3a1a4faf47f1c46fd296646052e06d40f99aa53d Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Mon, 7 Sep 2026 23:37:29 +1000 Subject: [PATCH 03/36] fix(aws): scope the connector role to configuration reads only Four wildcard grants reached customer data, not just cloud configuration: wafv2:Get* covers GetSampledRequests (live HTTP requests with Cookie and Authorization headers), glue:Get* covers GetConnection (JDBC PASSWORD) and GetTable*/GetPartitions, codebuild:BatchGet* and backup:Get* both expand well past the single call each collector makes. - Replace those four with the nine actions the collectors actually call. - Deny 55 data-plane actions the Allow wildcards still reached: raw object and message content (s3-outposts:GetObject, logs:StartLiveTail, states:DescribeExecution, cognito-sync:QueryRecords, aps:QueryMetrics, rum:GetAppMonitorData, ssm:ListCommandInvocations, support:DescribeCases), cleartext credentials (lightsail:GetRelationalDatabaseMasterUserPassword, storagegateway:DescribeChapCredentials), end-user PII (cognito-idp:ListUsers and ListUsersInGroup), and the credential oracles transfer:TestIdentityProvider and verifiedpermissions:IsAuthorizedWithToken. - Delete the 12 Allow entries the new Deny statement shadowed, rather than leaving the same action in both lists. - Delete 7 Allow entries already subsumed by a sibling wildcard. - Bump the Helm chart to 0.3.0 so tenants can actually pull the new ClusterRole; helm-release.yml keys the published version off Chart.yaml, so the RBAC added in the previous commit was unreachable at 0.2.0. Verified against the collectors on monorepo ea4b7c0: 240/241 AWS calls remain covered (the one exception, sts:AssumeRole, is test-only and is the role's trust policy, not a grant). No collector calls any denied action. Real IAM-counted headroom on readonly-part1 goes 223 -> 326 chars; the drift gate under-reports by 15 chars per statement because document_size() omits "Condition". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0126R5BKvayttBu8nKmFZBVR --- .../nullify-cloudformation-template.json | 96 +++++++++++++------ .../modules/nullify-aws-integration/data.tf | 96 +++++++++++++------ .../modules/nullify-aws-integration/data.tf | 96 +++++++++++++------ helm-charts/nullify-k8s-collector/Chart.yaml | 2 +- 4 files changed, 202 insertions(+), 88 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 39cfa8d..8be22e3 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -165,7 +165,62 @@ "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", - "logs:DeleteLogStream" + "logs:DeleteLogStream", + "athena:GetQueryResults", + "cloudtrail:GetQueryResults", + "cognito-idp:AdminListUserAuthEvents", + "cognito-idp:ListUsers", + "dynamodb:BatchGetItem", + "dynamodb:GetItem", + "dynamodb:Query", + "dynamodb:Scan", + "es:ESHttp*", + "glue:GetConnection", + "glue:GetConnections", + "glue:GetPartition*", + "glue:GetTable*", + "kinesis:GetRecords", + "kinesis:GetShardIterator", + "lakeformation:GetDataAccess", + "lakeformation:GetTemporaryGlue*", + "lightsail:GetInstanceAccessDetails", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:StartQuery", + "rds-data:*", + "rekognition:Detect*", + "sdb:Select*", + "sqs:ReceiveMessage", + "wafv2:GetRateBasedStatementManagedKeys", + "wafv2:GetSampledRequests", + "workmail:Search*", + "clouddirectory:BatchRead", + "cloudfront-keyvaluestore:GetKey", + "cloudfront-keyvaluestore:ListKeys", + "cognito-idp:ListUsersInGroup", + "cognito-sync:ListRecords", + "cognito-sync:QueryRecords", + "dynamodb:PartiQLSelect", + "kinesis:SubscribeToShard", + "lightsail:GetRelationalDatabaseMasterUserPassword", + "logs:StartLiveTail", + "medialive:GetInputDeviceThumbnail", + "pi:DescribeDimensionKeys", + "rum:GetAppMonitorData", + "aps:QueryMetrics", + "s3-outposts:GetObject", + "s3-outposts:GetObjectVersion", + "ssm:ListCommandInvocations", + "states:DescribeExecution", + "states:GetExecutionHistory", + "storagegateway:DescribeChapCredentials", + "support:DescribeCases", + "support:DescribeCommunications", + "support:SearchForCases", + "transfer:TestIdentityProvider", + "verifiedpermissions:IsAuthorized", + "verifiedpermissions:IsAuthorizedWithToken", + "workmailmessageflow:GetRawMessageContent" ], "Resource": "*", "Condition": {} @@ -340,7 +395,6 @@ "application-autoscaling:Describe*", "application-autoscaling:ListTagsForResource", "application-signals:List*", - "application-signals:ListTagsForResource", "applicationinsights:List*", "appmesh:Describe*", "appmesh:List*", @@ -355,7 +409,6 @@ "appstream:Describe*", "appstream:List*", "appsync:List*", - "apptest:ListTagsForResource", "apptest:List*", "aps:DescribeAlertManagerDefinition", "aps:DescribeLoggingConfiguration", @@ -363,8 +416,6 @@ "aps:DescribeScraper", "aps:DescribeWorkspace", "aps:List*", - "aps:ListWorkspaces", - "aps:QueryMetrics", "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", "arc-zonal-shift:ListAutoshifts", "arc-zonal-shift:ListManagedResources", @@ -375,13 +426,13 @@ "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", - "auditmanager:ListTagsForResource", "auditmanager:ValidateAssessmentReportIntegrity", "autoscaling-plans:Describe*", "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", - "backup:Get*", + "backup:GetBackupPlan", + "backup:GetBackupVaultAccessPolicy", "backup:List*", "batch:Describe*", "batch:List*", @@ -389,7 +440,6 @@ "billingconductor:List*", "cloud9:Describe*", "cloud9:List*", - "clouddirectory:BatchRead", "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", @@ -412,7 +462,7 @@ "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", - "codebuild:BatchGet*", + "codebuild:BatchGetProjects", "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", @@ -427,7 +477,6 @@ "cognito-idp:List*", "cognito-sync:Describe*", "cognito-sync:List*", - "cognito-sync:QueryRecords", "devops-guru:List*", "discovery:List*", "identitystore:Describe*", @@ -471,15 +520,12 @@ "es:List*", "events:Describe*", "events:List*", - "events:ListTagsForResource", "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", "discovery:Describe*", "config:Describe*", "config:SelectResourceConfig", - "iam:SimulateCustomPolicy", - "iam:SimulatePrincipalPolicy", "iotfleetwise:List*", "iotroborunner:List*", "iotsitewise:Describe*", @@ -507,8 +553,12 @@ "iam:Get*", "iam:List*", "glue:Describe*", - "glue:Get*", - "glue:List*", + "glue:GetCrawlers", + "glue:GetDataCatalogEncryptionSettings", + "glue:GetDevEndpoints", + "glue:GetJobs", + "glue:GetSecurityConfigurations", +"glue:List*", "kinesisanalytics:Describe*", "kinesisanalytics:Discover*", "kinesisanalytics:List*", @@ -562,10 +612,7 @@ "license-manager:List*", "lightsail:Get*", "logs:Describe*", - "logs:FilterLogEvents", "logs:List*", - "logs:StartLiveTail", - "logs:StartQuery", "logs:StopLiveTail", "logs:StopQuery", "logs:TestMetricFilter", @@ -623,7 +670,6 @@ "outposts:List*", "personalize:Describe*", "personalize:List*", - "pi:DescribeDimensionKeys", "pi:ListAvailableResourceDimensions", "pi:ListAvailableResourceMetrics", "pipes:DescribePipe", @@ -642,7 +688,6 @@ "redshift:List*", "redshift:View*", "rekognition:Describe*", - "rekognition:Detect*", "rekognition:List*", "resource-explorer-2:List*", "resource-groups:List*", @@ -669,7 +714,6 @@ "route53resolver:Get*", "route53resolver:List*", "rum:GetAppMonitor", - "rum:GetAppMonitorData", "rum:ListAppMonitors", "s3-object-lambda:List*", "s3-outposts:Get*", @@ -689,7 +733,6 @@ "schemas:List*", "schemas:Search*", "sdb:List*", - "sdb:Select*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", "securityhub:Describe*", @@ -744,7 +787,6 @@ "sts:GetCallerIdentity", "sts:GetSessionToken", "support:Describe*", - "support:SearchForCases", "tag:Describe*", "tag:Get*", "tax:ListTaxRegistrations", @@ -753,23 +795,20 @@ "transcribe:List*", "transfer:Describe*", "transfer:List*", - "transfer:TestIdentityProvider", "translate:DescribeTextTranslationJob", "translate:ListParallelData", "translate:ListTerminologies", "translate:ListTextTranslationJobs", "trustedadvisor:Describe*", "trustedadvisor:List*", - "verifiedpermissions:IsAuthorized", - "verifiedpermissions:IsAuthorizedWithToken", "verifiedpermissions:List*", "vpc-lattice:Get*", "vpc-lattice:List*", "waf-regional:List*", "waf:List*", - "wafv2:CheckCapacity", +"wafv2:CheckCapacity", "wafv2:Describe*", - "wafv2:Get*", + "wafv2:GetWebACL", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", @@ -778,7 +817,6 @@ "workdocs:Describe*", "workmail:Describe*", "workmail:List*", - "workmail:Search*", "workspaces-web:GetBrowserSettings", "workspaces-web:GetIdentityProvider", "workspaces-web:GetNetworkSettings", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 6316220..9b8a902 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -98,7 +98,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "application-autoscaling:Describe*", "application-autoscaling:ListTagsForResource", "application-signals:List*", - "application-signals:ListTagsForResource", "applicationinsights:List*", "appmesh:Describe*", "appmesh:List*", @@ -113,7 +112,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "appstream:Describe*", "appstream:List*", "appsync:List*", - "apptest:ListTagsForResource", "apptest:List*", "aps:DescribeAlertManagerDefinition", "aps:DescribeLoggingConfiguration", @@ -121,8 +119,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "aps:DescribeScraper", "aps:DescribeWorkspace", "aps:List*", - "aps:ListWorkspaces", - "aps:QueryMetrics", "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", "arc-zonal-shift:ListAutoshifts", "arc-zonal-shift:ListManagedResources", @@ -133,13 +129,13 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", - "auditmanager:ListTagsForResource", "auditmanager:ValidateAssessmentReportIntegrity", "autoscaling-plans:Describe*", "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", - "backup:Get*", + "backup:GetBackupPlan", + "backup:GetBackupVaultAccessPolicy", "backup:List*", "batch:Describe*", "batch:List*", @@ -147,7 +143,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "billingconductor:List*", "cloud9:Describe*", "cloud9:List*", - "clouddirectory:BatchRead", "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", @@ -170,7 +165,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", - "codebuild:BatchGet*", + "codebuild:BatchGetProjects", "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", @@ -185,7 +180,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cognito-idp:List*", "cognito-sync:Describe*", "cognito-sync:List*", - "cognito-sync:QueryRecords", "config:Describe*", "config:SelectResourceConfig", "devops-guru:List*", @@ -229,20 +223,21 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "elastictranscoder:Read*", "events:Describe*", "events:List*", - "events:ListTagsForResource", "emr-containers:Describe*", "emr-containers:List*", "es:Describe*", "es:List*", "glue:Describe*", - "glue:Get*", - "glue:List*", + "glue:GetCrawlers", + "glue:GetDataCatalogEncryptionSettings", + "glue:GetDevEndpoints", + "glue:GetJobs", + "glue:GetSecurityConfigurations", +"glue:List*", "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", "iam:Simulate*", - "iam:SimulateCustomPolicy", - "iam:SimulatePrincipalPolicy", "identity-sync:ListSyncFilters", "identitystore:Describe*", "iotfleetwise:List*", @@ -308,10 +303,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "license-manager:List*", "lightsail:Get*", "logs:Describe*", - "logs:FilterLogEvents", "logs:List*", - "logs:StartLiveTail", - "logs:StartQuery", "logs:StopLiveTail", "logs:StopQuery", "logs:TestMetricFilter", @@ -369,7 +361,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "outposts:List*", "personalize:Describe*", "personalize:List*", - "pi:DescribeDimensionKeys", "pi:ListAvailableResourceDimensions", "pi:ListAvailableResourceMetrics", "pipes:DescribePipe", @@ -388,7 +379,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "redshift:List*", "redshift:View*", "rekognition:Describe*", - "rekognition:Detect*", "rekognition:List*", "resource-explorer-2:List*", "resource-groups:List*", @@ -415,7 +405,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "route53resolver:Get*", "route53resolver:List*", "rum:GetAppMonitor", - "rum:GetAppMonitorData", "rum:ListAppMonitors", "s3-object-lambda:List*", "s3-outposts:Get*", @@ -435,7 +424,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "schemas:List*", "schemas:Search*", "sdb:List*", - "sdb:Select*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", "securityhub:Describe*", @@ -490,7 +478,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "sts:GetCallerIdentity", "sts:GetSessionToken", "support:Describe*", - "support:SearchForCases", "tag:Describe*", "tag:Get*", "tax:ListTaxRegistrations", @@ -499,23 +486,20 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "transcribe:List*", "transfer:Describe*", "transfer:List*", - "transfer:TestIdentityProvider", "translate:DescribeTextTranslationJob", "translate:ListParallelData", "translate:ListTerminologies", "translate:ListTextTranslationJobs", "trustedadvisor:Describe*", "trustedadvisor:List*", - "verifiedpermissions:IsAuthorized", - "verifiedpermissions:IsAuthorizedWithToken", "verifiedpermissions:List*", "vpc-lattice:Get*", "vpc-lattice:List*", "waf-regional:List*", "waf:List*", - "wafv2:CheckCapacity", +"wafv2:CheckCapacity", "wafv2:Describe*", - "wafv2:Get*", + "wafv2:GetWebACL", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", @@ -524,7 +508,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "workdocs:Describe*", "workmail:Describe*", "workmail:List*", - "workmail:Search*", "workspaces-web:GetBrowserSettings", "workspaces-web:GetIdentityProvider", "workspaces-web:GetNetworkSettings", @@ -616,7 +599,62 @@ data "aws_iam_policy_document" "deny_actions_policy" { "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", - "logs:DeleteLogStream" + "logs:DeleteLogStream", + "athena:GetQueryResults", + "cloudtrail:GetQueryResults", + "cognito-idp:AdminListUserAuthEvents", + "cognito-idp:ListUsers", + "dynamodb:BatchGetItem", + "dynamodb:GetItem", + "dynamodb:Query", + "dynamodb:Scan", + "es:ESHttp*", + "glue:GetConnection", + "glue:GetConnections", + "glue:GetPartition*", + "glue:GetTable*", + "kinesis:GetRecords", + "kinesis:GetShardIterator", + "lakeformation:GetDataAccess", + "lakeformation:GetTemporaryGlue*", + "lightsail:GetInstanceAccessDetails", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:StartQuery", + "rds-data:*", + "rekognition:Detect*", + "sdb:Select*", + "sqs:ReceiveMessage", + "wafv2:GetRateBasedStatementManagedKeys", + "wafv2:GetSampledRequests", + "workmail:Search*", + "clouddirectory:BatchRead", + "cloudfront-keyvaluestore:GetKey", + "cloudfront-keyvaluestore:ListKeys", + "cognito-idp:ListUsersInGroup", + "cognito-sync:ListRecords", + "cognito-sync:QueryRecords", + "dynamodb:PartiQLSelect", + "kinesis:SubscribeToShard", + "lightsail:GetRelationalDatabaseMasterUserPassword", + "logs:StartLiveTail", + "medialive:GetInputDeviceThumbnail", + "pi:DescribeDimensionKeys", + "rum:GetAppMonitorData", + "aps:QueryMetrics", + "s3-outposts:GetObject", + "s3-outposts:GetObjectVersion", + "ssm:ListCommandInvocations", + "states:DescribeExecution", + "states:GetExecutionHistory", + "storagegateway:DescribeChapCredentials", + "support:DescribeCases", + "support:DescribeCommunications", + "support:SearchForCases", + "transfer:TestIdentityProvider", + "verifiedpermissions:IsAuthorized", + "verifiedpermissions:IsAuthorizedWithToken", + "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 6316220..9b8a902 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -98,7 +98,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "application-autoscaling:Describe*", "application-autoscaling:ListTagsForResource", "application-signals:List*", - "application-signals:ListTagsForResource", "applicationinsights:List*", "appmesh:Describe*", "appmesh:List*", @@ -113,7 +112,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "appstream:Describe*", "appstream:List*", "appsync:List*", - "apptest:ListTagsForResource", "apptest:List*", "aps:DescribeAlertManagerDefinition", "aps:DescribeLoggingConfiguration", @@ -121,8 +119,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "aps:DescribeScraper", "aps:DescribeWorkspace", "aps:List*", - "aps:ListWorkspaces", - "aps:QueryMetrics", "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", "arc-zonal-shift:ListAutoshifts", "arc-zonal-shift:ListManagedResources", @@ -133,13 +129,13 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "athena:GetWorkGroup", "athena:List*", "auditmanager:List*", - "auditmanager:ListTagsForResource", "auditmanager:ValidateAssessmentReportIntegrity", "autoscaling-plans:Describe*", "autoscaling:Describe*", "backup-gateway:List*", "backup:Describe*", - "backup:Get*", + "backup:GetBackupPlan", + "backup:GetBackupVaultAccessPolicy", "backup:List*", "batch:Describe*", "batch:List*", @@ -147,7 +143,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "billingconductor:List*", "cloud9:Describe*", "cloud9:List*", - "clouddirectory:BatchRead", "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", @@ -170,7 +165,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudwatch:Describe*", "cloudwatch:GenerateQuery", "codeartifact:List*", - "codebuild:BatchGet*", + "codebuild:BatchGetProjects", "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", @@ -185,7 +180,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cognito-idp:List*", "cognito-sync:Describe*", "cognito-sync:List*", - "cognito-sync:QueryRecords", "config:Describe*", "config:SelectResourceConfig", "devops-guru:List*", @@ -229,20 +223,21 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "elastictranscoder:Read*", "events:Describe*", "events:List*", - "events:ListTagsForResource", "emr-containers:Describe*", "emr-containers:List*", "es:Describe*", "es:List*", "glue:Describe*", - "glue:Get*", - "glue:List*", + "glue:GetCrawlers", + "glue:GetDataCatalogEncryptionSettings", + "glue:GetDevEndpoints", + "glue:GetJobs", + "glue:GetSecurityConfigurations", +"glue:List*", "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", "iam:Simulate*", - "iam:SimulateCustomPolicy", - "iam:SimulatePrincipalPolicy", "identity-sync:ListSyncFilters", "identitystore:Describe*", "iotfleetwise:List*", @@ -308,10 +303,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "license-manager:List*", "lightsail:Get*", "logs:Describe*", - "logs:FilterLogEvents", "logs:List*", - "logs:StartLiveTail", - "logs:StartQuery", "logs:StopLiveTail", "logs:StopQuery", "logs:TestMetricFilter", @@ -369,7 +361,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "outposts:List*", "personalize:Describe*", "personalize:List*", - "pi:DescribeDimensionKeys", "pi:ListAvailableResourceDimensions", "pi:ListAvailableResourceMetrics", "pipes:DescribePipe", @@ -388,7 +379,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "redshift:List*", "redshift:View*", "rekognition:Describe*", - "rekognition:Detect*", "rekognition:List*", "resource-explorer-2:List*", "resource-groups:List*", @@ -415,7 +405,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "route53resolver:Get*", "route53resolver:List*", "rum:GetAppMonitor", - "rum:GetAppMonitorData", "rum:ListAppMonitors", "s3-object-lambda:List*", "s3-outposts:Get*", @@ -435,7 +424,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "schemas:List*", "schemas:Search*", "sdb:List*", - "sdb:Select*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", "securityhub:Describe*", @@ -490,7 +478,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "sts:GetCallerIdentity", "sts:GetSessionToken", "support:Describe*", - "support:SearchForCases", "tag:Describe*", "tag:Get*", "tax:ListTaxRegistrations", @@ -499,23 +486,20 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "transcribe:List*", "transfer:Describe*", "transfer:List*", - "transfer:TestIdentityProvider", "translate:DescribeTextTranslationJob", "translate:ListParallelData", "translate:ListTerminologies", "translate:ListTextTranslationJobs", "trustedadvisor:Describe*", "trustedadvisor:List*", - "verifiedpermissions:IsAuthorized", - "verifiedpermissions:IsAuthorizedWithToken", "verifiedpermissions:List*", "vpc-lattice:Get*", "vpc-lattice:List*", "waf-regional:List*", "waf:List*", - "wafv2:CheckCapacity", +"wafv2:CheckCapacity", "wafv2:Describe*", - "wafv2:Get*", + "wafv2:GetWebACL", "wafv2:List*", "wellarchitected:ExportLens", "wellarchitected:Get*", @@ -524,7 +508,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "workdocs:Describe*", "workmail:Describe*", "workmail:List*", - "workmail:Search*", "workspaces-web:GetBrowserSettings", "workspaces-web:GetIdentityProvider", "workspaces-web:GetNetworkSettings", @@ -616,7 +599,62 @@ data "aws_iam_policy_document" "deny_actions_policy" { "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", - "logs:DeleteLogStream" + "logs:DeleteLogStream", + "athena:GetQueryResults", + "cloudtrail:GetQueryResults", + "cognito-idp:AdminListUserAuthEvents", + "cognito-idp:ListUsers", + "dynamodb:BatchGetItem", + "dynamodb:GetItem", + "dynamodb:Query", + "dynamodb:Scan", + "es:ESHttp*", + "glue:GetConnection", + "glue:GetConnections", + "glue:GetPartition*", + "glue:GetTable*", + "kinesis:GetRecords", + "kinesis:GetShardIterator", + "lakeformation:GetDataAccess", + "lakeformation:GetTemporaryGlue*", + "lightsail:GetInstanceAccessDetails", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:StartQuery", + "rds-data:*", + "rekognition:Detect*", + "sdb:Select*", + "sqs:ReceiveMessage", + "wafv2:GetRateBasedStatementManagedKeys", + "wafv2:GetSampledRequests", + "workmail:Search*", + "clouddirectory:BatchRead", + "cloudfront-keyvaluestore:GetKey", + "cloudfront-keyvaluestore:ListKeys", + "cognito-idp:ListUsersInGroup", + "cognito-sync:ListRecords", + "cognito-sync:QueryRecords", + "dynamodb:PartiQLSelect", + "kinesis:SubscribeToShard", + "lightsail:GetRelationalDatabaseMasterUserPassword", + "logs:StartLiveTail", + "medialive:GetInputDeviceThumbnail", + "pi:DescribeDimensionKeys", + "rum:GetAppMonitorData", + "aps:QueryMetrics", + "s3-outposts:GetObject", + "s3-outposts:GetObjectVersion", + "ssm:ListCommandInvocations", + "states:DescribeExecution", + "states:GetExecutionHistory", + "storagegateway:DescribeChapCredentials", + "support:DescribeCases", + "support:DescribeCommunications", + "support:SearchForCases", + "transfer:TestIdentityProvider", + "verifiedpermissions:IsAuthorized", + "verifiedpermissions:IsAuthorizedWithToken", + "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] } diff --git a/helm-charts/nullify-k8s-collector/Chart.yaml b/helm-charts/nullify-k8s-collector/Chart.yaml index 509867f..566a7ab 100644 --- a/helm-charts/nullify-k8s-collector/Chart.yaml +++ b/helm-charts/nullify-k8s-collector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nullify-k8s-collector description: A Helm chart for Nullify's Kubernetes information collector for cloud security scanning type: application -version: 0.2.0 +version: 0.3.0 maintainers: - name: Nullify url: https://nullify.ai From 0179faf381e842b40628cf716091a3f42d2acee2 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 8 Sep 2026 03:59:31 +1000 Subject: [PATCH 04/36] fix(aws): deny the data reads a second audit found still reachable An adversarial pass over the remaining Allow wildcards found reach the first narrowing missed. None of it is called by any collector. - Deny 22 actions still reachable through a wildcard: Lake Formation row data (GetWorkUnitResults, GetWorkUnits), live stream samples (kinesisanalytics:DiscoverInputSchema), Lightsail container application logs, support case attachment bytes, a plaintext OAuth client secret (cognito-idp:DescribeUserPoolClient), usable AppSync API keys (appsync:ListApiKeys), notification event bodies, and the WorkMail / IdentityStore / AppStream / Rekognition / SSO / WorkSpaces user directories. - Drop 11 Allow entries instead of shadowing them: amplify:GetWebhook (the URL embeds a bearer token and POSTing it starts a build, a write the Deny statement cannot reach), account:GetContactInformation / GetAlternateContact / GetPrimaryEmail and one:ListUsers (named-person PII), cloudtrail:LookupEvents (per-event username and source IP), and the four write actions hiding behind read-sounding names -- cloudformation:Detect* (AWS classes it Write; it schedules drift detection and mutates stack state), cloudwatch:GenerateQuery, logs:StopLiveTail, logs:StopQuery. - Drop glue:Describe*, which matches no real Glue action, and one of the two copies of notifications:List*. Verified with a from-scratch IAM evaluator: 232/232 collector actions still allowed, every action above resolves to explicitDeny or implicitDeny, and no action appears in both Allow and Deny. readonly-part1 headroom 223 -> 545. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0126R5BKvayttBu8nKmFZBVR --- .../nullify-cloudformation-template.json | 36 +++++++++++-------- .../modules/nullify-aws-integration/data.tf | 36 +++++++++++-------- .../modules/nullify-aws-integration/data.tf | 36 +++++++++++-------- 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 8be22e3..43f7827 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -220,6 +220,28 @@ "transfer:TestIdentityProvider", "verifiedpermissions:IsAuthorized", "verifiedpermissions:IsAuthorizedWithToken", + "appstream:DescribeSessions", + "appstream:DescribeUsers", + "appsync:ListApiKeys", + "cognito-identity:LookupDeveloperIdentity", + "cognito-idp:AdminListDevices", + "cognito-idp:DescribeUserPoolClient", + "identitystore:DescribeUser", + "kinesisanalytics:DiscoverInputSchema", + "lakeformation:GetWorkUnitResults", + "lakeformation:GetWorkUnits", + "lightsail:GetContainerLog", + "notifications:GetManagedNotificationEvent", + "notifications:GetNotificationEvent", + "rekognition:ListFaces", + "rekognition:ListUsers", + "sso:ListAccountAssignments", + "sso:ListAccountAssignmentsForPrincipal", + "support:DescribeAttachment", + "workmail:DescribeUser", + "workmail:ListAliases", + "workmail:ListUsers", + "workspaces:DescribeWorkspaces", "workmailmessageflow:GetRawMessageContent" ], "Resource": "*", @@ -365,9 +387,6 @@ "access-analyzer:List*", "access-analyzer:ValidatePolicy", "account:GetAccountInformation", - "account:GetAlternateContact", - "account:GetContactInformation", - "account:GetPrimaryEmail", "account:GetRegionOptStatus", "account:ListRegions", "acm-pca:Describe*", @@ -378,7 +397,6 @@ "airflow:List*", "amplify:GetBranch", "amplify:GetDomainAssociation", - "amplify:GetWebhook", "amplify:List*", "aoss:BatchGetLifecyclePolicy", "aoss:BatchGetVpcEndpoint", @@ -443,7 +461,6 @@ "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", - "cloudformation:Detect*", "cloudformation:Estimate*", "cloudformation:List*", "cloudformation:ValidateTemplate", @@ -458,9 +475,7 @@ "cloudtrail:Describe*", "cloudtrail:Get*", "cloudtrail:List*", - "cloudtrail:LookupEvents", "cloudwatch:Describe*", - "cloudwatch:GenerateQuery", "codeartifact:List*", "codebuild:BatchGetProjects", "codebuild:Describe*", @@ -552,7 +567,6 @@ "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", - "glue:Describe*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", @@ -613,8 +627,6 @@ "lightsail:Get*", "logs:Describe*", "logs:List*", - "logs:StopLiveTail", - "logs:StopQuery", "logs:TestMetricFilter", "machinelearning:Describe*", "managedblockchain:GetMember", @@ -655,13 +667,9 @@ "nimble:List*", "notifications-contacts:List*", "notifications:Get*", - "notifications:List*", - "notifications:GetManagedNotificationEvent", "notifications:GetNotificationConfiguration", "notifications:GetNotificationsAccessForOrganization", - "notifications:GetNotificationEvent", "notifications:List*", - "one:ListUsers", "opsworks-cm:Describe*", "opsworks-cm:List*", "opsworks:Describe*", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 9b8a902..3b434d2 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -68,9 +68,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "access-analyzer:List*", "access-analyzer:ValidatePolicy", "account:GetAccountInformation", - "account:GetAlternateContact", - "account:GetContactInformation", - "account:GetPrimaryEmail", "account:GetRegionOptStatus", "account:ListRegions", "acm-pca:Describe*", @@ -81,7 +78,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "airflow:List*", "amplify:GetBranch", "amplify:GetDomainAssociation", - "amplify:GetWebhook", "amplify:List*", "aoss:BatchGetLifecyclePolicy", "aoss:BatchGetVpcEndpoint", @@ -146,7 +142,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", - "cloudformation:Detect*", "cloudformation:Estimate*", "cloudformation:List*", "cloudformation:ValidateTemplate", @@ -161,9 +156,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudtrail:Describe*", "cloudtrail:Get*", "cloudtrail:List*", - "cloudtrail:LookupEvents", "cloudwatch:Describe*", - "cloudwatch:GenerateQuery", "codeartifact:List*", "codebuild:BatchGetProjects", "codebuild:Describe*", @@ -227,7 +220,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "emr-containers:List*", "es:Describe*", "es:List*", - "glue:Describe*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", @@ -304,8 +296,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "lightsail:Get*", "logs:Describe*", "logs:List*", - "logs:StopLiveTail", - "logs:StopQuery", "logs:TestMetricFilter", "machinelearning:Describe*", "managedblockchain:GetMember", @@ -346,13 +336,9 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "nimble:List*", "notifications-contacts:List*", "notifications:Get*", - "notifications:List*", - "notifications:GetManagedNotificationEvent", "notifications:GetNotificationConfiguration", "notifications:GetNotificationsAccessForOrganization", - "notifications:GetNotificationEvent", "notifications:List*", - "one:ListUsers", "opsworks-cm:Describe*", "opsworks-cm:List*", "opsworks:Describe*", @@ -654,6 +640,28 @@ data "aws_iam_policy_document" "deny_actions_policy" { "transfer:TestIdentityProvider", "verifiedpermissions:IsAuthorized", "verifiedpermissions:IsAuthorizedWithToken", + "appstream:DescribeSessions", + "appstream:DescribeUsers", + "appsync:ListApiKeys", + "cognito-identity:LookupDeveloperIdentity", + "cognito-idp:AdminListDevices", + "cognito-idp:DescribeUserPoolClient", + "identitystore:DescribeUser", + "kinesisanalytics:DiscoverInputSchema", + "lakeformation:GetWorkUnitResults", + "lakeformation:GetWorkUnits", + "lightsail:GetContainerLog", + "notifications:GetManagedNotificationEvent", + "notifications:GetNotificationEvent", + "rekognition:ListFaces", + "rekognition:ListUsers", + "sso:ListAccountAssignments", + "sso:ListAccountAssignmentsForPrincipal", + "support:DescribeAttachment", + "workmail:DescribeUser", + "workmail:ListAliases", + "workmail:ListUsers", + "workspaces:DescribeWorkspaces", "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 9b8a902..3b434d2 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -68,9 +68,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "access-analyzer:List*", "access-analyzer:ValidatePolicy", "account:GetAccountInformation", - "account:GetAlternateContact", - "account:GetContactInformation", - "account:GetPrimaryEmail", "account:GetRegionOptStatus", "account:ListRegions", "acm-pca:Describe*", @@ -81,7 +78,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "airflow:List*", "amplify:GetBranch", "amplify:GetDomainAssociation", - "amplify:GetWebhook", "amplify:List*", "aoss:BatchGetLifecyclePolicy", "aoss:BatchGetVpcEndpoint", @@ -146,7 +142,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "clouddirectory:List*", "clouddirectory:LookupPolicy", "cloudformation:Describe*", - "cloudformation:Detect*", "cloudformation:Estimate*", "cloudformation:List*", "cloudformation:ValidateTemplate", @@ -161,9 +156,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "cloudtrail:Describe*", "cloudtrail:Get*", "cloudtrail:List*", - "cloudtrail:LookupEvents", "cloudwatch:Describe*", - "cloudwatch:GenerateQuery", "codeartifact:List*", "codebuild:BatchGetProjects", "codebuild:Describe*", @@ -227,7 +220,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "emr-containers:List*", "es:Describe*", "es:List*", - "glue:Describe*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", @@ -304,8 +296,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "lightsail:Get*", "logs:Describe*", "logs:List*", - "logs:StopLiveTail", - "logs:StopQuery", "logs:TestMetricFilter", "machinelearning:Describe*", "managedblockchain:GetMember", @@ -346,13 +336,9 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "nimble:List*", "notifications-contacts:List*", "notifications:Get*", - "notifications:List*", - "notifications:GetManagedNotificationEvent", "notifications:GetNotificationConfiguration", "notifications:GetNotificationsAccessForOrganization", - "notifications:GetNotificationEvent", "notifications:List*", - "one:ListUsers", "opsworks-cm:Describe*", "opsworks-cm:List*", "opsworks:Describe*", @@ -654,6 +640,28 @@ data "aws_iam_policy_document" "deny_actions_policy" { "transfer:TestIdentityProvider", "verifiedpermissions:IsAuthorized", "verifiedpermissions:IsAuthorizedWithToken", + "appstream:DescribeSessions", + "appstream:DescribeUsers", + "appsync:ListApiKeys", + "cognito-identity:LookupDeveloperIdentity", + "cognito-idp:AdminListDevices", + "cognito-idp:DescribeUserPoolClient", + "identitystore:DescribeUser", + "kinesisanalytics:DiscoverInputSchema", + "lakeformation:GetWorkUnitResults", + "lakeformation:GetWorkUnits", + "lightsail:GetContainerLog", + "notifications:GetManagedNotificationEvent", + "notifications:GetNotificationEvent", + "rekognition:ListFaces", + "rekognition:ListUsers", + "sso:ListAccountAssignments", + "sso:ListAccountAssignmentsForPrincipal", + "support:DescribeAttachment", + "workmail:DescribeUser", + "workmail:ListAliases", + "workmail:ListUsers", + "workspaces:DescribeWorkspaces", "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] From fa61e872ede23099e1f23635b8cf8a06d8562624 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 8 Sep 2026 09:51:24 +1000 Subject: [PATCH 05/36] fix(k8s): drop the unused get verb and harden the terraform collector Under an assume-breach model the collector's RBAC and pod spec are what an attacker inherits, so both are cut to what the code actually needs. - Drop `get` from all 38 grants. `collect.go` has no `.Get(` call on any kind, including CRDs; all five list sites go through `.List`, so `get` was dead weight in the ClusterRole a compromised image would hold. - Grant `nonResourceURLs: ["/version"]`. `Discovery().ServerVersion()` is called and was never granted; it works today only because EKS, GKE and AKS bind `system:public-info-viewer` to `system:authenticated`, and under `Strict: true` a cluster without that default fails collection outright. - Give the terraform CronJob the security context the Helm chart already sets (no privilege escalation, read-only root filesystem, run as non-root, drop ALL capabilities). Terraform-deployed collectors ran with none of it. - Add the trailing newline `terraform-v6/modules/k8s-resources/main.tf` was missing, the same defect this branch already fixed in its `data.tf`. All three manifests stay byte-identical in what they grant: 38 resources at `list`, plus `/version` at `get`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0126R5BKvayttBu8nKmFZBVR --- .../modules/k8s-resources/main.tf | 47 ++++++++++++------- .../terraform/modules/k8s-resources/main.tf | 45 ++++++++++++------ .../templates/clusterrole.yaml | 32 +++++++------ 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 342fb61..ead382b 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -50,7 +50,7 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "persistentvolumes", "persistentvolumeclaims", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -61,25 +61,25 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "statefulsets", "daemonsets", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["batch"] resources = ["jobs", "cronjobs"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["autoscaling"] resources = ["horizontalpodautoscalers"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["policy"] resources = ["poddisruptionbudgets"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -89,13 +89,13 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "ingressclasses", "networkpolicies", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["discovery.k8s.io"] resources = ["endpointslices"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -106,37 +106,37 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "clusterroles", "clusterrolebindings", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["storage.k8s.io"] resources = ["storageclasses", "csidrivers"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["node.k8s.io"] resources = ["runtimeclasses"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["scheduling.k8s.io"] resources = ["priorityclasses"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["flowcontrol.apiserver.k8s.io"] resources = ["flowschemas"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["certificates.k8s.io"] resources = ["certificatesigningrequests"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -147,13 +147,18 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "validatingadmissionpolicies", "validatingadmissionpolicybindings", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["apiextensions.k8s.io"] resources = ["customresourcedefinitions"] - verbs = ["get", "list"] + verbs = ["list"] + } + + rule { + non_resource_urls = ["/version"] + verbs = ["get"] } } @@ -254,6 +259,16 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + security_context { + allow_privilege_escalation = false + read_only_root_filesystem = true + run_as_non_root = true + + capabilities { + drop = ["ALL"] + } + } + resources { requests = { memory = "256Mi" @@ -270,4 +285,4 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index df9b9d4..ead382b 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -50,7 +50,7 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "persistentvolumes", "persistentvolumeclaims", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -61,25 +61,25 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "statefulsets", "daemonsets", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["batch"] resources = ["jobs", "cronjobs"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["autoscaling"] resources = ["horizontalpodautoscalers"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["policy"] resources = ["poddisruptionbudgets"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -89,13 +89,13 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "ingressclasses", "networkpolicies", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["discovery.k8s.io"] resources = ["endpointslices"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -106,37 +106,37 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "clusterroles", "clusterrolebindings", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["storage.k8s.io"] resources = ["storageclasses", "csidrivers"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["node.k8s.io"] resources = ["runtimeclasses"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["scheduling.k8s.io"] resources = ["priorityclasses"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["flowcontrol.apiserver.k8s.io"] resources = ["flowschemas"] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["certificates.k8s.io"] resources = ["certificatesigningrequests"] - verbs = ["get", "list"] + verbs = ["list"] } rule { @@ -147,13 +147,18 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "validatingadmissionpolicies", "validatingadmissionpolicybindings", ] - verbs = ["get", "list"] + verbs = ["list"] } rule { api_groups = ["apiextensions.k8s.io"] resources = ["customresourcedefinitions"] - verbs = ["get", "list"] + verbs = ["list"] + } + + rule { + non_resource_urls = ["/version"] + verbs = ["get"] } } @@ -254,6 +259,16 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + security_context { + allow_privilege_escalation = false + read_only_root_filesystem = true + run_as_non_root = true + + capabilities { + drop = ["ALL"] + } + } + resources { requests = { memory = "256Mi" diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index 39af3df..db9f326 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -9,47 +9,49 @@ rules: # Access for collecting cluster information - apiGroups: [""] resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "secrets", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["apps"] resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["autoscaling"] resources: ["horizontalpodautoscalers"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["networking.k8s.io"] resources: ["ingresses", "ingressclasses", "networkpolicies"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "csidrivers"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["flowcontrol.apiserver.k8s.io"] resources: ["flowschemas"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["certificates.k8s.io"] resources: ["certificatesigningrequests"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] - verbs: ["get", "list"] + verbs: ["list"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - verbs: ["get", "list"] + verbs: ["list"] + - nonResourceURLs: ["/version"] + verbs: ["get"] {{- end }} \ No newline at end of file From 950f2bd972d4e6a13fd9bd30ab3ad06c0db625b3 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 8 Sep 2026 09:58:46 +1000 Subject: [PATCH 06/36] fix(aws): cut the role to the actions the collectors call The readonly policy was AWS's SecurityAudit shape minus a deny list, so it granted ~300 services the collectors never touch. If Nullify is compromised the attacker inherits this role in every onboarded account, and that unused surface is pure attacker capability with no scanning value. Rebuild the two readonly policies as an explicit allow list: 433 account-wide Allow entries -> 107. - Drop 134 services outright, none of which any collector calls: sdb, lakeformation, lightsail, support, workmail, rekognition, sso, sso-directory, identitystore, route53domains, storagegateway, servicecatalog, waf, machinelearning, the mediapackage family and the rest. - Drop cloudformation, elasticbeanstalk, amplify and serverlessrepo, which have no processor at all and leak DescribeStacks parameters, Beanstalk plaintext env config, Amplify basicAuthCredentials and presigned source archives. - Drop `iam:Simulate*`, whose only use to an attacker holding the IAM graph is testing escalation paths offline without generating AccessDenied in CloudTrail. - Narrow 9 families whose wildcard mixes configuration reads with a data or secret sibling: `lambda:Get*` (GetLayerVersion returns a presigned code URL), `s3:List*` (ListBucket enumerates object keys), `guardduty:Get*` (GetFindings), `appsync:List*` (ListApiKeys), `cognito-idp:Describe*`/`List*`, `cloudtrail:Get*`, `ssm:List*`, `states:Describe*`. - Keep `ec2:Describe*` and `iam:Get*`/`List*` as wildcards deliberately. Their families are inventory metadata, enumerating them is brittle, and `iam:Get*`/`List*` also cover the OIDC-provider reads in admin/internal/cloud/cloudpolicies that a processors-only derivation misses. Verified against 234 required actions drawn from the collectors, the admin preflight and cloudpolicies: all 234 still resolve to allowed, zero regressions. Every dropped item was confirmed to have no caller anywhere in context, admin, cspm or scpm. readonly-part1 falls from 5599 to 1355 of 6144 chars. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0126R5BKvayttBu8nKmFZBVR --- .../nullify-cloudformation-template.json | 402 ++---------------- .../modules/nullify-aws-integration/data.tf | 392 ++--------------- .../modules/nullify-aws-integration/data.tf | 392 ++--------------- 3 files changed, 104 insertions(+), 1082 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 43f7827..c35ade0 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -376,224 +376,60 @@ { "Effect": "Allow", "Action": [ - "a4b:List*", - "access-analyzer:GetAccessPreview", - "access-analyzer:GetAnalyzedResource", - "access-analyzer:GetAnalyzer", - "access-analyzer:GetArchiveRule", - "access-analyzer:GetFinding", - "access-analyzer:GetFindingsStatistics", - "access-analyzer:GetGeneratedPolicy", "access-analyzer:List*", - "access-analyzer:ValidatePolicy", - "account:GetAccountInformation", - "account:GetRegionOptStatus", - "account:ListRegions", - "acm-pca:Describe*", - "acm-pca:List*", "acm:Describe*", "acm:List*", - "aiops:List*", - "airflow:List*", - "amplify:GetBranch", - "amplify:GetDomainAssociation", - "amplify:List*", - "aoss:BatchGetLifecyclePolicy", - "aoss:BatchGetVpcEndpoint", - "aoss:GetAccessPolicy", - "aoss:GetAccountSettings", - "aoss:GetPoliciesStats", - "aoss:GetSecurityConfig", - "aoss:GetSecurityPolicy", - "aoss:List*", "apigateway:GET", - "appconfig:List*", - "appfabric:List*", - "appflow:List*", - "application-autoscaling:Describe*", - "application-autoscaling:ListTagsForResource", - "application-signals:List*", - "applicationinsights:List*", - "appmesh:Describe*", - "appmesh:List*", - "apprunner:DescribeAutoScalingConfiguration", - "apprunner:DescribeCustomDomains", - "apprunner:DescribeObservabilityConfiguration", "apprunner:DescribeService", - "apprunner:DescribeVpcConnector", - "apprunner:DescribeVpcIngressConnection", - "apprunner:DescribeWebAclForService", "apprunner:List*", - "appstream:Describe*", - "appstream:List*", - "appsync:List*", - "apptest:List*", - "aps:DescribeAlertManagerDefinition", - "aps:DescribeLoggingConfiguration", - "aps:DescribeRuleGroupsNamespace", - "aps:DescribeScraper", - "aps:DescribeWorkspace", - "aps:List*", - "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", - "arc-zonal-shift:ListAutoshifts", - "arc-zonal-shift:ListManagedResources", - "arc-zonal-shift:ListZonalShifts", - "artifact:ListAgreements", - "artifact:ListCustomerAgreements", - "artifact:ListReports", + "appsync:ListGraphqlApis", "athena:GetWorkGroup", "athena:List*", - "auditmanager:List*", - "auditmanager:ValidateAssessmentReportIntegrity", - "autoscaling-plans:Describe*", - "autoscaling:Describe*", - "backup-gateway:List*", - "backup:Describe*", "backup:GetBackupPlan", "backup:GetBackupVaultAccessPolicy", "backup:List*", - "batch:Describe*", - "batch:List*", - "bedrock:List*", - "billingconductor:List*", - "cloud9:Describe*", - "cloud9:List*", - "clouddirectory:List*", - "clouddirectory:LookupPolicy", - "cloudformation:Describe*", - "cloudformation:Estimate*", - "cloudformation:List*", - "cloudformation:ValidateTemplate", - "cloudfront-keyvaluestore:Describe*", - "cloudfront-keyvaluestore:List*", - "cloudfront:Describe*", "cloudfront:List*", - "cloudhsm:Describe*", - "cloudhsm:List*", - "cloudsearch:Describe*", - "cloudsearch:List*", "cloudtrail:Describe*", - "cloudtrail:Get*", + "cloudtrail:GetEventDataStore", + "cloudtrail:GetTrailStatus", "cloudtrail:List*", "cloudwatch:Describe*", - "codeartifact:List*", "codebuild:BatchGetProjects", - "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", - "cognito-identity:GetIdentityPoolAnalytics", - "cognito-identity:GetIdentityPoolDailyAnalytics", - "cognito-identity:GetIdentityPoolRoles", - "cognito-identity:GetIdentityProviderDailyAnalytics", "cognito-identity:List*", - "cognito-identity:Lookup*", - "cognito-idp:AdminList*", - "cognito-idp:Describe*", - "cognito-idp:List*", - "cognito-sync:Describe*", - "cognito-sync:List*", - "devops-guru:List*", - "discovery:List*", - "identitystore:Describe*", - "identity-sync:ListSyncFilters", - "ecr-public:BatchCheckLayerAvailability", - "ecr-public:DescribeImages", - "ecr-public:DescribeImageTags", - "ecr-public:DescribeRegistries", - "ecr-public:DescribeRepositories", - "ecr-public:GetRepositoryCatalogData", - "ecr-public:GetRepositoryPolicy", - "ecr-public:ListTagsForResource", - "ecr:BatchCheck*", + "cognito-idp:DescribeUserPool", + "cognito-idp:ListUserPools", + "config:Describe*", + "dynamodb:Describe*", + "dynamodb:GetResourcePolicy", + "dynamodb:List*", + "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", + "ec2:SearchTransitGateway*", "ecr:Describe*", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", - "ecr:List*", "ecs:Describe*", "ecs:List*", "eks:Describe*", "eks:List*", - "elastic-inference:Describe*", - "elastic-inference:List*", "elasticache:Describe*", - "elasticache:List*", - "elasticbeanstalk:Check*", - "elasticbeanstalk:Describe*", - "elasticbeanstalk:List*", "elasticfilesystem:Describe*", - "elasticfilesystem:ListTagsForResource", "elasticloadbalancing:Describe*", "elasticmapreduce:Describe*", "elasticmapreduce:GetBlockPublicAccessConfiguration", "elasticmapreduce:List*", - "elasticmapreduce:View*", - "emr-containers:Describe*", - "emr-containers:List*", - "elastictranscoder:List*", - "elastictranscoder:Read*", "es:Describe*", "es:List*", - "events:Describe*", "events:List*", - "dynamodb:Describe*", - "dynamodb:GetResourcePolicy", - "dynamodb:List*", - "discovery:Describe*", - "config:Describe*", - "config:SelectResourceConfig", - "iotfleetwise:List*", - "iotroborunner:List*", - "iotsitewise:Describe*", - "iotsitewise:List*", - "iotwireless:List*", - "ivs:List*", - "ivschat:List*", - "kafka:Describe*", - "kafka:List*", - "kafkaconnect:List*", - "kendra:BatchGetDocumentStatus", - "kendra:DescribeDataSource", - "kendra:DescribeExperience", - "kendra:DescribeFaq", - "kendra:DescribeIndex", - "kendra:DescribePrincipalMapping", - "kendra:DescribeQuerySuggestionsBlockList", - "kendra:DescribeQuerySuggestionsConfig", - "kendra:DescribeThesaurus", - "kendra:List*", - "kinesis:Describe*", - "kinesis:List*", - "iam:Simulate*", - "iam:GenerateCredentialReport", - "iam:Get*", - "iam:List*", + "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", - "glue:GetJobs", - "glue:GetSecurityConfigurations", -"glue:List*", - "kinesisanalytics:Describe*", - "kinesisanalytics:Discover*", - "kinesisanalytics:List*", - "kinesisvideo:Describe*", - "kinesisvideo:List*", - "kms:Describe*", - "kms:GetKeyPolicy", - "kms:GetKeyRotationStatus", - "kms:List*", - "lakeformation:Describe*", - "lakeformation:Get*", - "lakeformation:List*", - "lakeformation:Search*", - "lambda:List*", - "lambda:Get*", - "ec2:Describe*", - "ec2:GetEbsEncryptionByDefault", - "ec2:GetManagedPrefixListEntries", - "ec2:GetSnapshotBlockPublicAccessState", - "ec2:GetTransitGateway*", - "ec2:SearchTransitGateway*" + "glue:GetJobs" ], "Resource": "*", "Condition": {} @@ -615,221 +451,59 @@ { "Effect": "Allow", "Action": [ - "globalaccelerator:Describe*", - "globalaccelerator:List*", - "guardduty:Describe*", - "guardduty:Get*", + "glue:GetSecurityConfigurations", + "guardduty:GetDetector", "guardduty:List*", - "launchwizard:Describe*", - "launchwizard:GetWorkload", - "launchwizard:List*", - "license-manager:List*", - "lightsail:Get*", + "iam:GenerateCredentialReport", + "iam:Get*", + "iam:List*", + "kafka:List*", + "kinesis:Describe*", + "kinesis:List*", + "kms:Describe*", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", + "kms:List*", + "lambda:GetFunction", + "lambda:GetFunctionUrlConfig", + "lambda:GetPolicy", + "lambda:List*", "logs:Describe*", - "logs:List*", - "logs:TestMetricFilter", - "machinelearning:Describe*", - "managedblockchain:GetMember", - "managedblockchain:GetNetwork", - "managedblockchain:GetNode", - "managedblockchain:GetProposal", - "managedblockchain:List*", - "mediaconnect:Describe*", - "mediaconnect:List*", - "mediaconvert:List*", - "medialive:Describe*", - "medialive:Get*", - "medialive:List*", - "mediapackage-vod:Describe*", - "mediapackage-vod:List*", - "mediapackage:Describe*", - "mediapackage:List*", - "mediapackagev2:GetChannel", - "mediapackagev2:GetChannelGroup", - "mediapackagev2:GetChannelPolicy", - "mediapackagev2:GetHeadObject", - "mediapackagev2:GetOriginEndpoint", - "mediapackagev2:GetOriginEndpointPolicy", - "mediapackagev2:List*", - "mediastore:Describe*", - "mediastore:List*", "memorydb:Describe*", "memorydb:List*", "mq:Describe*", "mq:List*", - "network-firewall:Describe*", - "network-firewall:List*", - "networkmanager:DescribeGlobalNetworks", - "networkmanager:Get*", - "networkmanager:List*", "neptune:Describe*", "neptune:List*", - "nimble:List*", - "notifications-contacts:List*", - "notifications:Get*", - "notifications:GetNotificationConfiguration", - "notifications:GetNotificationsAccessForOrganization", - "notifications:List*", - "opsworks-cm:Describe*", - "opsworks-cm:List*", - "opsworks:Describe*", + "network-firewall:Describe*", + "network-firewall:List*", "organizations:Describe*", "organizations:List*", - "outposts:List*", - "personalize:Describe*", - "personalize:List*", - "pi:ListAvailableResourceDimensions", - "pi:ListAvailableResourceMetrics", - "pipes:DescribePipe", - "pipes:ListPipes", - "polly:Describe*", - "polly:List*", - "pricing:DescribeServices", - "pricing:ListPriceLists", - "qbusiness:List*", - "ram:List*", - "rbin:List*", "rds:Describe*", - "rds:List*", - "redshift-serverless:List*", "redshift:Describe*", - "redshift:List*", - "redshift:View*", - "rekognition:Describe*", - "rekognition:List*", - "resource-explorer-2:List*", - "resource-groups:List*", - "resource-groups:Search*", - "robomaker:BatchDescribe*", - "robomaker:Describe*", - "robomaker:List*", - "route53-recovery-cluster:Get*", - "route53-recovery-cluster:List*", - "route53-recovery-control-config:Describe*", - "route53-recovery-control-config:Get*", - "route53-recovery-control-config:List*", - "route53-recovery-readiness:Get*", - "route53-recovery-readiness:List*", - "route53:Get*", "route53:List*", - "route53:Test*", - "route53domains:Check*", - "route53domains:Get*", - "route53domains:List*", - "route53domains:View*", - "route53profiles:Get*", - "route53profiles:List*", - "route53resolver:Get*", - "route53resolver:List*", - "rum:GetAppMonitor", - "rum:ListAppMonitors", - "s3-object-lambda:List*", - "s3-outposts:Get*", - "s3-outposts:List*", - "s3:Describe*", "s3:GetAccountPublicAccessBlock", "s3:GetBucket*", "s3:GetEncryptionConfiguration", "s3:GetLifecycleConfiguration", "s3:GetReplicationConfiguration", - "s3:List*", + "s3:ListAllMyBuckets", "sagemaker:Describe*", "sagemaker:List*", - "scheduler:List*", - "schemas:Describe*", - "schemas:Get*", - "schemas:List*", - "schemas:Search*", - "sdb:List*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", - "securityhub:Describe*", - "securityhub:List*", - "securitylake:ListDataLakeExceptions", - "securitylake:ListDataLakes", - "securitylake:ListLogSources", - "securitylake:ListSubscribers", - "securitylake:ListTagsForResource", - "serverlessrepo:Get*", - "serverlessrepo:List*", - "serverlessrepo:SearchApplications", - "servicecatalog:Describe*", - "servicecatalog:List*", - "servicecatalog:Scan*", - "servicecatalog:Search*", - "servicediscovery:DiscoverInstances", - "servicediscovery:DiscoverInstancesRevision", - "servicediscovery:List*", - "servicequotas:List*", - "ses:Describe*", - "ses:List*", - "shield:Describe*", - "shield:List*", - "signer:DescribeSigningJob", - "signer:List*", - "signin:ListTrustedIdentityPropagationApplicationsForConsole", - "sms-voice:Describe*", - "sms-voice:List*", - "snowball:Describe*", - "snowball:List*", - "sns:Check*", "sns:GetTopicAttributes", "sns:List*", "sqs:GetQueueAttributes", "sqs:List*", - "ssm-contacts:List*", - "ssm-sap:List*", - "ssm-quicksetup:List*", "ssm:Describe*", "ssm:GetDocument", - "ssm:List*", - "sso-directory:List*", - "sso:List*", - "sso:Search*", - "states:Describe*", + "ssm:ListDocuments", + "states:DescribeStateMachine", "states:List*", - "states:ValidateStateMachineDefinition", - "storagegateway:Describe*", - "storagegateway:List*", - "sts:GetAccessKeyInfo", "sts:GetCallerIdentity", - "sts:GetSessionToken", - "support:Describe*", - "tag:Describe*", - "tag:Get*", - "tax:ListTaxRegistrations", - "timestream:Describe*", - "tnb:List*", - "transcribe:List*", - "transfer:Describe*", - "transfer:List*", - "translate:DescribeTextTranslationJob", - "translate:ListParallelData", - "translate:ListTerminologies", - "translate:ListTextTranslationJobs", - "trustedadvisor:Describe*", - "trustedadvisor:List*", - "verifiedpermissions:List*", - "vpc-lattice:Get*", - "vpc-lattice:List*", - "waf-regional:List*", - "waf:List*", -"wafv2:CheckCapacity", - "wafv2:Describe*", "wafv2:GetWebACL", - "wafv2:List*", - "wellarchitected:ExportLens", - "wellarchitected:Get*", - "wellarchitected:List*", - "workdocs:CheckAlias", - "workdocs:Describe*", - "workmail:Describe*", - "workmail:List*", - "workspaces-web:GetBrowserSettings", - "workspaces-web:GetIdentityProvider", - "workspaces-web:GetNetworkSettings", - "workspaces-web:List*", - "workspaces:Describe*" + "wafv2:List*" ], "Resource": "*", "Condition": {} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 3b434d2..2d51140 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -57,448 +57,122 @@ data "aws_iam_policy_document" "readonly_policy_part1" { statement { effect = "Allow" actions = [ - "a4b:List*", - "access-analyzer:GetAccessPreview", - "access-analyzer:GetAnalyzedResource", - "access-analyzer:GetAnalyzer", - "access-analyzer:GetArchiveRule", - "access-analyzer:GetFinding", - "access-analyzer:GetFindingsStatistics", - "access-analyzer:GetGeneratedPolicy", "access-analyzer:List*", - "access-analyzer:ValidatePolicy", - "account:GetAccountInformation", - "account:GetRegionOptStatus", - "account:ListRegions", - "acm-pca:Describe*", - "acm-pca:List*", "acm:Describe*", "acm:List*", - "aiops:List*", - "airflow:List*", - "amplify:GetBranch", - "amplify:GetDomainAssociation", - "amplify:List*", - "aoss:BatchGetLifecyclePolicy", - "aoss:BatchGetVpcEndpoint", - "aoss:GetAccessPolicy", - "aoss:GetAccountSettings", - "aoss:GetPoliciesStats", - "aoss:GetSecurityConfig", - "aoss:GetSecurityPolicy", - "aoss:List*", "apigateway:GET", - "appconfig:List*", - "appfabric:List*", - "appflow:List*", - "application-autoscaling:Describe*", - "application-autoscaling:ListTagsForResource", - "application-signals:List*", - "applicationinsights:List*", - "appmesh:Describe*", - "appmesh:List*", - "apprunner:DescribeAutoScalingConfiguration", - "apprunner:DescribeCustomDomains", - "apprunner:DescribeObservabilityConfiguration", "apprunner:DescribeService", - "apprunner:DescribeVpcConnector", - "apprunner:DescribeVpcIngressConnection", - "apprunner:DescribeWebAclForService", "apprunner:List*", - "appstream:Describe*", - "appstream:List*", - "appsync:List*", - "apptest:List*", - "aps:DescribeAlertManagerDefinition", - "aps:DescribeLoggingConfiguration", - "aps:DescribeRuleGroupsNamespace", - "aps:DescribeScraper", - "aps:DescribeWorkspace", - "aps:List*", - "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", - "arc-zonal-shift:ListAutoshifts", - "arc-zonal-shift:ListManagedResources", - "arc-zonal-shift:ListZonalShifts", - "artifact:ListAgreements", - "artifact:ListCustomerAgreements", - "artifact:ListReports", + "appsync:ListGraphqlApis", "athena:GetWorkGroup", "athena:List*", - "auditmanager:List*", - "auditmanager:ValidateAssessmentReportIntegrity", - "autoscaling-plans:Describe*", - "autoscaling:Describe*", - "backup-gateway:List*", - "backup:Describe*", "backup:GetBackupPlan", "backup:GetBackupVaultAccessPolicy", "backup:List*", - "batch:Describe*", - "batch:List*", - "bedrock:List*", - "billingconductor:List*", - "cloud9:Describe*", - "cloud9:List*", - "clouddirectory:List*", - "clouddirectory:LookupPolicy", - "cloudformation:Describe*", - "cloudformation:Estimate*", - "cloudformation:List*", - "cloudformation:ValidateTemplate", - "cloudfront-keyvaluestore:Describe*", - "cloudfront-keyvaluestore:List*", - "cloudfront:Describe*", "cloudfront:List*", - "cloudhsm:Describe*", - "cloudhsm:List*", - "cloudsearch:Describe*", - "cloudsearch:List*", "cloudtrail:Describe*", - "cloudtrail:Get*", + "cloudtrail:GetEventDataStore", + "cloudtrail:GetTrailStatus", "cloudtrail:List*", "cloudwatch:Describe*", - "codeartifact:List*", "codebuild:BatchGetProjects", - "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", - "cognito-identity:GetIdentityPoolAnalytics", - "cognito-identity:GetIdentityPoolDailyAnalytics", - "cognito-identity:GetIdentityPoolRoles", - "cognito-identity:GetIdentityProviderDailyAnalytics", "cognito-identity:List*", - "cognito-identity:Lookup*", - "cognito-idp:AdminList*", - "cognito-idp:Describe*", - "cognito-idp:List*", - "cognito-sync:Describe*", - "cognito-sync:List*", + "cognito-idp:DescribeUserPool", + "cognito-idp:ListUserPools", "config:Describe*", - "config:SelectResourceConfig", - "devops-guru:List*", - "discovery:Describe*", - "discovery:List*", "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", - "ecr-public:BatchCheckLayerAvailability", - "ecr-public:DescribeImages", - "ecr-public:DescribeImageTags", - "ecr-public:DescribeRegistries", - "ecr-public:DescribeRepositories", - "ecr-public:GetRepositoryCatalogData", - "ecr-public:GetRepositoryPolicy", - "ecr-public:ListTagsForResource", - "ecr:BatchCheck*", + "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", + "ec2:SearchTransitGateway*", "ecr:Describe*", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", - "ecr:List*", "ecs:Describe*", "ecs:List*", "eks:Describe*", "eks:List*", - "elastic-inference:Describe*", - "elastic-inference:List*", "elasticache:Describe*", - "elasticache:List*", - "elasticbeanstalk:Check*", - "elasticbeanstalk:Describe*", - "elasticbeanstalk:List*", "elasticfilesystem:Describe*", - "elasticfilesystem:ListTagsForResource", "elasticloadbalancing:Describe*", "elasticmapreduce:Describe*", "elasticmapreduce:GetBlockPublicAccessConfiguration", "elasticmapreduce:List*", - "elasticmapreduce:View*", - "elastictranscoder:List*", - "elastictranscoder:Read*", - "events:Describe*", - "events:List*", - "emr-containers:Describe*", - "emr-containers:List*", "es:Describe*", "es:List*", + "events:List*", + "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", - "glue:GetJobs", + "glue:GetJobs" + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "readonly_policy_part2" { + statement { + effect = "Allow" + actions = [ "glue:GetSecurityConfigurations", -"glue:List*", + "guardduty:GetDetector", + "guardduty:List*", "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", - "iam:Simulate*", - "identity-sync:ListSyncFilters", - "identitystore:Describe*", - "iotfleetwise:List*", - "iotroborunner:List*", - "iotsitewise:Describe*", - "iotsitewise:List*", - "iotwireless:List*", - "ivs:List*", - "ivschat:List*", - "kafka:Describe*", "kafka:List*", - "kafkaconnect:List*", - "kendra:BatchGetDocumentStatus", - "kendra:DescribeDataSource", - "kendra:DescribeExperience", - "kendra:DescribeFaq", - "kendra:DescribeIndex", - "kendra:DescribePrincipalMapping", - "kendra:DescribeQuerySuggestionsBlockList", - "kendra:DescribeQuerySuggestionsConfig", - "kendra:DescribeThesaurus", - "kendra:List*", "kinesis:Describe*", "kinesis:List*", - "kinesisanalytics:Describe*", - "kinesisanalytics:Discover*", - "kinesisanalytics:List*", - "kinesisvideo:Describe*", - "kinesisvideo:List*", "kms:Describe*", "kms:GetKeyPolicy", "kms:GetKeyRotationStatus", "kms:List*", - "lakeformation:Describe*", - "lakeformation:Get*", - "lakeformation:List*", - "lakeformation:Search*", - "lambda:Get*", + "lambda:GetFunction", + "lambda:GetFunctionUrlConfig", + "lambda:GetPolicy", "lambda:List*", - "ec2:Describe*", - "ec2:GetEbsEncryptionByDefault", - "ec2:GetManagedPrefixListEntries", - "ec2:GetSnapshotBlockPublicAccessState", - "ec2:GetTransitGateway*", - "ec2:SearchTransitGateway*" - ] - resources = ["*"] - } -} - -data "aws_iam_policy_document" "readonly_policy_part2" { - statement { - effect = "Allow" - actions = [ - "globalaccelerator:Describe*", - "globalaccelerator:List*", - "guardduty:Describe*", - "guardduty:Get*", - "guardduty:List*", - "launchwizard:Describe*", - "launchwizard:GetWorkload", - "launchwizard:List*", - "license-manager:List*", - "lightsail:Get*", "logs:Describe*", - "logs:List*", - "logs:TestMetricFilter", - "machinelearning:Describe*", - "managedblockchain:GetMember", - "managedblockchain:GetNetwork", - "managedblockchain:GetNode", - "managedblockchain:GetProposal", - "managedblockchain:List*", - "mediaconnect:Describe*", - "mediaconnect:List*", - "mediaconvert:List*", - "medialive:Describe*", - "medialive:Get*", - "medialive:List*", - "mediapackage-vod:Describe*", - "mediapackage-vod:List*", - "mediapackage:Describe*", - "mediapackage:List*", - "mediapackagev2:GetChannel", - "mediapackagev2:GetChannelGroup", - "mediapackagev2:GetChannelPolicy", - "mediapackagev2:GetHeadObject", - "mediapackagev2:GetOriginEndpoint", - "mediapackagev2:GetOriginEndpointPolicy", - "mediapackagev2:List*", - "mediastore:Describe*", - "mediastore:List*", "memorydb:Describe*", "memorydb:List*", "mq:Describe*", "mq:List*", - "network-firewall:Describe*", - "network-firewall:List*", - "networkmanager:DescribeGlobalNetworks", - "networkmanager:Get*", - "networkmanager:List*", "neptune:Describe*", "neptune:List*", - "nimble:List*", - "notifications-contacts:List*", - "notifications:Get*", - "notifications:GetNotificationConfiguration", - "notifications:GetNotificationsAccessForOrganization", - "notifications:List*", - "opsworks-cm:Describe*", - "opsworks-cm:List*", - "opsworks:Describe*", + "network-firewall:Describe*", + "network-firewall:List*", "organizations:Describe*", "organizations:List*", - "outposts:List*", - "personalize:Describe*", - "personalize:List*", - "pi:ListAvailableResourceDimensions", - "pi:ListAvailableResourceMetrics", - "pipes:DescribePipe", - "pipes:ListPipes", - "polly:Describe*", - "polly:List*", - "pricing:DescribeServices", - "pricing:ListPriceLists", - "qbusiness:List*", - "ram:List*", - "rbin:List*", "rds:Describe*", - "rds:List*", - "redshift-serverless:List*", "redshift:Describe*", - "redshift:List*", - "redshift:View*", - "rekognition:Describe*", - "rekognition:List*", - "resource-explorer-2:List*", - "resource-groups:List*", - "resource-groups:Search*", - "robomaker:BatchDescribe*", - "robomaker:Describe*", - "robomaker:List*", - "route53-recovery-cluster:Get*", - "route53-recovery-cluster:List*", - "route53-recovery-control-config:Describe*", - "route53-recovery-control-config:Get*", - "route53-recovery-control-config:List*", - "route53-recovery-readiness:Get*", - "route53-recovery-readiness:List*", - "route53:Get*", "route53:List*", - "route53:Test*", - "route53domains:Check*", - "route53domains:Get*", - "route53domains:List*", - "route53domains:View*", - "route53profiles:Get*", - "route53profiles:List*", - "route53resolver:Get*", - "route53resolver:List*", - "rum:GetAppMonitor", - "rum:ListAppMonitors", - "s3-object-lambda:List*", - "s3-outposts:Get*", - "s3-outposts:List*", - "s3:Describe*", "s3:GetAccountPublicAccessBlock", "s3:GetBucket*", "s3:GetEncryptionConfiguration", "s3:GetLifecycleConfiguration", "s3:GetReplicationConfiguration", - "s3:List*", + "s3:ListAllMyBuckets", "sagemaker:Describe*", "sagemaker:List*", - "scheduler:List*", - "schemas:Describe*", - "schemas:Get*", - "schemas:List*", - "schemas:Search*", - "sdb:List*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", - "securityhub:Describe*", - "securityhub:List*", - "securitylake:ListDataLakeExceptions", - "securitylake:ListDataLakes", - "securitylake:ListLogSources", - "securitylake:ListSubscribers", - "securitylake:ListTagsForResource", - "serverlessrepo:Get*", - "serverlessrepo:List*", - "serverlessrepo:SearchApplications", - "servicecatalog:Describe*", - "servicecatalog:List*", - "servicecatalog:Scan*", - "servicecatalog:Search*", - "servicediscovery:DiscoverInstances", - "servicediscovery:DiscoverInstancesRevision", - "servicediscovery:List*", - "servicequotas:List*", - "ses:Describe*", - "ses:List*", - "shield:Describe*", - "shield:List*", - "signer:DescribeSigningJob", - "signer:List*", - "signin:ListTrustedIdentityPropagationApplicationsForConsole", - "sms-voice:Describe*", - "sms-voice:List*", - "snowball:Describe*", - "snowball:List*", - "sns:Check*", "sns:GetTopicAttributes", "sns:List*", "sqs:GetQueueAttributes", "sqs:List*", - "ssm-contacts:List*", - "ssm-sap:List*", - "ssm-quicksetup:List*", "ssm:Describe*", "ssm:GetDocument", - "ssm:List*", - "sso-directory:List*", - "sso:List*", - "sso:Search*", - "states:Describe*", + "ssm:ListDocuments", + "states:DescribeStateMachine", "states:List*", - "states:ValidateStateMachineDefinition", - "storagegateway:Describe*", - "storagegateway:List*", - "sts:GetAccessKeyInfo", "sts:GetCallerIdentity", - "sts:GetSessionToken", - "support:Describe*", - "tag:Describe*", - "tag:Get*", - "tax:ListTaxRegistrations", - "timestream:Describe*", - "tnb:List*", - "transcribe:List*", - "transfer:Describe*", - "transfer:List*", - "translate:DescribeTextTranslationJob", - "translate:ListParallelData", - "translate:ListTerminologies", - "translate:ListTextTranslationJobs", - "trustedadvisor:Describe*", - "trustedadvisor:List*", - "verifiedpermissions:List*", - "vpc-lattice:Get*", - "vpc-lattice:List*", - "waf-regional:List*", - "waf:List*", -"wafv2:CheckCapacity", - "wafv2:Describe*", "wafv2:GetWebACL", - "wafv2:List*", - "wellarchitected:ExportLens", - "wellarchitected:Get*", - "wellarchitected:List*", - "workdocs:CheckAlias", - "workdocs:Describe*", - "workmail:Describe*", - "workmail:List*", - "workspaces-web:GetBrowserSettings", - "workspaces-web:GetIdentityProvider", - "workspaces-web:GetNetworkSettings", - "workspaces-web:List*", - "workspaces:Describe*" + "wafv2:List*" ] resources = ["*"] } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 3b434d2..2d51140 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -57,448 +57,122 @@ data "aws_iam_policy_document" "readonly_policy_part1" { statement { effect = "Allow" actions = [ - "a4b:List*", - "access-analyzer:GetAccessPreview", - "access-analyzer:GetAnalyzedResource", - "access-analyzer:GetAnalyzer", - "access-analyzer:GetArchiveRule", - "access-analyzer:GetFinding", - "access-analyzer:GetFindingsStatistics", - "access-analyzer:GetGeneratedPolicy", "access-analyzer:List*", - "access-analyzer:ValidatePolicy", - "account:GetAccountInformation", - "account:GetRegionOptStatus", - "account:ListRegions", - "acm-pca:Describe*", - "acm-pca:List*", "acm:Describe*", "acm:List*", - "aiops:List*", - "airflow:List*", - "amplify:GetBranch", - "amplify:GetDomainAssociation", - "amplify:List*", - "aoss:BatchGetLifecyclePolicy", - "aoss:BatchGetVpcEndpoint", - "aoss:GetAccessPolicy", - "aoss:GetAccountSettings", - "aoss:GetPoliciesStats", - "aoss:GetSecurityConfig", - "aoss:GetSecurityPolicy", - "aoss:List*", "apigateway:GET", - "appconfig:List*", - "appfabric:List*", - "appflow:List*", - "application-autoscaling:Describe*", - "application-autoscaling:ListTagsForResource", - "application-signals:List*", - "applicationinsights:List*", - "appmesh:Describe*", - "appmesh:List*", - "apprunner:DescribeAutoScalingConfiguration", - "apprunner:DescribeCustomDomains", - "apprunner:DescribeObservabilityConfiguration", "apprunner:DescribeService", - "apprunner:DescribeVpcConnector", - "apprunner:DescribeVpcIngressConnection", - "apprunner:DescribeWebAclForService", "apprunner:List*", - "appstream:Describe*", - "appstream:List*", - "appsync:List*", - "apptest:List*", - "aps:DescribeAlertManagerDefinition", - "aps:DescribeLoggingConfiguration", - "aps:DescribeRuleGroupsNamespace", - "aps:DescribeScraper", - "aps:DescribeWorkspace", - "aps:List*", - "arc-zonal-shift:GetAutoshiftObserverNotificationStatus", - "arc-zonal-shift:ListAutoshifts", - "arc-zonal-shift:ListManagedResources", - "arc-zonal-shift:ListZonalShifts", - "artifact:ListAgreements", - "artifact:ListCustomerAgreements", - "artifact:ListReports", + "appsync:ListGraphqlApis", "athena:GetWorkGroup", "athena:List*", - "auditmanager:List*", - "auditmanager:ValidateAssessmentReportIntegrity", - "autoscaling-plans:Describe*", - "autoscaling:Describe*", - "backup-gateway:List*", - "backup:Describe*", "backup:GetBackupPlan", "backup:GetBackupVaultAccessPolicy", "backup:List*", - "batch:Describe*", - "batch:List*", - "bedrock:List*", - "billingconductor:List*", - "cloud9:Describe*", - "cloud9:List*", - "clouddirectory:List*", - "clouddirectory:LookupPolicy", - "cloudformation:Describe*", - "cloudformation:Estimate*", - "cloudformation:List*", - "cloudformation:ValidateTemplate", - "cloudfront-keyvaluestore:Describe*", - "cloudfront-keyvaluestore:List*", - "cloudfront:Describe*", "cloudfront:List*", - "cloudhsm:Describe*", - "cloudhsm:List*", - "cloudsearch:Describe*", - "cloudsearch:List*", "cloudtrail:Describe*", - "cloudtrail:Get*", + "cloudtrail:GetEventDataStore", + "cloudtrail:GetTrailStatus", "cloudtrail:List*", "cloudwatch:Describe*", - "codeartifact:List*", "codebuild:BatchGetProjects", - "codebuild:Describe*", "codebuild:List*", "cognito-identity:Describe*", - "cognito-identity:GetIdentityPoolAnalytics", - "cognito-identity:GetIdentityPoolDailyAnalytics", - "cognito-identity:GetIdentityPoolRoles", - "cognito-identity:GetIdentityProviderDailyAnalytics", "cognito-identity:List*", - "cognito-identity:Lookup*", - "cognito-idp:AdminList*", - "cognito-idp:Describe*", - "cognito-idp:List*", - "cognito-sync:Describe*", - "cognito-sync:List*", + "cognito-idp:DescribeUserPool", + "cognito-idp:ListUserPools", "config:Describe*", - "config:SelectResourceConfig", - "devops-guru:List*", - "discovery:Describe*", - "discovery:List*", "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", - "ecr-public:BatchCheckLayerAvailability", - "ecr-public:DescribeImages", - "ecr-public:DescribeImageTags", - "ecr-public:DescribeRegistries", - "ecr-public:DescribeRepositories", - "ecr-public:GetRepositoryCatalogData", - "ecr-public:GetRepositoryPolicy", - "ecr-public:ListTagsForResource", - "ecr:BatchCheck*", + "ec2:Describe*", + "ec2:GetEbsEncryptionByDefault", + "ec2:GetManagedPrefixListEntries", + "ec2:GetSnapshotBlockPublicAccessState", + "ec2:SearchTransitGateway*", "ecr:Describe*", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", - "ecr:List*", "ecs:Describe*", "ecs:List*", "eks:Describe*", "eks:List*", - "elastic-inference:Describe*", - "elastic-inference:List*", "elasticache:Describe*", - "elasticache:List*", - "elasticbeanstalk:Check*", - "elasticbeanstalk:Describe*", - "elasticbeanstalk:List*", "elasticfilesystem:Describe*", - "elasticfilesystem:ListTagsForResource", "elasticloadbalancing:Describe*", "elasticmapreduce:Describe*", "elasticmapreduce:GetBlockPublicAccessConfiguration", "elasticmapreduce:List*", - "elasticmapreduce:View*", - "elastictranscoder:List*", - "elastictranscoder:Read*", - "events:Describe*", - "events:List*", - "emr-containers:Describe*", - "emr-containers:List*", "es:Describe*", "es:List*", + "events:List*", + "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", "glue:GetDevEndpoints", - "glue:GetJobs", + "glue:GetJobs" + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "readonly_policy_part2" { + statement { + effect = "Allow" + actions = [ "glue:GetSecurityConfigurations", -"glue:List*", + "guardduty:GetDetector", + "guardduty:List*", "iam:GenerateCredentialReport", "iam:Get*", "iam:List*", - "iam:Simulate*", - "identity-sync:ListSyncFilters", - "identitystore:Describe*", - "iotfleetwise:List*", - "iotroborunner:List*", - "iotsitewise:Describe*", - "iotsitewise:List*", - "iotwireless:List*", - "ivs:List*", - "ivschat:List*", - "kafka:Describe*", "kafka:List*", - "kafkaconnect:List*", - "kendra:BatchGetDocumentStatus", - "kendra:DescribeDataSource", - "kendra:DescribeExperience", - "kendra:DescribeFaq", - "kendra:DescribeIndex", - "kendra:DescribePrincipalMapping", - "kendra:DescribeQuerySuggestionsBlockList", - "kendra:DescribeQuerySuggestionsConfig", - "kendra:DescribeThesaurus", - "kendra:List*", "kinesis:Describe*", "kinesis:List*", - "kinesisanalytics:Describe*", - "kinesisanalytics:Discover*", - "kinesisanalytics:List*", - "kinesisvideo:Describe*", - "kinesisvideo:List*", "kms:Describe*", "kms:GetKeyPolicy", "kms:GetKeyRotationStatus", "kms:List*", - "lakeformation:Describe*", - "lakeformation:Get*", - "lakeformation:List*", - "lakeformation:Search*", - "lambda:Get*", + "lambda:GetFunction", + "lambda:GetFunctionUrlConfig", + "lambda:GetPolicy", "lambda:List*", - "ec2:Describe*", - "ec2:GetEbsEncryptionByDefault", - "ec2:GetManagedPrefixListEntries", - "ec2:GetSnapshotBlockPublicAccessState", - "ec2:GetTransitGateway*", - "ec2:SearchTransitGateway*" - ] - resources = ["*"] - } -} - -data "aws_iam_policy_document" "readonly_policy_part2" { - statement { - effect = "Allow" - actions = [ - "globalaccelerator:Describe*", - "globalaccelerator:List*", - "guardduty:Describe*", - "guardduty:Get*", - "guardduty:List*", - "launchwizard:Describe*", - "launchwizard:GetWorkload", - "launchwizard:List*", - "license-manager:List*", - "lightsail:Get*", "logs:Describe*", - "logs:List*", - "logs:TestMetricFilter", - "machinelearning:Describe*", - "managedblockchain:GetMember", - "managedblockchain:GetNetwork", - "managedblockchain:GetNode", - "managedblockchain:GetProposal", - "managedblockchain:List*", - "mediaconnect:Describe*", - "mediaconnect:List*", - "mediaconvert:List*", - "medialive:Describe*", - "medialive:Get*", - "medialive:List*", - "mediapackage-vod:Describe*", - "mediapackage-vod:List*", - "mediapackage:Describe*", - "mediapackage:List*", - "mediapackagev2:GetChannel", - "mediapackagev2:GetChannelGroup", - "mediapackagev2:GetChannelPolicy", - "mediapackagev2:GetHeadObject", - "mediapackagev2:GetOriginEndpoint", - "mediapackagev2:GetOriginEndpointPolicy", - "mediapackagev2:List*", - "mediastore:Describe*", - "mediastore:List*", "memorydb:Describe*", "memorydb:List*", "mq:Describe*", "mq:List*", - "network-firewall:Describe*", - "network-firewall:List*", - "networkmanager:DescribeGlobalNetworks", - "networkmanager:Get*", - "networkmanager:List*", "neptune:Describe*", "neptune:List*", - "nimble:List*", - "notifications-contacts:List*", - "notifications:Get*", - "notifications:GetNotificationConfiguration", - "notifications:GetNotificationsAccessForOrganization", - "notifications:List*", - "opsworks-cm:Describe*", - "opsworks-cm:List*", - "opsworks:Describe*", + "network-firewall:Describe*", + "network-firewall:List*", "organizations:Describe*", "organizations:List*", - "outposts:List*", - "personalize:Describe*", - "personalize:List*", - "pi:ListAvailableResourceDimensions", - "pi:ListAvailableResourceMetrics", - "pipes:DescribePipe", - "pipes:ListPipes", - "polly:Describe*", - "polly:List*", - "pricing:DescribeServices", - "pricing:ListPriceLists", - "qbusiness:List*", - "ram:List*", - "rbin:List*", "rds:Describe*", - "rds:List*", - "redshift-serverless:List*", "redshift:Describe*", - "redshift:List*", - "redshift:View*", - "rekognition:Describe*", - "rekognition:List*", - "resource-explorer-2:List*", - "resource-groups:List*", - "resource-groups:Search*", - "robomaker:BatchDescribe*", - "robomaker:Describe*", - "robomaker:List*", - "route53-recovery-cluster:Get*", - "route53-recovery-cluster:List*", - "route53-recovery-control-config:Describe*", - "route53-recovery-control-config:Get*", - "route53-recovery-control-config:List*", - "route53-recovery-readiness:Get*", - "route53-recovery-readiness:List*", - "route53:Get*", "route53:List*", - "route53:Test*", - "route53domains:Check*", - "route53domains:Get*", - "route53domains:List*", - "route53domains:View*", - "route53profiles:Get*", - "route53profiles:List*", - "route53resolver:Get*", - "route53resolver:List*", - "rum:GetAppMonitor", - "rum:ListAppMonitors", - "s3-object-lambda:List*", - "s3-outposts:Get*", - "s3-outposts:List*", - "s3:Describe*", "s3:GetAccountPublicAccessBlock", "s3:GetBucket*", "s3:GetEncryptionConfiguration", "s3:GetLifecycleConfiguration", "s3:GetReplicationConfiguration", - "s3:List*", + "s3:ListAllMyBuckets", "sagemaker:Describe*", "sagemaker:List*", - "scheduler:List*", - "schemas:Describe*", - "schemas:Get*", - "schemas:List*", - "schemas:Search*", - "sdb:List*", "secretsmanager:GetResourcePolicy", "secretsmanager:List*", - "securityhub:Describe*", - "securityhub:List*", - "securitylake:ListDataLakeExceptions", - "securitylake:ListDataLakes", - "securitylake:ListLogSources", - "securitylake:ListSubscribers", - "securitylake:ListTagsForResource", - "serverlessrepo:Get*", - "serverlessrepo:List*", - "serverlessrepo:SearchApplications", - "servicecatalog:Describe*", - "servicecatalog:List*", - "servicecatalog:Scan*", - "servicecatalog:Search*", - "servicediscovery:DiscoverInstances", - "servicediscovery:DiscoverInstancesRevision", - "servicediscovery:List*", - "servicequotas:List*", - "ses:Describe*", - "ses:List*", - "shield:Describe*", - "shield:List*", - "signer:DescribeSigningJob", - "signer:List*", - "signin:ListTrustedIdentityPropagationApplicationsForConsole", - "sms-voice:Describe*", - "sms-voice:List*", - "snowball:Describe*", - "snowball:List*", - "sns:Check*", "sns:GetTopicAttributes", "sns:List*", "sqs:GetQueueAttributes", "sqs:List*", - "ssm-contacts:List*", - "ssm-sap:List*", - "ssm-quicksetup:List*", "ssm:Describe*", "ssm:GetDocument", - "ssm:List*", - "sso-directory:List*", - "sso:List*", - "sso:Search*", - "states:Describe*", + "ssm:ListDocuments", + "states:DescribeStateMachine", "states:List*", - "states:ValidateStateMachineDefinition", - "storagegateway:Describe*", - "storagegateway:List*", - "sts:GetAccessKeyInfo", "sts:GetCallerIdentity", - "sts:GetSessionToken", - "support:Describe*", - "tag:Describe*", - "tag:Get*", - "tax:ListTaxRegistrations", - "timestream:Describe*", - "tnb:List*", - "transcribe:List*", - "transfer:Describe*", - "transfer:List*", - "translate:DescribeTextTranslationJob", - "translate:ListParallelData", - "translate:ListTerminologies", - "translate:ListTextTranslationJobs", - "trustedadvisor:Describe*", - "trustedadvisor:List*", - "verifiedpermissions:List*", - "vpc-lattice:Get*", - "vpc-lattice:List*", - "waf-regional:List*", - "waf:List*", -"wafv2:CheckCapacity", - "wafv2:Describe*", "wafv2:GetWebACL", - "wafv2:List*", - "wellarchitected:ExportLens", - "wellarchitected:Get*", - "wellarchitected:List*", - "workdocs:CheckAlias", - "workdocs:Describe*", - "workmail:Describe*", - "workmail:List*", - "workspaces-web:GetBrowserSettings", - "workspaces-web:GetIdentityProvider", - "workspaces-web:GetNetworkSettings", - "workspaces-web:List*", - "workspaces:Describe*" + "wafv2:List*" ] resources = ["*"] } From ea6c7192bff0614cfd31b9cf99950d53d96e6118 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 8 Sep 2026 10:23:35 +1000 Subject: [PATCH 07/36] fix(aws): enumerate ec2 and iam instead of wildcarding them `ec2:Describe*` and `iam:Get*`/`List*` were kept as wildcards on the argument that both families are inventory metadata. That is wrong for ec2 and too generous for iam. - `ec2:Describe*` covers 192 actions; the collectors call 24. The unused remainder includes `DescribeVpnConnections`, which returns `CustomerGatewayConfiguration` with the IPsec pre-shared key in plaintext -- a live credential into the customer's on-prem network -- plus `DescribeInstanceAttribute` (UserData), `DescribeClientVpnConnections` (client identities and source IPs) and `DescribeTags` (owner emails). - `iam:Get*`/`List*` cover 67; the collectors and admin's OIDC resolver call 25. The remainder includes `GetAccountAuthorizationDetails`, which returns the entire IAM graph in one call, `GetServiceLastAccessedDetails`, which tells an attacker which grants are stale enough to abuse unnoticed, and `GetSAMLProvider`/`GetLoginProfile` for phishing the workforce. Allow 107 -> 152: enumeration is longer than the wildcards it replaces but grants far less. - Deny 15 further actions as defence in depth against a future re-broadening, matching the discipline already applied to Cognito and WorkMail. - Deny `apigateway:GET` on `/apikeys*` and `/usageplans*` via a second, resource-scoped statement. API Gateway exposes one HTTP-verb-shaped action covering its whole control plane, including `GET /apikeys?includeValues=true`; path ARNs are the only way to separate the five calls the collectors make from plaintext API key values. 234/234 required actions still resolve to allowed. Co-Authored-By: Claude Opus 5 (1M context) --- .../nullify-cloudformation-template.json | 80 ++++++++++++++++-- .../modules/nullify-aws-integration/data.tf | 81 +++++++++++++++++-- .../modules/nullify-aws-integration/data.tf | 81 +++++++++++++++++-- 3 files changed, 227 insertions(+), 15 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index c35ade0..6693b52 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -242,10 +242,35 @@ "workmail:ListAliases", "workmail:ListUsers", "workspaces:DescribeWorkspaces", + "ec2:DescribeClientVpnAuthorizationRules", + "ec2:DescribeClientVpnConnections", + "ec2:DescribeInstanceAttribute", + "ec2:DescribeVpnConnections", + "eks:DescribeIdentityProviderConfig", + "iam:GetAccountAuthorizationDetails", + "iam:GetLoginProfile", + "iam:GetOrganizationsAccessReport", + "iam:GetSAMLProvider", + "iam:GetServiceLastAccessedDetails", + "iam:GetServiceLastAccessedDetailsWithEntities", + "iam:ListPoliciesGrantingServiceAccess", + "kms:ListGrants", + "ssm:DescribeSessions", + "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ], "Resource": "*", "Condition": {} + }, + { + "Effect": "Deny", + "Action": "apigateway:GET", + "Resource": [ + "arn:aws:apigateway:*::/apikeys", + "arn:aws:apigateway:*::/apikeys/*", + "arn:aws:apigateway:*::/usageplans", + "arn:aws:apigateway:*::/usageplans/*" + ] } ] } @@ -404,7 +429,30 @@ "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", - "ec2:Describe*", + "ec2:DescribeAddresses", + "ec2:DescribeFlowLogs", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeManagedPrefixLists", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkAcls", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSnapshotAttribute", + "ec2:DescribeSnapshots", + "ec2:DescribeSubnets", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeTransitGatewayRouteTables", + "ec2:DescribeTransitGateways", + "ec2:DescribeVolumes", + "ec2:DescribeVpcEndpoints", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeVpcs", "ec2:GetEbsEncryptionByDefault", "ec2:GetManagedPrefixListEntries", "ec2:GetSnapshotBlockPublicAccessState", @@ -428,8 +476,7 @@ "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", - "glue:GetDevEndpoints", - "glue:GetJobs" + "glue:GetDevEndpoints" ], "Resource": "*", "Condition": {} @@ -451,12 +498,35 @@ { "Effect": "Allow", "Action": [ + "glue:GetJobs", "glue:GetSecurityConfigurations", "guardduty:GetDetector", "guardduty:List*", "iam:GenerateCredentialReport", - "iam:Get*", - "iam:List*", + "iam:GetAccessKeyLastUsed", + "iam:GetAccountPasswordPolicy", + "iam:GetAccountSummary", + "iam:GetCredentialReport", + "iam:GetGroupPolicy", + "iam:GetOpenIDConnectProvider", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:GetUserPolicy", + "iam:ListAccessKeys", + "iam:ListAttachedGroupPolicies", + "iam:ListAttachedRolePolicies", + "iam:ListAttachedUserPolicies", + "iam:ListGroupPolicies", + "iam:ListGroups", + "iam:ListGroupsForUser", + "iam:ListMFADevices", + "iam:ListOpenIDConnectProviders", + "iam:ListPolicies", + "iam:ListRolePolicies", + "iam:ListRoles", + "iam:ListUserPolicies", + "iam:ListUsers", "kafka:List*", "kinesis:Describe*", "kinesis:List*", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 2d51140..db9c99b 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -85,7 +85,30 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", - "ec2:Describe*", + "ec2:DescribeAddresses", + "ec2:DescribeFlowLogs", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeManagedPrefixLists", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkAcls", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSnapshotAttribute", + "ec2:DescribeSnapshots", + "ec2:DescribeSubnets", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeTransitGatewayRouteTables", + "ec2:DescribeTransitGateways", + "ec2:DescribeVolumes", + "ec2:DescribeVpcEndpoints", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeVpcs", "ec2:GetEbsEncryptionByDefault", "ec2:GetManagedPrefixListEntries", "ec2:GetSnapshotBlockPublicAccessState", @@ -109,8 +132,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", - "glue:GetDevEndpoints", - "glue:GetJobs" + "glue:GetDevEndpoints" ] resources = ["*"] } @@ -120,12 +142,35 @@ data "aws_iam_policy_document" "readonly_policy_part2" { statement { effect = "Allow" actions = [ + "glue:GetJobs", "glue:GetSecurityConfigurations", "guardduty:GetDetector", "guardduty:List*", "iam:GenerateCredentialReport", - "iam:Get*", - "iam:List*", + "iam:GetAccessKeyLastUsed", + "iam:GetAccountPasswordPolicy", + "iam:GetAccountSummary", + "iam:GetCredentialReport", + "iam:GetGroupPolicy", + "iam:GetOpenIDConnectProvider", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:GetUserPolicy", + "iam:ListAccessKeys", + "iam:ListAttachedGroupPolicies", + "iam:ListAttachedRolePolicies", + "iam:ListAttachedUserPolicies", + "iam:ListGroupPolicies", + "iam:ListGroups", + "iam:ListGroupsForUser", + "iam:ListMFADevices", + "iam:ListOpenIDConnectProviders", + "iam:ListPolicies", + "iam:ListRolePolicies", + "iam:ListRoles", + "iam:ListUserPolicies", + "iam:ListUsers", "kafka:List*", "kinesis:Describe*", "kinesis:List*", @@ -336,8 +381,34 @@ data "aws_iam_policy_document" "deny_actions_policy" { "workmail:ListAliases", "workmail:ListUsers", "workspaces:DescribeWorkspaces", + "ec2:DescribeClientVpnAuthorizationRules", + "ec2:DescribeClientVpnConnections", + "ec2:DescribeInstanceAttribute", + "ec2:DescribeVpnConnections", + "eks:DescribeIdentityProviderConfig", + "iam:GetAccountAuthorizationDetails", + "iam:GetLoginProfile", + "iam:GetOrganizationsAccessReport", + "iam:GetSAMLProvider", + "iam:GetServiceLastAccessedDetails", + "iam:GetServiceLastAccessedDetailsWithEntities", + "iam:ListPoliciesGrantingServiceAccess", + "kms:ListGrants", + "ssm:DescribeSessions", + "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] } + + statement { + effect = "Deny" + actions = ["apigateway:GET"] + resources = [ + "arn:aws:apigateway:*::/apikeys", + "arn:aws:apigateway:*::/apikeys/*", + "arn:aws:apigateway:*::/usageplans", + "arn:aws:apigateway:*::/usageplans/*", + ] + } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 2d51140..db9c99b 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -85,7 +85,30 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "dynamodb:Describe*", "dynamodb:GetResourcePolicy", "dynamodb:List*", - "ec2:Describe*", + "ec2:DescribeAddresses", + "ec2:DescribeFlowLogs", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeManagedPrefixLists", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkAcls", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSnapshotAttribute", + "ec2:DescribeSnapshots", + "ec2:DescribeSubnets", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeTransitGatewayRouteTables", + "ec2:DescribeTransitGateways", + "ec2:DescribeVolumes", + "ec2:DescribeVpcEndpoints", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeVpcs", "ec2:GetEbsEncryptionByDefault", "ec2:GetManagedPrefixListEntries", "ec2:GetSnapshotBlockPublicAccessState", @@ -109,8 +132,7 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "globalaccelerator:List*", "glue:GetCrawlers", "glue:GetDataCatalogEncryptionSettings", - "glue:GetDevEndpoints", - "glue:GetJobs" + "glue:GetDevEndpoints" ] resources = ["*"] } @@ -120,12 +142,35 @@ data "aws_iam_policy_document" "readonly_policy_part2" { statement { effect = "Allow" actions = [ + "glue:GetJobs", "glue:GetSecurityConfigurations", "guardduty:GetDetector", "guardduty:List*", "iam:GenerateCredentialReport", - "iam:Get*", - "iam:List*", + "iam:GetAccessKeyLastUsed", + "iam:GetAccountPasswordPolicy", + "iam:GetAccountSummary", + "iam:GetCredentialReport", + "iam:GetGroupPolicy", + "iam:GetOpenIDConnectProvider", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:GetUserPolicy", + "iam:ListAccessKeys", + "iam:ListAttachedGroupPolicies", + "iam:ListAttachedRolePolicies", + "iam:ListAttachedUserPolicies", + "iam:ListGroupPolicies", + "iam:ListGroups", + "iam:ListGroupsForUser", + "iam:ListMFADevices", + "iam:ListOpenIDConnectProviders", + "iam:ListPolicies", + "iam:ListRolePolicies", + "iam:ListRoles", + "iam:ListUserPolicies", + "iam:ListUsers", "kafka:List*", "kinesis:Describe*", "kinesis:List*", @@ -336,8 +381,34 @@ data "aws_iam_policy_document" "deny_actions_policy" { "workmail:ListAliases", "workmail:ListUsers", "workspaces:DescribeWorkspaces", + "ec2:DescribeClientVpnAuthorizationRules", + "ec2:DescribeClientVpnConnections", + "ec2:DescribeInstanceAttribute", + "ec2:DescribeVpnConnections", + "eks:DescribeIdentityProviderConfig", + "iam:GetAccountAuthorizationDetails", + "iam:GetLoginProfile", + "iam:GetOrganizationsAccessReport", + "iam:GetSAMLProvider", + "iam:GetServiceLastAccessedDetails", + "iam:GetServiceLastAccessedDetailsWithEntities", + "iam:ListPoliciesGrantingServiceAccess", + "kms:ListGrants", + "ssm:DescribeSessions", + "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ] resources = ["*"] } + + statement { + effect = "Deny" + actions = ["apigateway:GET"] + resources = [ + "arn:aws:apigateway:*::/apikeys", + "arn:aws:apigateway:*::/apikeys/*", + "arn:aws:apigateway:*::/usageplans", + "arn:aws:apigateway:*::/usageplans/*", + ] + } } From 2841258a887ed90df56a39e0005af2caf325de4a Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 8 Sep 2026 10:42:24 +1000 Subject: [PATCH 08/36] fix(aws): correct two invalid action names and a neptune coverage gap IAM accepts a misspelled action in a policy and simply never matches it, so both of these were silent. - `medialive:GetInputDeviceThumbnail` is not an IAM action; the real one is `DescribeInputDeviceThumbnail`. Introduced in 3a1a4fa. Inert today -- nothing grants `medialive:*` any more -- but a dead Deny that a future broad grant would sail straight past. - `neptune:` is not a valid IAM prefix. Neptune's control plane is served by the RDS API and authorised with `rds:` actions, so `neptune:Describe*` and `neptune:List*` granted nothing. Drop both. - That mattered: enumerating the policy mapped `neptune.ListTagsForResource` to the dead `neptune:List*` rather than `rds:ListTagsForResource`, and the earlier `rds:List*` wildcard was gone, so the collector's Neptune tag read would have failed. Grant `rds:ListTagsForResource`. The three Describe calls were already covered by `rds:Describe*`. Every action string in all three templates was checked against AWS's policy generator dataset: 296/297 valid before this commit, and the one miss is fixed here. 231/231 required actions resolve to allowed with Neptune mapped to `rds:`. Co-Authored-By: Claude Opus 5 (1M context) --- .../cloudformation/nullify-cloudformation-template.json | 5 ++--- .../terraform-v6/modules/nullify-aws-integration/data.tf | 5 ++--- .../terraform/modules/nullify-aws-integration/data.tf | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 6693b52..0cb371c 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -204,7 +204,7 @@ "kinesis:SubscribeToShard", "lightsail:GetRelationalDatabaseMasterUserPassword", "logs:StartLiveTail", - "medialive:GetInputDeviceThumbnail", + "medialive:DescribeInputDeviceThumbnail", "pi:DescribeDimensionKeys", "rum:GetAppMonitorData", "aps:QueryMetrics", @@ -543,13 +543,12 @@ "memorydb:List*", "mq:Describe*", "mq:List*", - "neptune:Describe*", - "neptune:List*", "network-firewall:Describe*", "network-firewall:List*", "organizations:Describe*", "organizations:List*", "rds:Describe*", + "rds:ListTagsForResource", "redshift:Describe*", "route53:List*", "s3:GetAccountPublicAccessBlock", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index db9c99b..53f09f8 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -187,13 +187,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "memorydb:List*", "mq:Describe*", "mq:List*", - "neptune:Describe*", - "neptune:List*", "network-firewall:Describe*", "network-firewall:List*", "organizations:Describe*", "organizations:List*", "rds:Describe*", + "rds:ListTagsForResource", "redshift:Describe*", "route53:List*", "s3:GetAccountPublicAccessBlock", @@ -343,7 +342,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "kinesis:SubscribeToShard", "lightsail:GetRelationalDatabaseMasterUserPassword", "logs:StartLiveTail", - "medialive:GetInputDeviceThumbnail", + "medialive:DescribeInputDeviceThumbnail", "pi:DescribeDimensionKeys", "rum:GetAppMonitorData", "aps:QueryMetrics", diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index db9c99b..53f09f8 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -187,13 +187,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "memorydb:List*", "mq:Describe*", "mq:List*", - "neptune:Describe*", - "neptune:List*", "network-firewall:Describe*", "network-firewall:List*", "organizations:Describe*", "organizations:List*", "rds:Describe*", + "rds:ListTagsForResource", "redshift:Describe*", "route53:List*", "s3:GetAccountPublicAccessBlock", @@ -343,7 +342,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "kinesis:SubscribeToShard", "lightsail:GetRelationalDatabaseMasterUserPassword", "logs:StartLiveTail", - "medialive:GetInputDeviceThumbnail", + "medialive:DescribeInputDeviceThumbnail", "pi:DescribeDimensionKeys", "rum:GetAppMonitorData", "aps:QueryMetrics", From aec23c2bdc0e5d53b82fb5eef3424175be8c7af6 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 05:50:40 +1000 Subject: [PATCH 09/36] fix(aws): grant KMS key/* for alias ARNs, pin the collector image and look up clusters in their own region - KMS policy grants kms_key_arn and key/* in the same Nullify account and region; validation accepts key (incl. mrk-) and alias ARNs, like the CloudFormation template. - k8s-resources collector_image defaults to public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0 instead of the unowned Docker Hub nullify/k8s-collector:latest. - New nullify_s3_access_point_arn grants s3:PutObject/PutObjectAcl on /object/k8s-collector/*. - v6 reads each EKS cluster in the region from its ARN; v5 fails fast with a precondition and accepts eks_oidc_issuer_urls for other regions. - multi-cluster example: region-correct lookups and exactly two clusters. - Root drops the unused kubernetes provider and the dead cronjob_schedule pass-through; cross-variable validations become preconditions so the declared Terraform >= 1.5 floor holds. Co-Authored-By: Claude Opus 5 (1M context) --- .../terraform-v6/examples/basic/main.tf | 2 +- .../examples/basic/terraform.tfvars.example | 2 +- .../terraform-v6/examples/basic/variables.tf | 4 +- .../examples/multi-cluster-complete/main.tf | 46 +++++++---- .../terraform.tfvars.example | 11 ++- .../multi-cluster-complete/variables.tf | 20 +++-- aws-integration-setup/terraform-v6/main.tf | 4 +- .../modules/k8s-resources/providers.tf | 2 +- .../modules/k8s-resources/variables.tf | 15 ++-- .../modules/nullify-aws-integration/data.tf | 53 +++++++++---- .../modules/nullify-aws-integration/locals.tf | 13 ++- .../nullify-aws-integration/outputs.tf | 7 +- .../nullify-aws-integration/variables.tf | 29 ++++--- .../nullify-aws-integration/versions.tf | 4 +- .../terraform-v6/providers.tf | 24 ------ .../terraform-v6/terraform.tfvars.example | 10 +-- .../terraform-v6/variables.tf | 25 +++--- .../terraform-v6/versions.tf | 8 +- .../terraform/examples/basic/main.tf | 4 +- .../examples/basic/terraform.tfvars.example | 2 +- .../terraform/examples/basic/variables.tf | 4 +- .../examples/multi-cluster-complete/main.tf | 64 +++++++++++---- .../terraform.tfvars.example | 11 ++- .../multi-cluster-complete/variables.tf | 20 +++-- aws-integration-setup/terraform/main.tf | 5 +- .../modules/k8s-resources/providers.tf | 2 +- .../modules/k8s-resources/variables.tf | 15 ++-- .../modules/nullify-aws-integration/data.tf | 79 +++++++++++++++---- .../modules/nullify-aws-integration/locals.tf | 13 ++- .../nullify-aws-integration/outputs.tf | 7 +- .../nullify-aws-integration/variables.tf | 40 +++++++--- .../nullify-aws-integration/versions.tf | 4 +- aws-integration-setup/terraform/providers.tf | 24 ------ .../terraform/terraform.tfvars.example | 13 ++- aws-integration-setup/terraform/variables.tf | 31 +++++--- aws-integration-setup/terraform/versions.tf | 8 +- 36 files changed, 390 insertions(+), 235 deletions(-) diff --git a/aws-integration-setup/terraform-v6/examples/basic/main.tf b/aws-integration-setup/terraform-v6/examples/basic/main.tf index 97a6ee3..168d981 100644 --- a/aws-integration-setup/terraform-v6/examples/basic/main.tf +++ b/aws-integration-setup/terraform-v6/examples/basic/main.tf @@ -2,7 +2,7 @@ # This example shows the minimal setup for AWS integration without Kubernetes terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { diff --git a/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example index 4e44e69..5ec8c91 100644 --- a/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example @@ -16,7 +16,7 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Tags to apply to all AWS resources diff --git a/aws-integration-setup/terraform-v6/examples/basic/variables.tf b/aws-integration-setup/terraform-v6/examples/basic/variables.tf index 7b4741d..3af486d 100644 --- a/aws-integration-setup/terraform-v6/examples/basic/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/basic/variables.tf @@ -44,6 +44,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index 4195bad..0c386e4 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -13,26 +13,38 @@ terraform { } } +locals { + primary_name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + primary_region = split(":", var.eks_cluster_arns[0])[3] + secondary_name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + secondary_region = split(":", var.eks_cluster_arns[1])[3] + + collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name +} + provider "aws" { region = var.aws_region } data "aws_eks_cluster" "primary" { - name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + region = local.primary_region + name = local.primary_name } -# aws_eks_cluster_auth was removed as a data source in AWS provider v6. -# Use the ephemeral resource instead (requires Terraform >= 1.10). +# The ephemeral token stays out of plan and state (Terraform >= 1.10). ephemeral "aws_eks_cluster_auth" "primary" { - name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + region = local.primary_region + name = local.primary_name } data "aws_eks_cluster" "secondary" { - name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + region = local.secondary_region + name = local.secondary_name } ephemeral "aws_eks_cluster_auth" "secondary" { - name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + region = local.secondary_region + name = local.secondary_name } provider "kubernetes" { @@ -50,15 +62,15 @@ provider "kubernetes" { } module "nullify_aws_integration" { - source = "../../modules/nullify-aws-integration" - customer_name = var.customer_name - external_id = var.external_id - nullify_role_arn = var.nullify_role_arn - aws_region = var.aws_region - s3_bucket_name = var.s3_bucket_name - kms_key_arn = var.kms_key_arn + source = "../../modules/nullify-aws-integration" + customer_name = var.customer_name + external_id = var.external_id + nullify_role_arn = var.nullify_role_arn + aws_region = var.aws_region + s3_bucket_name = var.s3_bucket_name + nullify_s3_access_point_arn = var.nullify_s3_access_point_arn + kms_key_arn = var.kms_key_arn - # Multiple Kubernetes Clusters Configuration enable_kubernetes_integration = true eks_cluster_arns = var.eks_cluster_arns kubernetes_namespace = var.kubernetes_namespace @@ -71,7 +83,7 @@ module "k8s_resources_primary" { kubernetes = kubernetes.primary } iam_role_arn = module.nullify_aws_integration.role_arn - s3_bucket_name = var.s3_bucket_name + s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region kubernetes_namespace = var.kubernetes_namespace @@ -85,10 +97,10 @@ module "k8s_resources_secondary" { kubernetes = kubernetes.secondary } iam_role_arn = module.nullify_aws_integration.role_arn - s3_bucket_name = var.s3_bucket_name + s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example index 7b0e6f5..fc469ac 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example @@ -10,7 +10,7 @@ external_id = "your-external-id-from-nullify" # REQUIRED: Nullify cross-account role ARN (provided by Nullify) nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" -# REQUIRED: EKS cluster ARNs to integrate (must provide at least 2 for this example) +# REQUIRED: exactly two EKS cluster ARNs, primary first. They can be in different regions. eks_cluster_arns = [ "arn:aws:eks:ap-southeast-2:123456789012:cluster/primary-cluster", "arn:aws:eks:us-east-1:123456789012:cluster/secondary-cluster" @@ -22,7 +22,10 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) +# nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" + +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Kubernetes namespace for Nullify resources @@ -31,8 +34,8 @@ kubernetes_namespace = "nullify" # Cron schedule for the collector cronjob_schedule = "0 0 * * *" -# Collector image -collector_image = "nullify/k8s-collector:latest" +# Collector image (defaults to a pinned tag of Nullify's ECR Public image) +# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf index 3b2b42b..b6eb7ad 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf @@ -22,11 +22,11 @@ variable "nullify_role_arn" { variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (can be from different regions)" + description = "ARNs of exactly two EKS clusters, primary first (they can be in different regions). This example wires one Kubernetes provider per cluster; add a provider and module block per extra cluster" validation { - condition = length(var.eks_cluster_arns) > 0 - error_message = "You must provide at least one cluster ARN." + condition = length(var.eks_cluster_arns) == 2 + error_message = "This example deploys to exactly two clusters. For one cluster, use the k8s-resources module with a single Kubernetes provider." } } @@ -42,6 +42,12 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional). When set, the collectors upload through it" + default = "" +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -56,8 +62,8 @@ variable "cronjob_schedule" { variable "collector_image" { type = string - description = "Docker image for the Kubernetes collector" - default = "nullify/k8s-collector:latest" + description = "Container image for the Kubernetes collector" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" } variable "tags" { @@ -72,6 +78,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/main.tf b/aws-integration-setup/terraform-v6/main.tf index d34aa41..e6bc376 100644 --- a/aws-integration-setup/terraform-v6/main.tf +++ b/aws-integration-setup/terraform-v6/main.tf @@ -9,11 +9,11 @@ module "nullify_aws_integration" { # Optional variables with defaults aws_region = var.aws_region s3_bucket_name = var.s3_bucket_name + nullify_s3_access_point_arn = var.nullify_s3_access_point_arn kms_key_arn = var.kms_key_arn enable_kubernetes_integration = var.enable_kubernetes_integration eks_cluster_arns = var.eks_cluster_arns kubernetes_namespace = var.kubernetes_namespace service_account_name = var.service_account_name - cronjob_schedule = var.cronjob_schedule tags = var.tags -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf index 51aeaa9..895c880 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { kubernetes = { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf index bfb724e..9d9c482 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf @@ -11,7 +11,7 @@ variable "service_account_name" { variable "s3_bucket_name" { type = string - description = "The name of the S3 bucket for storing scan results" + description = "The collector's S3 upload target: the bucket name, or the S3 access point ARN Nullify provides" default = "" } @@ -29,8 +29,8 @@ variable "kubernetes_namespace" { variable "collector_image" { type = string - description = "Docker image for the Kubernetes collector" - default = "nullify/k8s-collector:latest" + description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" } variable "cronjob_schedule" { @@ -41,12 +41,17 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" + + validation { + condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/(mrk-)?[a-f0-9-]+|alias/[A-Za-z0-9/_-]+)$", var.kms_key_arn)) + error_message = "Must be empty, a KMS key ARN (arn:aws:kms:::key/) or a KMS alias ARN (arn:aws:kms:::alias/)" + } } variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" default = false -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 46cbf2a..5614ed4 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -1,15 +1,16 @@ data "aws_caller_identity" "current" {} data "aws_eks_cluster" "clusters" { - count = var.enable_kubernetes_integration ? length(var.eks_cluster_arns) : 0 - name = element(split("/", var.eks_cluster_arns[count.index]), length(split("/", var.eks_cluster_arns[count.index])) - 1) + count = var.enable_kubernetes_integration ? length(var.eks_cluster_arns) : 0 + region = split(":", var.eks_cluster_arns[count.index])[3] + name = element(split("/", var.eks_cluster_arns[count.index]), length(split("/", var.eks_cluster_arns[count.index])) - 1) } locals { all_clusters_info = var.enable_kubernetes_integration ? [ for i, cluster in data.aws_eks_cluster.clusters : { oidc_id = split("/", cluster.identity[0].oidc[0].issuer)[4] - region = split(":", var.eks_cluster_arns[i])[3] # Extract region from ARN + region = split(":", var.eks_cluster_arns[i])[3] } ] : [] @@ -21,6 +22,13 @@ locals { } data "aws_iam_policy_document" "assume_role_policy" { + lifecycle { + precondition { + condition = !var.enable_kubernetes_integration || length(var.eks_cluster_arns) > 0 + error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + } + } + statement { effect = "Allow" principals { @@ -512,17 +520,32 @@ data "aws_iam_policy_document" "readonly_policy_part2" { data "aws_iam_policy_document" "s3_access_policy" { count = local.enable_s3_access ? 1 : 0 - statement { - effect = "Allow" - actions = [ - "s3:PutObject", - "s3:ListBucket", - "s3:PutObjectAcl" - ] - resources = [ - local.s3_bucket_arn, - "${local.s3_bucket_arn}/*" - ] + dynamic "statement" { + for_each = var.s3_bucket_name != "" ? [local.s3_bucket_arn] : [] + content { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:ListBucket", + "s3:PutObjectAcl" + ] + resources = [ + statement.value, + "${statement.value}/*" + ] + } + } + + dynamic "statement" { + for_each = var.nullify_s3_access_point_arn != "" ? [var.nullify_s3_access_point_arn] : [] + content { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:PutObjectAcl" + ] + resources = ["${statement.value}/object/k8s-collector/*"] + } } } @@ -538,7 +561,7 @@ data "aws_iam_policy_document" "kms_access_policy" { "kms:ReEncryptFrom", "kms:ReEncryptTo" ] - resources = [var.kms_key_arn] + resources = local.kms_policy_resources } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf index 69a332a..76f998f 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf @@ -17,14 +17,21 @@ locals { oidc_subject = "system:serviceaccount:${var.kubernetes_namespace}:${var.service_account_name}" # S3 configuration - enable_s3_access = var.s3_bucket_name != "" + enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" - # KMS configuration + # KMS configuration. IAM ignores alias ARNs in Resource, so key/* in the same + # account and region is granted too; Nullify's key policy is the real gate. enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + + kms_policy_resources = local.enable_kms_access ? [ + var.kms_key_arn, + "arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*", + ] : [] # Common tags common_tags = merge(var.tags, { Customer = var.customer_name }) -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf index 77fb9e3..121639e 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf @@ -75,6 +75,11 @@ output "policy_arns" { } } +output "kms_policy_resources" { + description = "Resources the KMS policy grants: kms_key_arn and key/* in its account and region (empty when kms_key_arn is not set)" + value = local.kms_policy_resources +} + # Configuration summary output "deployment_summary" { description = "Summary of the Nullify integration deployment" @@ -88,4 +93,4 @@ output "deployment_summary" { kubernetes_integration = var.enable_kubernetes_integration total_clusters_configured = var.enable_kubernetes_integration ? length(local.all_oidc_ids) : 0 } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf index 9d5f937..049f308 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf @@ -30,18 +30,18 @@ variable "nullify_role_arn" { variable "enable_kubernetes_integration" { type = bool - description = "Whether to enable Kubernetes integration resources" + description = "Whether the role trusts the in-cluster collector's service account (IRSA) on the clusters in eks_cluster_arns" default = false } variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (OIDC provider IDs will be fetched automatically)" + description = "ARNs of the EKS clusters whose collector service account the role trusts. Each cluster is looked up in the region in its ARN" default = [] validation { - condition = !var.enable_kubernetes_integration || (var.enable_kubernetes_integration && length(var.eks_cluster_arns) > 0) - error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" } } @@ -57,6 +57,17 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional). When set, the role may also put objects under /object/k8s-collector/*; the s3_bucket_name grants stay" + default = "" + + validation { + condition = var.nullify_s3_access_point_arn == "" || can(regex("^arn:aws:s3:[a-z0-9-]+:[0-9]{12}:accesspoint/[a-z0-9-]{3,50}$", var.nullify_s3_access_point_arn)) + error_message = "Must be empty or an S3 access point ARN: arn:aws:s3:::accesspoint/" + } +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -71,7 +82,7 @@ variable "service_account_name" { variable "cronjob_schedule" { type = string - description = "Cron schedule for the Kubernetes collector job" + description = "Deprecated and unused: set the collector schedule on the k8s-resources module. Kept so callers that pass it keep working" default = "0 0 * * *" } @@ -86,11 +97,11 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. IAM ignores alias ARNs, so the KMS policy grants this ARN and key/* in the same Nullify account and region. Nullify's key policy decides which of those keys the role can use" default = "" validation { - condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/[a-f0-9-]+|alias/.+)$", var.kms_key_arn)) - error_message = "Must be a valid KMS key ARN (key ID or alias format) or empty string. Example: arn:aws:kms:region:account-id:key/key-id or arn:aws:kms:region:account-id:alias/alias-name" + condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/(mrk-)?[a-f0-9-]+|alias/[A-Za-z0-9/_-]+)$", var.kms_key_arn)) + error_message = "Must be empty, a KMS key ARN (arn:aws:kms:::key/) or a KMS alias ARN (arn:aws:kms:::alias/)" } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf index 8be97e6..f13e466 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { @@ -7,4 +7,4 @@ terraform { version = "~> 6.0" } } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/providers.tf b/aws-integration-setup/terraform-v6/providers.tf index 7a1bc9f..c9d7ccb 100644 --- a/aws-integration-setup/terraform-v6/providers.tf +++ b/aws-integration-setup/terraform-v6/providers.tf @@ -1,27 +1,3 @@ -# AWS Provider provider "aws" { region = var.aws_region } - -# Kubernetes Provider - configured for use with EKS -provider "kubernetes" { - # To use this provider, you need to configure it with your EKS cluster details - # Uncomment and configure the following lines for your specific EKS cluster: - - # host = data.aws_eks_cluster.cluster.endpoint - # cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - # token = ephemeral.aws_eks_cluster_auth.cluster.token - - # Or alternatively, use the AWS CLI configuration: - # config_path = "~/.kube/config" - # config_context = "your-eks-context" -} - -# Uncomment these if using the EKS cluster configuration above -# data "aws_eks_cluster" "cluster" { -# name = var.eks_cluster_name -# } -# -# ephemeral "aws_eks_cluster_auth" "cluster" { -# name = var.eks_cluster_name -# } \ No newline at end of file diff --git a/aws-integration-setup/terraform-v6/terraform.tfvars.example b/aws-integration-setup/terraform-v6/terraform.tfvars.example index d9e15b0..0cfe9d3 100644 --- a/aws-integration-setup/terraform-v6/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/terraform.tfvars.example @@ -17,13 +17,16 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) +# nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" + +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Enable Kubernetes integration (set to true if you have EKS clusters) enable_kubernetes_integration = false -# EKS cluster ARNs (required when enable_kubernetes_integration = true) +# EKS cluster ARNs in any region (required when enable_kubernetes_integration = true) # eks_cluster_arns = [ # "arn:aws:eks:ap-southeast-2:123456789012:cluster/my-cluster" # ] @@ -34,9 +37,6 @@ enable_kubernetes_integration = false # Kubernetes service account name # service_account_name = "nullify-k8s-collector-sa" -# Cron schedule for the collector (default: daily at midnight) -# cronjob_schedule = "0 0 * * *" - # Tags to apply to all AWS resources tags = { ManagedBy = "Terraform" diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index 9d5f937..9f924bb 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -30,18 +30,18 @@ variable "nullify_role_arn" { variable "enable_kubernetes_integration" { type = bool - description = "Whether to enable Kubernetes integration resources" + description = "Whether the role trusts the in-cluster collector's service account (IRSA) on the clusters in eks_cluster_arns" default = false } variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (OIDC provider IDs will be fetched automatically)" + description = "ARNs of the EKS clusters whose collector service account the role trusts (required when enable_kubernetes_integration is true). Clusters can be in any region" default = [] validation { - condition = !var.enable_kubernetes_integration || (var.enable_kubernetes_integration && length(var.eks_cluster_arns) > 0) - error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" } } @@ -57,6 +57,12 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional)" + default = "" +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -71,7 +77,7 @@ variable "service_account_name" { variable "cronjob_schedule" { type = string - description = "Cron schedule for the Kubernetes collector job" + description = "Deprecated and unused: this configuration deploys no collector. Set the schedule on the k8s-resources module instead" default = "0 0 * * *" } @@ -86,11 +92,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" - - validation { - condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/[a-f0-9-]+|alias/.+)$", var.kms_key_arn)) - error_message = "Must be a valid KMS key ARN (key ID or alias format) or empty string. Example: arn:aws:kms:region:account-id:key/key-id or arn:aws:kms:region:account-id:alias/alias-name" - } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/versions.tf b/aws-integration-setup/terraform-v6/versions.tf index 2cf4201..f13e466 100644 --- a/aws-integration-setup/terraform-v6/versions.tf +++ b/aws-integration-setup/terraform-v6/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { source = "hashicorp/aws" version = "~> 6.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.20" - } } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/examples/basic/main.tf b/aws-integration-setup/terraform/examples/basic/main.tf index 34d53f9..12b49af 100644 --- a/aws-integration-setup/terraform/examples/basic/main.tf +++ b/aws-integration-setup/terraform/examples/basic/main.tf @@ -2,7 +2,7 @@ # This example shows the minimal setup for AWS integration without Kubernetes terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { @@ -44,4 +44,4 @@ output "role_arn" { output "deployment_summary" { description = "Summary of the deployment" value = module.nullify_aws_integration.deployment_summary -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example b/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example index 4e44e69..5ec8c91 100644 --- a/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example @@ -16,7 +16,7 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Tags to apply to all AWS resources diff --git a/aws-integration-setup/terraform/examples/basic/variables.tf b/aws-integration-setup/terraform/examples/basic/variables.tf index bb4f1a6..3af486d 100644 --- a/aws-integration-setup/terraform/examples/basic/variables.tf +++ b/aws-integration-setup/terraform/examples/basic/variables.tf @@ -44,6 +44,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index b9a1b38..821bebc 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { @@ -13,24 +13,47 @@ terraform { } } +locals { + primary_name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + primary_region = split(":", var.eks_cluster_arns[0])[3] + secondary_name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + secondary_region = split(":", var.eks_cluster_arns[1])[3] + + collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name +} + provider "aws" { region = var.aws_region } +provider "aws" { + alias = "primary" + region = local.primary_region +} + +provider "aws" { + alias = "secondary" + region = local.secondary_region +} + data "aws_eks_cluster" "primary" { - name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + provider = aws.primary + name = local.primary_name } data "aws_eks_cluster_auth" "primary" { - name = element(split("/", var.eks_cluster_arns[0]), length(split("/", var.eks_cluster_arns[0])) - 1) + provider = aws.primary + name = local.primary_name } data "aws_eks_cluster" "secondary" { - name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + provider = aws.secondary + name = local.secondary_name } data "aws_eks_cluster_auth" "secondary" { - name = element(split("/", var.eks_cluster_arns[1]), length(split("/", var.eks_cluster_arns[1])) - 1) + provider = aws.secondary + name = local.secondary_name } provider "kubernetes" { @@ -48,19 +71,26 @@ provider "kubernetes" { } module "nullify_aws_integration" { - source = "../../modules/nullify-aws-integration" - customer_name = var.customer_name - external_id = var.external_id - nullify_role_arn = var.nullify_role_arn - aws_region = var.aws_region - s3_bucket_name = var.s3_bucket_name - kms_key_arn = var.kms_key_arn - - # Multiple Kubernetes Clusters Configuration + source = "../../modules/nullify-aws-integration" + customer_name = var.customer_name + external_id = var.external_id + nullify_role_arn = var.nullify_role_arn + aws_region = var.aws_region + s3_bucket_name = var.s3_bucket_name + nullify_s3_access_point_arn = var.nullify_s3_access_point_arn + kms_key_arn = var.kms_key_arn + enable_kubernetes_integration = true eks_cluster_arns = var.eks_cluster_arns kubernetes_namespace = var.kubernetes_namespace tags = var.tags + + # AWS provider v5 looks clusters up only in the provider's region, so pass + # the issuers read through each cluster's own regional provider. + eks_oidc_issuer_urls = [ + data.aws_eks_cluster.primary.identity[0].oidc[0].issuer, + data.aws_eks_cluster.secondary.identity[0].oidc[0].issuer, + ] } module "k8s_resources_primary" { @@ -69,7 +99,7 @@ module "k8s_resources_primary" { kubernetes = kubernetes.primary } iam_role_arn = module.nullify_aws_integration.role_arn - s3_bucket_name = var.s3_bucket_name + s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region kubernetes_namespace = var.kubernetes_namespace @@ -83,10 +113,10 @@ module "k8s_resources_secondary" { kubernetes = kubernetes.secondary } iam_role_arn = module.nullify_aws_integration.role_arn - s3_bucket_name = var.s3_bucket_name + s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example index 7b0e6f5..fc469ac 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example @@ -10,7 +10,7 @@ external_id = "your-external-id-from-nullify" # REQUIRED: Nullify cross-account role ARN (provided by Nullify) nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" -# REQUIRED: EKS cluster ARNs to integrate (must provide at least 2 for this example) +# REQUIRED: exactly two EKS cluster ARNs, primary first. They can be in different regions. eks_cluster_arns = [ "arn:aws:eks:ap-southeast-2:123456789012:cluster/primary-cluster", "arn:aws:eks:us-east-1:123456789012:cluster/secondary-cluster" @@ -22,7 +22,10 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) +# nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" + +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Kubernetes namespace for Nullify resources @@ -31,8 +34,8 @@ kubernetes_namespace = "nullify" # Cron schedule for the collector cronjob_schedule = "0 0 * * *" -# Collector image -collector_image = "nullify/k8s-collector:latest" +# Collector image (defaults to a pinned tag of Nullify's ECR Public image) +# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf index 3b2b42b..b6eb7ad 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf @@ -22,11 +22,11 @@ variable "nullify_role_arn" { variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (can be from different regions)" + description = "ARNs of exactly two EKS clusters, primary first (they can be in different regions). This example wires one Kubernetes provider per cluster; add a provider and module block per extra cluster" validation { - condition = length(var.eks_cluster_arns) > 0 - error_message = "You must provide at least one cluster ARN." + condition = length(var.eks_cluster_arns) == 2 + error_message = "This example deploys to exactly two clusters. For one cluster, use the k8s-resources module with a single Kubernetes provider." } } @@ -42,6 +42,12 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional). When set, the collectors upload through it" + default = "" +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -56,8 +62,8 @@ variable "cronjob_schedule" { variable "collector_image" { type = string - description = "Docker image for the Kubernetes collector" - default = "nullify/k8s-collector:latest" + description = "Container image for the Kubernetes collector" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" } variable "tags" { @@ -72,6 +78,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/main.tf b/aws-integration-setup/terraform/main.tf index 4dcc792..0a52fa7 100644 --- a/aws-integration-setup/terraform/main.tf +++ b/aws-integration-setup/terraform/main.tf @@ -9,11 +9,12 @@ module "nullify_aws_integration" { # Optional variables with defaults aws_region = var.aws_region s3_bucket_name = var.s3_bucket_name + nullify_s3_access_point_arn = var.nullify_s3_access_point_arn kms_key_arn = var.kms_key_arn enable_kubernetes_integration = var.enable_kubernetes_integration eks_cluster_arns = var.eks_cluster_arns + eks_oidc_issuer_urls = var.eks_oidc_issuer_urls kubernetes_namespace = var.kubernetes_namespace service_account_name = var.service_account_name - cronjob_schedule = var.cronjob_schedule tags = var.tags -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/providers.tf b/aws-integration-setup/terraform/modules/k8s-resources/providers.tf index 51aeaa9..895c880 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/providers.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { kubernetes = { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf index 79acd9f..9d9c482 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf @@ -11,7 +11,7 @@ variable "service_account_name" { variable "s3_bucket_name" { type = string - description = "The name of the S3 bucket for storing scan results" + description = "The collector's S3 upload target: the bucket name, or the S3 access point ARN Nullify provides" default = "" } @@ -29,8 +29,8 @@ variable "kubernetes_namespace" { variable "collector_image" { type = string - description = "Docker image for the Kubernetes collector" - default = "nullify/k8s-collector:latest" + description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" } variable "cronjob_schedule" { @@ -41,12 +41,17 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" + + validation { + condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/(mrk-)?[a-f0-9-]+|alias/[A-Za-z0-9/_-]+)$", var.kms_key_arn)) + error_message = "Must be empty, a KMS key ARN (arn:aws:kms:::key/) or a KMS alias ARN (arn:aws:kms:::alias/)" + } } variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" default = false -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 71a6285..ae04233 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -1,15 +1,30 @@ data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +locals { + lookup_eks_clusters = var.enable_kubernetes_integration && length(var.eks_oidc_issuer_urls) == 0 +} + data "aws_eks_cluster" "clusters" { - count = var.enable_kubernetes_integration ? length(var.eks_cluster_arns) : 0 + count = local.lookup_eks_clusters ? length(var.eks_cluster_arns) : 0 name = element(split("/", var.eks_cluster_arns[count.index]), length(split("/", var.eks_cluster_arns[count.index])) - 1) + + lifecycle { + precondition { + condition = split(":", var.eks_cluster_arns[count.index])[3] == data.aws_region.current.name + error_message = "EKS cluster ${var.eks_cluster_arns[count.index]} is outside this module's AWS provider region (${data.aws_region.current.name}), and AWS provider v5 can only look clusters up in the provider's region. Set eks_oidc_issuer_urls from a data.aws_eks_cluster read through a provider in the cluster's region (see examples/multi-cluster-complete), or use terraform-v6, which looks each cluster up in its own region." + } + } } locals { + eks_oidc_issuer_urls = local.lookup_eks_clusters ? [for cluster in data.aws_eks_cluster.clusters : cluster.identity[0].oidc[0].issuer] : var.eks_oidc_issuer_urls + all_clusters_info = var.enable_kubernetes_integration ? [ - for i, cluster in data.aws_eks_cluster.clusters : { - oidc_id = split("/", cluster.identity[0].oidc[0].issuer)[4] - region = split(":", var.eks_cluster_arns[i])[3] # Extract region from ARN + for i, issuer in local.eks_oidc_issuer_urls : { + oidc_id = split("/", issuer)[4] + region = try(split(":", var.eks_cluster_arns[i])[3], "") } ] : [] @@ -21,6 +36,23 @@ locals { } data "aws_iam_policy_document" "assume_role_policy" { + lifecycle { + precondition { + condition = !var.enable_kubernetes_integration || length(var.eks_cluster_arns) > 0 + error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + } + + precondition { + condition = length(var.eks_oidc_issuer_urls) == 0 || length(var.eks_oidc_issuer_urls) == length(var.eks_cluster_arns) + error_message = "eks_oidc_issuer_urls must have one entry per cluster in eks_cluster_arns, in the same order" + } + + precondition { + condition = alltrue([for i, url in var.eks_oidc_issuer_urls : split("/", url)[2] == "oidc.eks.${try(split(":", var.eks_cluster_arns[i])[3], "")}.amazonaws.com"]) + error_message = "Each eks_oidc_issuer_urls entry must be in the same region as the eks_cluster_arns entry at the same position" + } + } + statement { effect = "Allow" principals { @@ -512,17 +544,32 @@ data "aws_iam_policy_document" "readonly_policy_part2" { data "aws_iam_policy_document" "s3_access_policy" { count = local.enable_s3_access ? 1 : 0 - statement { - effect = "Allow" - actions = [ - "s3:PutObject", - "s3:ListBucket", - "s3:PutObjectAcl" - ] - resources = [ - local.s3_bucket_arn, - "${local.s3_bucket_arn}/*" - ] + dynamic "statement" { + for_each = var.s3_bucket_name != "" ? [local.s3_bucket_arn] : [] + content { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:ListBucket", + "s3:PutObjectAcl" + ] + resources = [ + statement.value, + "${statement.value}/*" + ] + } + } + + dynamic "statement" { + for_each = var.nullify_s3_access_point_arn != "" ? [var.nullify_s3_access_point_arn] : [] + content { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:PutObjectAcl" + ] + resources = ["${statement.value}/object/k8s-collector/*"] + } } } @@ -538,7 +585,7 @@ data "aws_iam_policy_document" "kms_access_policy" { "kms:ReEncryptFrom", "kms:ReEncryptTo" ] - resources = [var.kms_key_arn] + resources = local.kms_policy_resources } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf index 6b4bfaf..76f998f 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf @@ -17,14 +17,21 @@ locals { oidc_subject = "system:serviceaccount:${var.kubernetes_namespace}:${var.service_account_name}" # S3 configuration - enable_s3_access = var.s3_bucket_name != "" + enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" - # KMS configuration + # KMS configuration. IAM ignores alias ARNs in Resource, so key/* in the same + # account and region is granted too; Nullify's key policy is the real gate. enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + + kms_policy_resources = local.enable_kms_access ? [ + var.kms_key_arn, + "arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*", + ] : [] # Common tags common_tags = merge(var.tags, { Customer = var.customer_name }) -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf index e1d2f0b..121639e 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf @@ -75,6 +75,11 @@ output "policy_arns" { } } +output "kms_policy_resources" { + description = "Resources the KMS policy grants: kms_key_arn and key/* in its account and region (empty when kms_key_arn is not set)" + value = local.kms_policy_resources +} + # Configuration summary output "deployment_summary" { description = "Summary of the Nullify integration deployment" @@ -88,4 +93,4 @@ output "deployment_summary" { kubernetes_integration = var.enable_kubernetes_integration total_clusters_configured = var.enable_kubernetes_integration ? length(local.all_oidc_ids) : 0 } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf index 1e5d975..97f9422 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf @@ -30,18 +30,29 @@ variable "nullify_role_arn" { variable "enable_kubernetes_integration" { type = bool - description = "Whether to enable Kubernetes integration resources" + description = "Whether the role trusts the in-cluster collector's service account (IRSA) on the clusters in eks_cluster_arns" default = false } variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (OIDC provider IDs will be fetched automatically)" + description = "ARNs of the EKS clusters whose collector service account the role trusts. Each cluster is looked up through this module's AWS provider, so it must be in the provider's region unless eks_oidc_issuer_urls is set" default = [] validation { - condition = !var.enable_kubernetes_integration || (var.enable_kubernetes_integration && length(var.eks_cluster_arns) > 0) - error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + } +} + +variable "eks_oidc_issuer_urls" { + type = list(string) + description = "OIDC issuer URLs of the clusters in eks_cluster_arns, in the same order (data.aws_eks_cluster..identity[0].oidc[0].issuer). AWS provider v5 can only look a cluster up in the provider's region, so set this when any cluster is in another region. Leave empty to look every cluster up in the provider's region" + default = [] + + validation { + condition = alltrue([for url in var.eks_oidc_issuer_urls : can(regex("^https://oidc\\.eks\\.[a-z0-9-]+\\.amazonaws\\.com/id/[A-Za-z0-9]+$", url))]) + error_message = "Each entry must be an EKS OIDC issuer URL: https://oidc.eks..amazonaws.com/id/" } } @@ -57,6 +68,17 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional). When set, the role may also put objects under /object/k8s-collector/*; the s3_bucket_name grants stay" + default = "" + + validation { + condition = var.nullify_s3_access_point_arn == "" || can(regex("^arn:aws:s3:[a-z0-9-]+:[0-9]{12}:accesspoint/[a-z0-9-]{3,50}$", var.nullify_s3_access_point_arn)) + error_message = "Must be empty or an S3 access point ARN: arn:aws:s3:::accesspoint/" + } +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -71,7 +93,7 @@ variable "service_account_name" { variable "cronjob_schedule" { type = string - description = "Cron schedule for the Kubernetes collector job" + description = "Deprecated and unused: set the collector schedule on the k8s-resources module. Kept so callers that pass it keep working" default = "0 0 * * *" } @@ -86,11 +108,11 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. IAM ignores alias ARNs, so the KMS policy grants this ARN and key/* in the same Nullify account and region. Nullify's key policy decides which of those keys the role can use" default = "" validation { - condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/[a-f0-9-]+|alias/.+)$", var.kms_key_arn)) - error_message = "Must be a valid KMS key ARN (key ID or alias format) or empty string. Example: arn:aws:kms:region:account-id:key/key-id or arn:aws:kms:region:account-id:alias/alias-name" + condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/(mrk-)?[a-f0-9-]+|alias/[A-Za-z0-9/_-]+)$", var.kms_key_arn)) + error_message = "Must be empty, a KMS key ARN (arn:aws:kms:::key/) or a KMS alias ARN (arn:aws:kms:::alias/)" } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf index b6ac9f9..946e64b 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { @@ -7,4 +7,4 @@ terraform { version = "~> 5.0" } } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/providers.tf b/aws-integration-setup/terraform/providers.tf index 986c45e..c9d7ccb 100644 --- a/aws-integration-setup/terraform/providers.tf +++ b/aws-integration-setup/terraform/providers.tf @@ -1,27 +1,3 @@ -# AWS Provider provider "aws" { region = var.aws_region } - -# Kubernetes Provider - configured for use with EKS -provider "kubernetes" { - # To use this provider, you need to configure it with your EKS cluster details - # Uncomment and configure the following lines for your specific EKS cluster: - - # host = data.aws_eks_cluster.cluster.endpoint - # cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - # token = data.aws_eks_cluster_auth.cluster.token - - # Or alternatively, use the AWS CLI configuration: - # config_path = "~/.kube/config" - # config_context = "your-eks-context" -} - -# Uncomment these data sources if using the EKS cluster configuration above -# data "aws_eks_cluster" "cluster" { -# name = var.eks_cluster_name -# } -# -# data "aws_eks_cluster_auth" "cluster" { -# name = var.eks_cluster_name -# } \ No newline at end of file diff --git a/aws-integration-setup/terraform/terraform.tfvars.example b/aws-integration-setup/terraform/terraform.tfvars.example index d9e15b0..d97fa0b 100644 --- a/aws-integration-setup/terraform/terraform.tfvars.example +++ b/aws-integration-setup/terraform/terraform.tfvars.example @@ -17,7 +17,10 @@ aws_region = "ap-southeast-2" # S3 bucket name for storing scan results (optional, provided by Nullify if needed) # s3_bucket_name = "nullify-scan-results-acme-corp" -# KMS key ARN for encryption (optional, provided by Nullify if needed) +# S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) +# nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" + +# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. # kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Enable Kubernetes integration (set to true if you have EKS clusters) @@ -28,15 +31,17 @@ enable_kubernetes_integration = false # "arn:aws:eks:ap-southeast-2:123456789012:cluster/my-cluster" # ] +# OIDC issuer URLs, same order as eks_cluster_arns. Only needed when a cluster is outside aws_region. +# eks_oidc_issuer_urls = [ +# "https://oidc.eks.ap-southeast-2.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" +# ] + # Kubernetes namespace for Nullify resources # kubernetes_namespace = "nullify" # Kubernetes service account name # service_account_name = "nullify-k8s-collector-sa" -# Cron schedule for the collector (default: daily at midnight) -# cronjob_schedule = "0 0 * * *" - # Tags to apply to all AWS resources tags = { ManagedBy = "Terraform" diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index 1e5d975..b395f47 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -30,21 +30,27 @@ variable "nullify_role_arn" { variable "enable_kubernetes_integration" { type = bool - description = "Whether to enable Kubernetes integration resources" + description = "Whether the role trusts the in-cluster collector's service account (IRSA) on the clusters in eks_cluster_arns" default = false } variable "eks_cluster_arns" { type = list(string) - description = "List of ARNs of EKS clusters to integrate with (OIDC provider IDs will be fetched automatically)" + description = "ARNs of the EKS clusters whose collector service account the role trusts (required when enable_kubernetes_integration is true). Clusters must be in aws_region unless eks_oidc_issuer_urls is set" default = [] validation { - condition = !var.enable_kubernetes_integration || (var.enable_kubernetes_integration && length(var.eks_cluster_arns) > 0) - error_message = "When Kubernetes integration is enabled, you must provide at least one cluster ARN in eks_cluster_arns" + condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" } } +variable "eks_oidc_issuer_urls" { + type = list(string) + description = "OIDC issuer URLs of the clusters in eks_cluster_arns, in the same order. Needed only when a cluster is outside aws_region: aws eks describe-cluster --region --name --query cluster.identity.oidc.issuer --output text" + default = [] +} + variable "aws_region" { type = string description = "The AWS region where resources are deployed" @@ -57,6 +63,12 @@ variable "s3_bucket_name" { default = "" } +variable "nullify_s3_access_point_arn" { + type = string + description = "The S3 access point ARN Nullify provides as the collector upload target (optional)" + default = "" +} + variable "kubernetes_namespace" { type = string description = "The Kubernetes namespace for Nullify resources" @@ -71,7 +83,7 @@ variable "service_account_name" { variable "cronjob_schedule" { type = string - description = "Cron schedule for the Kubernetes collector job" + description = "Deprecated and unused: this configuration deploys no collector. Set the schedule on the k8s-resources module instead" default = "0 0 * * *" } @@ -86,11 +98,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed)" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" - - validation { - condition = var.kms_key_arn == "" || can(regex("^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/[a-f0-9-]+|alias/.+)$", var.kms_key_arn)) - error_message = "Must be a valid KMS key ARN (key ID or alias format) or empty string. Example: arn:aws:kms:region:account-id:key/key-id or arn:aws:kms:region:account-id:alias/alias-name" - } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform/versions.tf b/aws-integration-setup/terraform/versions.tf index 059f306..946e64b 100644 --- a/aws-integration-setup/terraform/versions.tf +++ b/aws-integration-setup/terraform/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5" required_providers { aws = { source = "hashicorp/aws" version = "~> 5.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.20" - } } -} \ No newline at end of file +} From 0c515b3c9fdcc62bd75abd1513cb1c17bef4fade Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 05:52:01 +1000 Subject: [PATCH 10/36] docs(aws): fix Terraform README drift and document KMS, access point and region handling - k8s-resources file is providers.tf; basic quick start copies its own tfvars; collector_image is a k8s-resources variable, not a root one. - Module sources point at Nullify-Platform/nullify-cloud-connector. - v6 README no longer claims data.aws_eks_cluster_auth was removed. - Requirements (Terraform >= 1.5) and upgrade notes. Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform-v6/README.md | 40 +++---- aws-integration-setup/terraform/README.md | 112 +++++++++++++------ 2 files changed, 92 insertions(+), 60 deletions(-) diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index 697e5c0..7c61be6 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -2,35 +2,26 @@ This directory mirrors `../terraform/` but targets **AWS Terraform provider v6** (`~> 6.0`). Use this if your project has already upgraded to AWS provider v6. -For AWS provider v5, use `../terraform/` instead. +For AWS provider v5, use `../terraform/` instead. `../terraform/README.md` documents the modules, variables and outputs; this file covers what differs. -## What Changed from v5 +## What Differs from v5 -The only breaking change that affects this configuration: - -**`data "aws_eks_cluster_auth"` was removed in AWS provider v6.** It is replaced by the `ephemeral` resource variant, which requires Terraform >= 1.10: +- **Per-resource `region`.** AWS provider v6 accepts a `region` argument on most resources and data sources. `nullify-aws-integration` reads each EKS cluster in the region from its ARN, so `eks_cluster_arns` can mix regions with one provider. There is no `eks_oidc_issuer_urls` variable here. +- **Ephemeral EKS token.** `examples/multi-cluster-complete` reads the cluster token with `ephemeral "aws_eks_cluster_auth"`, which keeps it out of plan and state and needs Terraform >= 1.10: ```hcl -# v5 (removed in v6) -data "aws_eks_cluster_auth" "primary" { - name = "..." -} -token = data.aws_eks_cluster_auth.primary.token - -# v6 ephemeral "aws_eks_cluster_auth" "primary" { - name = "..." + region = "eu-west-1" + name = "..." } token = ephemeral.aws_eks_cluster_auth.primary.token ``` -This change is applied in `examples/multi-cluster-complete/main.tf`. The core `nullify-aws-integration` module and `k8s-resources` module are unaffected. - ## Requirements -- Terraform >= 1.10 (for ephemeral resource support in the multi-cluster example) +- Terraform >= 1.5 (>= 1.10 for `examples/multi-cluster-complete`) - AWS provider ~> 6.0 -- Kubernetes provider ~> 2.20 +- Kubernetes provider ~> 2.20 (only `modules/k8s-resources` and `examples/multi-cluster-complete`) ## Architecture @@ -51,8 +42,8 @@ terraform-v6/ │ └── outputs.tf ├── examples/ │ ├── basic/ # AWS IAM only example -│ └── multi-cluster-complete/ # Full multi-cluster EKS example -├── versions.tf # AWS ~> 6.0, Kubernetes ~> 2.20 +│ └── multi-cluster-complete/ # Two-cluster EKS example +├── versions.tf # AWS ~> 6.0 ├── variables.tf ├── main.tf ├── outputs.tf @@ -91,13 +82,16 @@ terraform init && terraform apply - `aws_region`: AWS region for IAM resources (default: ap-southeast-2) - `s3_bucket_name`: S3 bucket for scan results (optional) -- `kms_key_arn`: KMS key ARN for key management operations (optional) +- `nullify_s3_access_point_arn`: S3 access point for collector uploads (optional) +- `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page (optional); see "KMS" in `../terraform/README.md` - `enable_kubernetes_integration`: Set to `true` for EKS integration -- `eks_cluster_arns`: List of EKS cluster ARNs to integrate with +- `eks_cluster_arns`: List of EKS cluster ARNs to integrate with, in any region - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) -- `cronjob_schedule`: Cron schedule for data collection (default: "0 0 * * *") -- `collector_image`: Docker image for collector (default: nullify/k8s-collector:latest) +- `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) - `tags`: Resource tags +- `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` + +`k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0`. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. ## Validation diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 58cf8b9..d7bfd16 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -2,10 +2,18 @@ This Terraform configuration provides modular components for integrating with Nullify's security scanning platform, supporting both AWS-only and full EKS cluster integrations. +This directory targets AWS provider v5. If your project uses AWS provider v6, use `../terraform-v6/`. + ## ⚠️ Important Notice **EKS Integration**: The AWS integration module only sets up IAM roles and trust policies. For full EKS integration, you must also deploy the Kubernetes resources using the separate `k8s-resources` module or Helm charts. +## Requirements + +- Terraform >= 1.5 +- AWS provider `~> 5.0` +- Kubernetes provider `~> 2.20` (only `modules/k8s-resources` and `examples/multi-cluster-complete`) + ## Architecture The configuration is organized into separate, focused modules: @@ -17,18 +25,19 @@ terraform/ │ │ ├── versions.tf # AWS provider requirements │ │ ├── variables.tf # Module input variables │ │ ├── locals.tf # Module local values -│ │ ├── data.tf # Data sources and policies +│ │ ├── data.tf # Data sources and policies │ │ ├── main.tf # Core IAM resources │ │ └── outputs.tf # Module outputs │ └── k8s-resources/ # Kubernetes resources only -│ ├── versions.tf # Kubernetes provider requirements +│ ├── providers.tf # Kubernetes provider requirements │ ├── variables.tf # Module input variables │ ├── main.tf # Kubernetes resources │ └── outputs.tf # Module outputs ├── examples/ │ ├── basic/ # AWS IAM only example -│ └── multi-cluster-complete/ # Full multi-cluster EKS example +│ └── multi-cluster-complete/ # Two-cluster EKS example ├── versions.tf # Root provider requirements +├── providers.tf # AWS provider ├── variables.tf # Root input variables ├── main.tf # Module instantiation ├── outputs.tf # Root outputs @@ -41,7 +50,7 @@ terraform/ - **AWS Module**: Creates IAM roles with multi-cluster OIDC trust policies - **K8s Module**: Deploys collector cronjob and RBAC to any cluster - **Independent Deployment**: Deploy AWS resources once, K8s resources per cluster -- **Multi-Region Support**: Automatic region detection from cluster ARNs +- **Multi-Region Support**: Region taken from each cluster ARN - **Simplified Management**: Each module has focused responsibilities ## Examples @@ -52,29 +61,35 @@ terraform/ - Minimal configuration for cloud-only scanning ### **Multi-Cluster Complete** (`examples/multi-cluster-complete/`) -- Full AWS IAM + multi-cluster EKS integration -- Supports clusters from different regions +- Full AWS IAM + EKS integration for exactly two clusters +- Clusters can be in different regions: each is read through an AWS provider in its own region - Automatic OIDC provider discovery -- Single deployment handles multiple clusters ## Multi-Cluster Support -The architecture now supports multiple EKS clusters seamlessly: +One IAM role can trust the collector on several clusters: ```hcl -# Supports clusters from different regions eks_cluster_arns = [ "arn:aws:eks:us-west-2:123456789012:cluster/prod-cluster", - "arn:aws:eks:eu-west-1:123456789012:cluster/eu-cluster", "arn:aws:eks:us-west-2:123456789012:cluster/staging-cluster" ] ``` -**Features:** -- Automatic region extraction from cluster ARNs -- Dynamic OIDC provider discovery -- Multi-region trust policy generation -- Single IAM role trusts all specified clusters +AWS provider v5 can only look an EKS cluster up in the provider's own region. The module therefore fails at plan time if a cluster is outside that region, unless you pass `eks_oidc_issuer_urls` (one per cluster, same order): + +```hcl +eks_cluster_arns = [ + "arn:aws:eks:us-west-2:123456789012:cluster/prod-cluster", + "arn:aws:eks:eu-west-1:123456789012:cluster/eu-cluster" +] +eks_oidc_issuer_urls = [ + "https://oidc.eks.us-west-2.amazonaws.com/id/EXAMPLE1111111111111111111111111", + "https://oidc.eks.eu-west-1.amazonaws.com/id/EXAMPLE2222222222222222222222222" +] +``` + +Get an issuer with `aws eks describe-cluster --region --name --query cluster.identity.oidc.issuer --output text`, or read it through a regional provider as `examples/multi-cluster-complete` does. `../terraform-v6/` looks every cluster up in its own region without this variable. ## Quick Start @@ -82,7 +97,7 @@ eks_cluster_arns = [ ```bash cd examples/basic/ -cp ../../terraform.tfvars.example terraform.tfvars +cp terraform.tfvars.example terraform.tfvars # Edit terraform.tfvars with your values terraform init && terraform apply ``` @@ -104,18 +119,29 @@ terraform init && terraform apply ## EKS Integration Variables +- `enable_kubernetes_integration`: Set to `true` to trust the in-cluster collector's service account - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with -- `enable_kubernetes_integration`: Set to `true` for EKS integration +- `eks_oidc_issuer_urls`: OIDC issuer per cluster, only needed for clusters outside `aws_region` ## Optional Variables - `aws_region`: AWS region for IAM resources (default: ap-southeast-2) - `s3_bucket_name`: S3 bucket for scan results (optional) -- `kms_key_arn`: KMS key ARN for key management operations (optional, provided by Nullify configure page if needed) +- `nullify_s3_access_point_arn`: S3 access point for collector uploads (optional). Grants `s3:PutObject` and `s3:PutObjectAcl` on `/object/k8s-collector/*` next to the bucket grants +- `kms_key_arn`: KMS ARN from the Nullify configure page (optional), see [KMS](#kms) - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) -- `cronjob_schedule`: Cron schedule for data collection (default: "0 0 * * *") -- `collector_image`: Docker image for collector (default: nullify/k8s-collector:latest) +- `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) - `tags`: Resource tags +- `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` + +## KMS + +`kms_key_arn` accepts either form shown on the Nullify configure page: + +- a key ARN: `arn:aws:kms:::key/`, including multi-Region `key/mrk-...` keys +- an alias ARN: `arn:aws:kms:::alias/` + +IAM ignores alias ARNs in a policy's `Resource`, so the KMS policy grants the ARN you pass and `arn:aws:kms:::key/*` in the same Nullify account and region. Nullify's key policy is the real gate: the role can only use Nullify keys whose key policy allows it. The `kms_policy_resources` module output lists both resources. ## Module Usage @@ -123,15 +149,15 @@ terraform init && terraform apply ```hcl module "nullify_aws_integration" { source = "./modules/nullify-aws-integration" - + customer_name = "your-company" external_id = "your-external-id" nullify_role_arn = "arn:aws:iam::NULLIFY-ACCOUNT:role/role-name" - + # Optional configurations - kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Optional - s3_bucket_name = "your-scan-results-bucket" # Optional - + kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" + s3_bucket_name = "your-scan-results-bucket" + # Optional EKS integration enable_kubernetes_integration = true eks_cluster_arns = [ @@ -144,19 +170,23 @@ module "nullify_aws_integration" { ```hcl module "k8s_resources" { source = "./modules/k8s-resources" - + providers = { kubernetes = kubernetes.cluster_a } - + iam_role_arn = module.nullify_aws_integration.role_arn - s3_bucket_name = "my-scan-results-bucket" - kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Optional + s3_bucket_name = "my-scan-results-bucket" # or the S3 access point ARN from Nullify + kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" aws_region = "us-west-2" - enable_debug = false # Set to true to enable debug logging + enable_debug = false } ``` +`k8s-resources` variables include `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. + +`collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0`, a pinned tag of Nullify's ECR Public image. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. + ## Outputs ### AWS Integration Module @@ -164,33 +194,42 @@ module "k8s_resources" { - `deployment_summary`: Complete deployment information - `cluster_integration_summary`: Multi-cluster setup details - `all_oidc_ids`: List of OIDC provider IDs +- `kms_policy_resources`: Resources granted by the KMS policy -### K8s Resources Module +### K8s Resources Module - `namespace_name`: Created namespace name - `service_account_name`: Service account name +- `cluster_role_name`, `cluster_role_binding_name`: Collector RBAC names - `cronjob_name`: Data collector cronjob name ## Using Modules in Other Projects -Reference the modules in other Terraform projects: +Reference the modules from this repository, pinned to a release tag: ```hcl module "nullify_aws" { - source = "git::https://github.com/your-org/nullify-terraform.git//terraform/modules/nullify-aws-integration?ref=v1.0.0" - + source = "git::https://github.com/Nullify-Platform/nullify-cloud-connector.git//aws-integration-setup/terraform/modules/nullify-aws-integration?ref=" + customer_name = "my-company" external_id = var.external_id # ... other variables } module "nullify_k8s" { - source = "git::https://github.com/your-org/nullify-terraform.git//terraform/modules/k8s-resources?ref=v1.0.0" - + source = "git::https://github.com/Nullify-Platform/nullify-cloud-connector.git//aws-integration-setup/terraform/modules/k8s-resources?ref=" + iam_role_arn = module.nullify_aws.role_arn # ... other variables } ``` +## Upgrade Notes + +- Terraform >= 1.5 is now required (the module already used cross-variable validation, which needs 1.9; those checks are now preconditions). +- The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. +- A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. +- The KMS policy now also grants `key/*` in the account and region of `kms_key_arn`. + ## Security Considerations - External ID provides additional cross-account security @@ -215,6 +254,5 @@ terraform show -json | jq '.values.root_module.resources[].values' ## Support -- Module documentation: `./modules/*/README.md` - Example configurations: `./examples/*/` - Contact Nullify support for integration assistance From 9ac0299e14fcb18cc7e9667e3d043081d06185ed Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 05:52:01 +1000 Subject: [PATCH 11/36] ci: validate terraform-v6 and run each AWS row at its declared Terraform floor Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 43 +++++++++++++++++------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index 5a8e70f..085cbd7 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -27,27 +27,46 @@ permissions: jobs: terraform: + name: terraform (${{ matrix.directory }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - directory: - - gcp-integration-setup/terraform - - gcp-integration-setup/terraform/examples/organization - - gcp-integration-setup/terraform/examples/folder - - gcp-integration-setup/terraform/examples/single-project - - aws-integration-setup/terraform - - aws-integration-setup/terraform/examples/basic - - aws-integration-setup/terraform/examples/multi-cluster-complete - - azure-integration-setup/terraform - - azure-integration-setup/terraform/examples/management-group - - azure-integration-setup/terraform/examples/subscriptions + # AWS rows run the declared floor (Terraform 1.5); the v6 + # multi-cluster example uses ephemeral resources, which need 1.10. + include: + - directory: gcp-integration-setup/terraform + terraform: 1.9.5 + - directory: gcp-integration-setup/terraform/examples/organization + terraform: 1.9.5 + - directory: gcp-integration-setup/terraform/examples/folder + terraform: 1.9.5 + - directory: gcp-integration-setup/terraform/examples/single-project + terraform: 1.9.5 + - directory: aws-integration-setup/terraform + terraform: 1.5.7 + - directory: aws-integration-setup/terraform/examples/basic + terraform: 1.5.7 + - directory: aws-integration-setup/terraform/examples/multi-cluster-complete + terraform: 1.5.7 + - directory: aws-integration-setup/terraform-v6 + terraform: 1.5.7 + - directory: aws-integration-setup/terraform-v6/examples/basic + terraform: 1.5.7 + - directory: aws-integration-setup/terraform-v6/examples/multi-cluster-complete + terraform: 1.10.5 + - directory: azure-integration-setup/terraform + terraform: 1.9.5 + - directory: azure-integration-setup/terraform/examples/management-group + terraform: 1.9.5 + - directory: azure-integration-setup/terraform/examples/subscriptions + terraform: 1.9.5 steps: - uses: actions/checkout@v4 - uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.9.5 + terraform_version: ${{ matrix.terraform }} terraform_wrapper: false - name: terraform fmt -check From a0b117d2c8373c2b778d958ceba8fac4fb43c85f Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 06:03:47 +1000 Subject: [PATCH 12/36] feat(aws): add Terraform modules for managed EKS scan access - modules/eks-managed-scan-access (v5 and v6): one STANDARD access entry per cluster, rbac by default (group nullify-readonly) or AmazonEKSAdminViewPolicy at cluster scope as a warned opt-in. Preconditions reject CONFIG_MAP clusters and cross-account clusters (v5: clusters outside the provider region). Entries are replaced when the role's unique_id changes. A check block warns when Nullify's regional egress IPs are missing from publicAccessCidrs; the module never manages them and outputs a merge command instead. - k8s-resources: enable_collector (default true, moved blocks keep state) and enable_managed_scan_rbac, a list-only nullify-readonly ClusterRole on the 26 scanned kinds plus get /version, bound to the group. - nullify-aws-integration: role_unique_id output. - terraform test suites with mocked providers for all three modules. Co-Authored-By: Claude Opus 5 (1M context) --- .../modules/eks-managed-scan-access/main.tf | 99 ++++++++ .../eks-managed-scan-access/outputs.tf | 41 +++ .../tests/eks_managed_scan_access.tftest.hcl | 205 +++++++++++++++ .../eks-managed-scan-access/variables.tf | 63 +++++ .../eks-managed-scan-access/versions.tf | 10 + .../modules/k8s-resources/main.tf | 31 ++- .../modules/k8s-resources/managed_scan.tf | 82 ++++++ .../modules/k8s-resources/moved.tf | 24 ++ .../modules/k8s-resources/outputs.tf | 37 ++- .../tests/k8s_resources.tftest.hcl | 105 ++++++++ .../modules/k8s-resources/variables.tf | 38 ++- .../nullify-aws-integration/outputs.tf | 5 + .../tests/nullify_aws_integration.tftest.hcl | 114 +++++++++ .../modules/eks-managed-scan-access/main.tf | 103 ++++++++ .../eks-managed-scan-access/outputs.tf | 41 +++ .../tests/eks_managed_scan_access.tftest.hcl | 234 ++++++++++++++++++ .../eks-managed-scan-access/variables.tf | 63 +++++ .../eks-managed-scan-access/versions.tf | 10 + .../terraform/modules/k8s-resources/main.tf | 29 ++- .../modules/k8s-resources/managed_scan.tf | 82 ++++++ .../terraform/modules/k8s-resources/moved.tf | 24 ++ .../modules/k8s-resources/outputs.tf | 35 ++- .../tests/k8s_resources.tftest.hcl | 105 ++++++++ .../modules/k8s-resources/variables.tf | 38 ++- .../nullify-aws-integration/outputs.tf | 5 + .../tests/nullify_aws_integration.tftest.hcl | 134 ++++++++++ 26 files changed, 1721 insertions(+), 36 deletions(-) create mode 100644 aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf create mode 100644 aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf create mode 100644 aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl create mode 100644 aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf create mode 100644 aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf create mode 100644 aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf create mode 100644 aws-integration-setup/terraform-v6/modules/k8s-resources/moved.tf create mode 100644 aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl create mode 100644 aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl create mode 100644 aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf create mode 100644 aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf create mode 100644 aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl create mode 100644 aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf create mode 100644 aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf create mode 100644 aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf create mode 100644 aws-integration-setup/terraform/modules/k8s-resources/moved.tf create mode 100644 aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl create mode 100644 aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf new file mode 100644 index 0000000..5a8cc5e --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -0,0 +1,99 @@ +locals { + nullify_egress_cidrs_by_region = { + "ap-southeast-2" = ["13.55.32.104/32", "3.105.146.106/32", "13.211.99.100/32"] + "eu-central-1" = ["18.198.60.231/32", "18.157.227.250/32", "18.185.152.197/32"] + "us-east-2" = ["52.15.146.50/32", "16.58.40.80/32", "3.133.15.210/32"] + } + + nullify_egress_cidrs = local.nullify_egress_cidrs_by_region[var.nullify_region] + principal_account = split(":", var.principal_arn)[4] + + clusters = { + for arn in var.cluster_arns : arn => { + name = split("/", arn)[1] + region = split(":", arn)[3] + account = split(":", arn)[4] + } + } +} + +data "aws_eks_cluster" "this" { + for_each = local.clusters + + region = each.value.region + name = each.value.name + + lifecycle { + precondition { + condition = each.value.account == local.principal_account + error_message = "Cluster ${each.key} is in account ${each.value.account}, but principal_arn is in account ${local.principal_account}. Nullify assumes the integration role in the cluster's own account: deploy nullify-aws-integration there and pass that role." + } + } +} + +resource "terraform_data" "principal_identity" { + input = var.principal_unique_id +} + +resource "aws_eks_access_entry" "nullify" { + for_each = local.clusters + + region = each.value.region + cluster_name = data.aws_eks_cluster.this[each.key].name + principal_arn = var.principal_arn + type = "STANDARD" + kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : null + tags = var.tags + + lifecycle { + replace_triggered_by = [terraform_data.principal_identity] + + precondition { + condition = contains(["API", "API_AND_CONFIG_MAP"], try(data.aws_eks_cluster.this[each.key].access_config[0].authentication_mode, "CONFIG_MAP")) + error_message = "Cluster ${each.value.name} (${each.value.region}) uses authentication mode CONFIG_MAP, which does not support access entries. Switch it (one-way) with: aws eks update-cluster-config --region ${each.value.region} --name ${each.value.name} --access-config authenticationMode=API_AND_CONFIG_MAP. Or remove it from cluster_arns and map the role in aws-auth instead (see README)." + } + } +} + +resource "aws_eks_access_policy_association" "admin_view" { + for_each = { for key, cluster in local.clusters : key => cluster if var.authorization == "admin_view_policy" } + + region = each.value.region + cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name + principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + + access_scope { + type = "cluster" + } + + lifecycle { + replace_triggered_by = [aws_eks_access_entry.nullify[each.key].access_entry_arn] + } +} + +locals { + endpoints = { + for key, cluster in data.aws_eks_cluster.this : key => { + public_access = cluster.vpc_config[0].endpoint_public_access + cidrs = cluster.vpc_config[0].public_access_cidrs + missing_cidrs = [for cidr in local.nullify_egress_cidrs : cidr if !contains(cluster.vpc_config[0].public_access_cidrs, cidr) && !contains(cluster.vpc_config[0].public_access_cidrs, "0.0.0.0/0")] + } + } + + unreachable_clusters = [for key, endpoint in local.endpoints : key if !endpoint.public_access || length(endpoint.missing_cidrs) > 0] +} + +check "nullify_can_reach_cluster_endpoints" { + assert { + condition = length(local.unreachable_clusters) == 0 + error_message = "Nullify's ${var.nullify_region} egress IPs cannot reach the public endpoint of: ${join(", ", local.unreachable_clusters)}. This module does not change publicAccessCidrs; see the endpoint_allowlist output for the missing CIDRs and an update command. The check matches exact CIDRs, so a wider range that already covers Nullify's IPs also warns." + } +} + +check "admin_view_policy_is_broader_than_required" { + assert { + condition = var.authorization != "admin_view_policy" + error_message = "authorization = admin_view_policy associates AmazonEKSAdminViewPolicy: get, list and watch on every resource, including Secrets, custom resources and pods/log. On EKS 1.34 and earlier get pods/exec is enough to exec into pods, and these grants do not show in kubectl auth can-i --list. The default rbac mode grants list on only the kinds Nullify reads." + } +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf new file mode 100644 index 0000000..6e6d1bc --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf @@ -0,0 +1,41 @@ +output "access_entry_arns" { + description = "Access entry ARN per cluster ARN" + value = { for key, entry in aws_eks_access_entry.nullify : key => entry.access_entry_arn } +} + +output "authorization" { + description = "How Kubernetes authorizes Nullify: rbac or admin_view_policy" + value = var.authorization +} + +output "kubernetes_group_name" { + description = "Group the RBAC binding must name (null for admin_view_policy)" + value = var.authorization == "rbac" ? var.kubernetes_group_name : null +} + +output "nullify_egress_cidrs" { + description = "Nullify egress CIDRs for nullify_region that each cluster's public endpoint must admit" + value = local.nullify_egress_cidrs +} + +output "clusters_for_nullify" { + description = "Clusters to register in Nullify" + value = [ + for key, cluster in local.clusters : { + arn = key + name = cluster.name + region = cluster.region + } + ] +} + +output "endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, the CIDRs to add, and an AWS CLI command that adds them to the existing list. This module never changes publicAccessCidrs" + value = { + for key, endpoint in local.endpoints : key => { + public_endpoint_enabled = endpoint.public_access + missing_cidrs = endpoint.missing_cidrs + update_command = endpoint.public_access && length(endpoint.missing_cidrs) > 0 ? format("aws eks update-cluster-config --region %s --name %s --resources-vpc-config '%s'", local.clusters[key].region, local.clusters[key].name, jsonencode({ publicAccessCidrs = concat(endpoint.cidrs, endpoint.missing_cidrs) })) : null + } + } +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl new file mode 100644 index 0000000..24d3da7 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -0,0 +1,205 @@ +mock_provider "aws" { + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API_AND_CONFIG_MAP" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +mock_provider "aws" { + alias = "config_map" + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "CONFIG_MAP" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +mock_provider "aws" { + alias = "restricted_endpoint" + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["10.0.0.0/8", "18.198.60.231/32"] + }] + } + } +} + +mock_provider "aws" { + alias = "private_endpoint" + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API" + }] + vpc_config = [{ + endpoint_public_access = false + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +variables { + principal_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + principal_unique_id = "AROAEXAMPLEUNIQUEID01" + nullify_region = "eu-central-1" + cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] +} + +run "rbac_is_the_default" { + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "Expected one access entry per cluster" + } + + assert { + condition = aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].region == "eu-west-1" + error_message = "The access entry must be created in the cluster's own region" + } + + assert { + condition = toset(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == toset(["nullify-readonly"]) + error_message = "rbac mode must put the entry in the nullify-readonly group" + } + + assert { + condition = length(aws_eks_access_policy_association.admin_view) == 0 + error_message = "rbac mode must not associate any access policy" + } + + assert { + condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null + error_message = "An endpoint open to 0.0.0.0/0 needs no update" + } + + assert { + condition = output.kubernetes_group_name == "nullify-readonly" + error_message = "kubernetes_group_name output must name the bound group" + } +} + +run "config_map_clusters_are_rejected" { + command = plan + + providers = { + aws = aws.config_map + } + + expect_failures = [aws_eks_access_entry.nullify] +} + +run "restricted_endpoint_warns_and_prints_the_missing_cidrs" { + command = plan + + providers = { + aws = aws.restricted_endpoint + } + + expect_failures = [check.nullify_can_reach_cluster_endpoints] + + assert { + condition = length(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].missing_cidrs) == 2 + error_message = "Only the two absent eu-central-1 CIDRs should be missing" + } + + assert { + condition = !contains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].missing_cidrs, "18.198.60.231/32") + error_message = "A CIDR already in publicAccessCidrs must not be reported missing" + } + + assert { + condition = strcontains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command, "10.0.0.0/8") && strcontains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command, "18.185.152.197/32") + error_message = "The update command must keep existing CIDRs and add the missing ones" + } +} + +run "private_endpoint_warns_without_a_command" { + command = plan + + providers = { + aws = aws.private_endpoint + } + + expect_failures = [check.nullify_can_reach_cluster_endpoints] + + assert { + condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null + error_message = "No publicAccessCidrs command applies while the public endpoint is disabled" + } +} + +run "admin_view_policy_is_cluster_scoped_and_warns" { + command = plan + + variables { + authorization = "admin_view_policy" + } + + expect_failures = [check.admin_view_policy_is_broader_than_required] + + assert { + condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].policy_arn == "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + error_message = "admin_view_policy must associate AmazonEKSAdminViewPolicy" + } + + assert { + condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].access_scope[0].type == "cluster" + error_message = "The access policy must be cluster scoped" + } + + assert { + condition = output.kubernetes_group_name == null + error_message = "admin_view_policy binds no Kubernetes group" + } +} + +run "view_policy_is_not_offered" { + command = plan + + variables { + authorization = "AmazonEKSViewPolicy" + } + + expect_failures = [var.authorization] +} + +run "system_groups_are_rejected" { + command = plan + + variables { + kubernetes_group_name = "system:masters" + } + + expect_failures = [var.kubernetes_group_name] +} + +run "cross_account_clusters_are_rejected" { + command = plan + + variables { + cluster_arns = ["arn:aws:eks:eu-west-1:999999999999:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.this] +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf new file mode 100644 index 0000000..2119d71 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -0,0 +1,63 @@ +variable "principal_arn" { + type = string + description = "ARN of the Nullify read-only role (module.nullify_aws_integration.role_arn). Nullify assumes it in the cluster's own account, so it must be in the same account as the clusters" + + validation { + condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.principal_arn)) + error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/" + } +} + +variable "principal_unique_id" { + type = string + description = "unique_id of the role (module.nullify_aws_integration.role_unique_id). EKS ties an access entry to the role's ID, so a recreated role with the same ARN is not authorized by the old entry; when this value changes, the entries are replaced" + default = "" +} + +variable "cluster_arns" { + type = list(string) + description = "ARNs of the EKS clusters Nullify scans. Each cluster is read and granted access in the region from its ARN" + + validation { + condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/" + } +} + +variable "authorization" { + type = string + description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to the list-only nullify-readonly ClusterRole with k8s-resources (enable_managed_scan_rbac), the nullify-k8s-readonly-access Helm chart or manifests/nullify-readonly-rbac.yaml. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + default = "rbac" + + validation { + condition = contains(["rbac", "admin_view_policy"], var.authorization) + error_message = "authorization must be rbac or admin_view_policy. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." + } +} + +variable "kubernetes_group_name" { + type = string + description = "Kubernetes group on the access entry when authorization is rbac" + default = "nullify-readonly" + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$", var.kubernetes_group_name)) + error_message = "Lowercase letters, digits, '.' and '-' only (max 63 characters). ':' is not allowed, which excludes system: groups." + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region, shown on the Nullify configure page. It selects the egress IPs the cluster's public endpoint must admit: ap-southeast-2, eu-central-1 or us-east-2" + + validation { + condition = contains(["ap-southeast-2", "eu-central-1", "us-east-2"], var.nullify_region) + error_message = "nullify_region must be ap-southeast-2, eu-central-1 or us-east-2" + } +} + +variable "tags" { + type = map(string) + description = "Tags to apply to the access entries" + default = {} +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf new file mode 100644 index 0000000..f13e466 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 238b300..6de986a 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -1,4 +1,6 @@ resource "kubernetes_namespace" "nullify" { + count = var.enable_collector ? 1 : 0 + metadata { name = var.kubernetes_namespace labels = { @@ -9,9 +11,11 @@ resource "kubernetes_namespace" "nullify" { } resource "kubernetes_service_account" "nullify_collector_sa" { + count = var.enable_collector ? 1 : 0 + metadata { name = var.service_account_name - namespace = kubernetes_namespace.nullify.metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name annotations = { "eks.amazonaws.com/role-arn" = var.iam_role_arn @@ -23,9 +27,18 @@ resource "kubernetes_service_account" "nullify_collector_sa" { "app.kubernetes.io/managed-by" = "terraform" } } + + lifecycle { + precondition { + condition = var.iam_role_arn != "" + error_message = "iam_role_arn is required when enable_collector is true: the collector's service account assumes it through IRSA." + } + } } resource "kubernetes_cluster_role" "nullify_readonly_role" { + count = var.enable_collector ? 1 : 0 + metadata { name = "nullify-k8s-collector-role" labels = { @@ -110,6 +123,8 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } resource "kubernetes_cluster_role_binding" "nullify_collector_binding" { + count = var.enable_collector ? 1 : 0 + metadata { name = "nullify-k8s-collector-binding" labels = { @@ -122,20 +137,22 @@ resource "kubernetes_cluster_role_binding" "nullify_collector_binding" { role_ref { api_group = "rbac.authorization.k8s.io" kind = "ClusterRole" - name = kubernetes_cluster_role.nullify_readonly_role.metadata[0].name + name = kubernetes_cluster_role.nullify_readonly_role[0].metadata[0].name } subject { kind = "ServiceAccount" - name = kubernetes_service_account.nullify_collector_sa.metadata[0].name - namespace = kubernetes_namespace.nullify.metadata[0].name + name = kubernetes_service_account.nullify_collector_sa[0].metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name } } resource "kubernetes_cron_job_v1" "k8s_collector" { + count = var.enable_collector ? 1 : 0 + metadata { name = "k8s-info-collector" - namespace = kubernetes_namespace.nullify.metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name labels = { "app.kubernetes.io/name" = "nullify" "app.kubernetes.io/component" = "k8s-collector" @@ -168,7 +185,7 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } spec { - service_account_name = kubernetes_service_account.nullify_collector_sa.metadata[0].name + service_account_name = kubernetes_service_account.nullify_collector_sa[0].metadata[0].name restart_policy = "OnFailure" container { @@ -222,4 +239,4 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } } -} \ No newline at end of file +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf new file mode 100644 index 0000000..4f6505d --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf @@ -0,0 +1,82 @@ +locals { + # The kinds Nullify's managed EKS scan lists. Keep in step with + # manifests/nullify-readonly-rbac.yaml and the nullify-k8s-readonly-access chart. + managed_scan_rules = [ + { + api_groups = [""] + resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"] + }, + { + api_groups = ["apps"] + resources = ["deployments", "daemonsets", "statefulsets", "replicasets"] + }, + { + api_groups = ["networking.k8s.io"] + resources = ["ingresses", "networkpolicies"] + }, + { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + }, + { + api_groups = ["rbac.authorization.k8s.io"] + resources = ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] + }, + { + api_groups = ["admissionregistration.k8s.io"] + resources = ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] + }, + ] +} + +resource "kubernetes_cluster_role_v1" "nullify_readonly" { + count = var.enable_managed_scan_rbac ? 1 : 0 + + metadata { + name = var.managed_scan_cluster_role_name + labels = { + "app.kubernetes.io/name" = "nullify" + "app.kubernetes.io/component" = "managed-scan-rbac" + "app.kubernetes.io/managed-by" = "terraform" + } + } + + dynamic "rule" { + for_each = local.managed_scan_rules + content { + api_groups = rule.value.api_groups + resources = rule.value.resources + verbs = ["list"] + } + } + + rule { + non_resource_urls = ["/version"] + verbs = ["get"] + } +} + +resource "kubernetes_cluster_role_binding_v1" "nullify_readonly" { + count = var.enable_managed_scan_rbac ? 1 : 0 + + metadata { + name = var.managed_scan_cluster_role_binding_name + labels = { + "app.kubernetes.io/name" = "nullify" + "app.kubernetes.io/component" = "managed-scan-rbac" + "app.kubernetes.io/managed-by" = "terraform" + } + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name + } + + subject { + api_group = "rbac.authorization.k8s.io" + kind = "Group" + name = var.managed_scan_kubernetes_group + } +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/moved.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/moved.tf new file mode 100644 index 0000000..fa4b929 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/moved.tf @@ -0,0 +1,24 @@ +moved { + from = kubernetes_namespace.nullify + to = kubernetes_namespace.nullify[0] +} + +moved { + from = kubernetes_service_account.nullify_collector_sa + to = kubernetes_service_account.nullify_collector_sa[0] +} + +moved { + from = kubernetes_cluster_role.nullify_readonly_role + to = kubernetes_cluster_role.nullify_readonly_role[0] +} + +moved { + from = kubernetes_cluster_role_binding.nullify_collector_binding + to = kubernetes_cluster_role_binding.nullify_collector_binding[0] +} + +moved { + from = kubernetes_cron_job_v1.k8s_collector + to = kubernetes_cron_job_v1.k8s_collector[0] +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/outputs.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/outputs.tf index feea808..6cc3136 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/outputs.tf @@ -1,24 +1,39 @@ output "namespace_name" { - description = "Name of the created Kubernetes namespace" - value = kubernetes_namespace.nullify.metadata[0].name + description = "Name of the created Kubernetes namespace (null when enable_collector is false)" + value = one(kubernetes_namespace.nullify[*].metadata[0].name) } output "service_account_name" { - description = "Name of the created Kubernetes service account" - value = kubernetes_service_account.nullify_collector_sa.metadata[0].name + description = "Name of the created Kubernetes service account (null when enable_collector is false)" + value = one(kubernetes_service_account.nullify_collector_sa[*].metadata[0].name) } output "cluster_role_name" { - description = "Name of the created cluster role" - value = kubernetes_cluster_role.nullify_readonly_role.metadata[0].name + description = "Name of the collector's cluster role (null when enable_collector is false)" + value = one(kubernetes_cluster_role.nullify_readonly_role[*].metadata[0].name) } output "cluster_role_binding_name" { - description = "Name of the created cluster role binding" - value = kubernetes_cluster_role_binding.nullify_collector_binding.metadata[0].name + description = "Name of the collector's cluster role binding (null when enable_collector is false)" + value = one(kubernetes_cluster_role_binding.nullify_collector_binding[*].metadata[0].name) } output "cronjob_name" { - description = "Name of the created CronJob" - value = kubernetes_cron_job_v1.k8s_collector.metadata[0].name -} \ No newline at end of file + description = "Name of the created CronJob (null when enable_collector is false)" + value = one(kubernetes_cron_job_v1.k8s_collector[*].metadata[0].name) +} + +output "managed_scan_cluster_role_name" { + description = "Name of the managed-scan ClusterRole (null when enable_managed_scan_rbac is false)" + value = one(kubernetes_cluster_role_v1.nullify_readonly[*].metadata[0].name) +} + +output "managed_scan_cluster_role_binding_name" { + description = "Name of the managed-scan ClusterRoleBinding (null when enable_managed_scan_rbac is false)" + value = one(kubernetes_cluster_role_binding_v1.nullify_readonly[*].metadata[0].name) +} + +output "managed_scan_kubernetes_group" { + description = "Kubernetes group bound to the managed-scan ClusterRole (null when enable_managed_scan_rbac is false)" + value = var.enable_managed_scan_rbac ? var.managed_scan_kubernetes_group : null +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl new file mode 100644 index 0000000..2b0b487 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -0,0 +1,105 @@ +mock_provider "kubernetes" {} + +run "collector_only_is_the_default" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + s3_bucket_name = "nullify-collector-uploads" + } + + assert { + condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 + error_message = "The collector must be deployed by default" + } + + assert { + condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + error_message = "The collector image must default to the pinned Nullify ECR Public tag" + } + + assert { + condition = length(kubernetes_cluster_role_v1.nullify_readonly) == 0 && length(kubernetes_cluster_role_binding_v1.nullify_readonly) == 0 + error_message = "Managed-scan RBAC must be opt-in" + } +} + +run "collector_requires_a_role_arn" { + command = plan + + expect_failures = [kubernetes_service_account.nullify_collector_sa] +} + +run "managed_scan_rbac_only" { + command = plan + + variables { + enable_collector = false + enable_managed_scan_rbac = true + } + + assert { + condition = length(kubernetes_namespace.nullify) == 0 && length(kubernetes_cron_job_v1.k8s_collector) == 0 && length(kubernetes_cluster_role.nullify_readonly_role) == 0 + error_message = "enable_collector = false must remove every collector resource" + } + + assert { + condition = output.namespace_name == null && output.cronjob_name == null + error_message = "Collector outputs must be null when the collector is disabled" + } + + assert { + condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 7 + error_message = "The managed-scan ClusterRole must have six resource rules and one /version rule" + } + + assert { + condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : length(rule.resources)]) == 26 + error_message = "The managed-scan ClusterRole must list exactly the 26 kinds the scanner reads" + } + + assert { + condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : tolist(rule.verbs) == tolist(["list"])]) + error_message = "Resource rules must grant list only" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[6].non_resource_urls) == tolist(["/version"]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[6].verbs) == tolist(["get"]) + error_message = "The only non-resource grant must be get /version" + } + + assert { + condition = contains(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources, "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") + error_message = "The role must list secrets and grant no subresource" + } + + assert { + condition = kubernetes_cluster_role_binding_v1.nullify_readonly[0].subject[0].kind == "Group" && kubernetes_cluster_role_binding_v1.nullify_readonly[0].subject[0].name == "nullify-readonly" + error_message = "The binding must name the nullify-readonly group" + } + + assert { + condition = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name == "nullify-readonly" + error_message = "The ClusterRole name must match the Helm chart and manifest" + } +} + +run "kms_alias_arns_are_accepted" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } +} + +run "invalid_kms_arns_are_rejected" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + kms_key_arn = "arn:aws:s3:::not-a-key" + } + + expect_failures = [var.kms_key_arn] +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf index 9d9c482..40f304e 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf @@ -1,6 +1,42 @@ +variable "enable_collector" { + type = bool + description = "Deploy the in-cluster collector: namespace, IRSA service account, RBAC and CronJob" + default = true +} + +variable "enable_managed_scan_rbac" { + type = bool + description = "Create the list-only ClusterRole and ClusterRoleBinding for Nullify's managed EKS scan, bound to managed_scan_kubernetes_group. Pair it with the eks-managed-scan-access module in rbac mode. Applying it needs rights to grant every listed permission, which in practice means cluster-admin" + default = false +} + +variable "managed_scan_kubernetes_group" { + type = string + description = "Kubernetes group on the Nullify access entry that the managed-scan ClusterRole is bound to" + default = "nullify-readonly" + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$", var.managed_scan_kubernetes_group)) + error_message = "Lowercase letters, digits, '.' and '-' only (max 63 characters). ':' is not allowed, which excludes system: groups." + } +} + +variable "managed_scan_cluster_role_name" { + type = string + description = "Name of the managed-scan ClusterRole. The default matches the nullify-k8s-readonly-access Helm chart and manifests/nullify-readonly-rbac.yaml" + default = "nullify-readonly" +} + +variable "managed_scan_cluster_role_binding_name" { + type = string + description = "Name of the managed-scan ClusterRoleBinding" + default = "nullify-readonly" +} + variable "iam_role_arn" { type = string - description = "The ARN of the IAM role for the service account annotation" + description = "The ARN of the IAM role for the collector service account annotation (required when enable_collector is true)" + default = "" } variable "service_account_name" { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf index 121639e..6552e44 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf @@ -3,6 +3,11 @@ output "role_arn" { value = aws_iam_role.nullify_readonly_role.arn } +output "role_unique_id" { + description = "Stable ID of the IAM role. Pass it to eks-managed-scan-access as principal_unique_id so a recreated role replaces the access entries" + value = aws_iam_role.nullify_readonly_role.unique_id +} + output "role_name" { description = "Name of the IAM Role created for Nullify" value = aws_iam_role.nullify_readonly_role.name diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl new file mode 100644 index 0000000..ad693b4 --- /dev/null +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -0,0 +1,114 @@ +mock_provider "aws" { + mock_data "aws_eks_cluster" { + defaults = { + identity = [{ + oidc = [{ + issuer = "https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" + }] + }] + } + } +} + +variables { + customer_name = "acme" + external_id = "external-id-123456" + nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" +} + +run "kms_alias_grants_the_alias_and_key_wildcard" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } + + assert { + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads", "arn:aws:kms:eu-central-1:111122223333:key/*"]) + error_message = "An alias ARN must grant the alias and key/* in the same account and region" + } +} + +run "kms_multi_region_key_is_accepted" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab" + } + + assert { + condition = contains(output.kms_policy_resources, "arn:aws:kms:us-east-2:111122223333:key/*") + error_message = "A key ARN must also grant key/* in the same account and region" + } +} + +run "kms_is_optional" { + command = plan + + assert { + condition = length(output.kms_policy_resources) == 0 && length(aws_iam_policy.kms_access_policy) == 0 + error_message = "No KMS policy without kms_key_arn" + } +} + +run "invalid_kms_arns_are_rejected" { + command = plan + + variables { + kms_key_arn = "arn:aws:s3:::not-a-key" + } + + expect_failures = [var.kms_key_arn] +} + +run "s3_access_point_enables_the_s3_policy" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:eu-central-1:111122223333:accesspoint/acme-k8s-collector" + } + + assert { + condition = length(aws_iam_policy.s3_access_policy) == 1 + error_message = "An access point alone must create the S3 policy" + } +} + +run "invalid_access_point_arns_are_rejected" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:::a-bucket" + } + + expect_failures = [var.nullify_s3_access_point_arn] +} + +run "clusters_are_read_in_their_own_region" { + command = plan + + variables { + enable_kubernetes_integration = true + eks_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + } + + assert { + condition = data.aws_eks_cluster.clusters[0].region == "us-east-1" + error_message = "Each cluster must be read in the region from its ARN" + } + + assert { + condition = output.all_oidc_ids[0] == "EXAMPLED539D4633E53DE1B716D3041E" + error_message = "The OIDC ID must come from the cluster's issuer" + } +} + +run "kubernetes_integration_needs_clusters" { + command = plan + + variables { + enable_kubernetes_integration = true + } + + expect_failures = [data.aws_iam_policy_document.assume_role_policy] +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf new file mode 100644 index 0000000..8cff96f --- /dev/null +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -0,0 +1,103 @@ +locals { + nullify_egress_cidrs_by_region = { + "ap-southeast-2" = ["13.55.32.104/32", "3.105.146.106/32", "13.211.99.100/32"] + "eu-central-1" = ["18.198.60.231/32", "18.157.227.250/32", "18.185.152.197/32"] + "us-east-2" = ["52.15.146.50/32", "16.58.40.80/32", "3.133.15.210/32"] + } + + nullify_egress_cidrs = local.nullify_egress_cidrs_by_region[var.nullify_region] + principal_account = split(":", var.principal_arn)[4] + + clusters = { + for arn in var.cluster_arns : arn => { + name = split("/", arn)[1] + region = split(":", arn)[3] + account = split(":", arn)[4] + } + } +} + +data "aws_region" "current" {} + +data "aws_eks_cluster" "this" { + for_each = local.clusters + + name = each.value.name + + lifecycle { + precondition { + condition = each.value.region == data.aws_region.current.name + error_message = "Cluster ${each.key} is in ${each.value.region}, but this module's AWS provider is in ${data.aws_region.current.name}. AWS provider v5 manages access entries only in the provider's region: instantiate this module once per region with providers = { aws = aws. }, or use terraform-v6." + } + + precondition { + condition = each.value.account == local.principal_account + error_message = "Cluster ${each.key} is in account ${each.value.account}, but principal_arn is in account ${local.principal_account}. Nullify assumes the integration role in the cluster's own account: deploy nullify-aws-integration there and pass that role." + } + } +} + +resource "terraform_data" "principal_identity" { + input = var.principal_unique_id +} + +resource "aws_eks_access_entry" "nullify" { + for_each = local.clusters + + cluster_name = data.aws_eks_cluster.this[each.key].name + principal_arn = var.principal_arn + type = "STANDARD" + kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : null + tags = var.tags + + lifecycle { + replace_triggered_by = [terraform_data.principal_identity] + + precondition { + condition = contains(["API", "API_AND_CONFIG_MAP"], try(data.aws_eks_cluster.this[each.key].access_config[0].authentication_mode, "CONFIG_MAP")) + error_message = "Cluster ${each.value.name} (${each.value.region}) uses authentication mode CONFIG_MAP, which does not support access entries. Switch it (one-way) with: aws eks update-cluster-config --region ${each.value.region} --name ${each.value.name} --access-config authenticationMode=API_AND_CONFIG_MAP. Or remove it from cluster_arns and map the role in aws-auth instead (see README)." + } + } +} + +resource "aws_eks_access_policy_association" "admin_view" { + for_each = { for key, cluster in local.clusters : key => cluster if var.authorization == "admin_view_policy" } + + cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name + principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + + access_scope { + type = "cluster" + } + + lifecycle { + replace_triggered_by = [aws_eks_access_entry.nullify[each.key].access_entry_arn] + } +} + +locals { + endpoints = { + for key, cluster in data.aws_eks_cluster.this : key => { + public_access = cluster.vpc_config[0].endpoint_public_access + cidrs = cluster.vpc_config[0].public_access_cidrs + missing_cidrs = [for cidr in local.nullify_egress_cidrs : cidr if !contains(cluster.vpc_config[0].public_access_cidrs, cidr) && !contains(cluster.vpc_config[0].public_access_cidrs, "0.0.0.0/0")] + } + } + + unreachable_clusters = [for key, endpoint in local.endpoints : key if !endpoint.public_access || length(endpoint.missing_cidrs) > 0] +} + +check "nullify_can_reach_cluster_endpoints" { + assert { + condition = length(local.unreachable_clusters) == 0 + error_message = "Nullify's ${var.nullify_region} egress IPs cannot reach the public endpoint of: ${join(", ", local.unreachable_clusters)}. This module does not change publicAccessCidrs; see the endpoint_allowlist output for the missing CIDRs and an update command. The check matches exact CIDRs, so a wider range that already covers Nullify's IPs also warns." + } +} + +check "admin_view_policy_is_broader_than_required" { + assert { + condition = var.authorization != "admin_view_policy" + error_message = "authorization = admin_view_policy associates AmazonEKSAdminViewPolicy: get, list and watch on every resource, including Secrets, custom resources and pods/log. On EKS 1.34 and earlier get pods/exec is enough to exec into pods, and these grants do not show in kubectl auth can-i --list. The default rbac mode grants list on only the kinds Nullify reads." + } +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf new file mode 100644 index 0000000..6e6d1bc --- /dev/null +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf @@ -0,0 +1,41 @@ +output "access_entry_arns" { + description = "Access entry ARN per cluster ARN" + value = { for key, entry in aws_eks_access_entry.nullify : key => entry.access_entry_arn } +} + +output "authorization" { + description = "How Kubernetes authorizes Nullify: rbac or admin_view_policy" + value = var.authorization +} + +output "kubernetes_group_name" { + description = "Group the RBAC binding must name (null for admin_view_policy)" + value = var.authorization == "rbac" ? var.kubernetes_group_name : null +} + +output "nullify_egress_cidrs" { + description = "Nullify egress CIDRs for nullify_region that each cluster's public endpoint must admit" + value = local.nullify_egress_cidrs +} + +output "clusters_for_nullify" { + description = "Clusters to register in Nullify" + value = [ + for key, cluster in local.clusters : { + arn = key + name = cluster.name + region = cluster.region + } + ] +} + +output "endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, the CIDRs to add, and an AWS CLI command that adds them to the existing list. This module never changes publicAccessCidrs" + value = { + for key, endpoint in local.endpoints : key => { + public_endpoint_enabled = endpoint.public_access + missing_cidrs = endpoint.missing_cidrs + update_command = endpoint.public_access && length(endpoint.missing_cidrs) > 0 ? format("aws eks update-cluster-config --region %s --name %s --resources-vpc-config '%s'", local.clusters[key].region, local.clusters[key].name, jsonencode({ publicAccessCidrs = concat(endpoint.cidrs, endpoint.missing_cidrs) })) : null + } + } +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl new file mode 100644 index 0000000..98c6f0b --- /dev/null +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -0,0 +1,234 @@ +mock_provider "aws" { + mock_data "aws_region" { + defaults = { + name = "eu-west-1" + } + } + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API_AND_CONFIG_MAP" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +mock_provider "aws" { + alias = "config_map" + + mock_data "aws_region" { + defaults = { + name = "eu-west-1" + } + } + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "CONFIG_MAP" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +mock_provider "aws" { + alias = "restricted_endpoint" + + mock_data "aws_region" { + defaults = { + name = "eu-west-1" + } + } + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API" + }] + vpc_config = [{ + endpoint_public_access = true + public_access_cidrs = ["10.0.0.0/8", "18.198.60.231/32"] + }] + } + } +} + +mock_provider "aws" { + alias = "private_endpoint" + + mock_data "aws_region" { + defaults = { + name = "eu-west-1" + } + } + + mock_data "aws_eks_cluster" { + defaults = { + access_config = [{ + authentication_mode = "API" + }] + vpc_config = [{ + endpoint_public_access = false + public_access_cidrs = ["0.0.0.0/0"] + }] + } + } +} + +variables { + principal_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + principal_unique_id = "AROAEXAMPLEUNIQUEID01" + nullify_region = "eu-central-1" + cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] +} + +run "rbac_is_the_default" { + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "Expected one access entry per cluster" + } + + assert { + condition = toset(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == toset(["nullify-readonly"]) + error_message = "rbac mode must put the entry in the nullify-readonly group" + } + + assert { + condition = length(aws_eks_access_policy_association.admin_view) == 0 + error_message = "rbac mode must not associate any access policy" + } + + assert { + condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null + error_message = "An endpoint open to 0.0.0.0/0 needs no update" + } + + assert { + condition = output.kubernetes_group_name == "nullify-readonly" + error_message = "kubernetes_group_name output must name the bound group" + } +} + +run "clusters_outside_the_provider_region_are_rejected" { + command = plan + + variables { + cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.this] +} + +run "config_map_clusters_are_rejected" { + command = plan + + providers = { + aws = aws.config_map + } + + expect_failures = [aws_eks_access_entry.nullify] +} + +run "restricted_endpoint_warns_and_prints_the_missing_cidrs" { + command = plan + + providers = { + aws = aws.restricted_endpoint + } + + expect_failures = [check.nullify_can_reach_cluster_endpoints] + + assert { + condition = length(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].missing_cidrs) == 2 + error_message = "Only the two absent eu-central-1 CIDRs should be missing" + } + + assert { + condition = !contains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].missing_cidrs, "18.198.60.231/32") + error_message = "A CIDR already in publicAccessCidrs must not be reported missing" + } + + assert { + condition = strcontains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command, "10.0.0.0/8") && strcontains(output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command, "18.185.152.197/32") + error_message = "The update command must keep existing CIDRs and add the missing ones" + } +} + +run "private_endpoint_warns_without_a_command" { + command = plan + + providers = { + aws = aws.private_endpoint + } + + expect_failures = [check.nullify_can_reach_cluster_endpoints] + + assert { + condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null + error_message = "No publicAccessCidrs command applies while the public endpoint is disabled" + } +} + +run "admin_view_policy_is_cluster_scoped_and_warns" { + command = plan + + variables { + authorization = "admin_view_policy" + } + + expect_failures = [check.admin_view_policy_is_broader_than_required] + + assert { + condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].policy_arn == "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + error_message = "admin_view_policy must associate AmazonEKSAdminViewPolicy" + } + + assert { + condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].access_scope[0].type == "cluster" + error_message = "The access policy must be cluster scoped" + } + + assert { + condition = output.kubernetes_group_name == null + error_message = "admin_view_policy binds no Kubernetes group" + } +} + +run "view_policy_is_not_offered" { + command = plan + + variables { + authorization = "AmazonEKSViewPolicy" + } + + expect_failures = [var.authorization] +} + +run "system_groups_are_rejected" { + command = plan + + variables { + kubernetes_group_name = "system:masters" + } + + expect_failures = [var.kubernetes_group_name] +} + +run "cross_account_clusters_are_rejected" { + command = plan + + variables { + cluster_arns = ["arn:aws:eks:eu-west-1:999999999999:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.this] +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf new file mode 100644 index 0000000..05a4e4c --- /dev/null +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -0,0 +1,63 @@ +variable "principal_arn" { + type = string + description = "ARN of the Nullify read-only role (module.nullify_aws_integration.role_arn). Nullify assumes it in the cluster's own account, so it must be in the same account as the clusters" + + validation { + condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.principal_arn)) + error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/" + } +} + +variable "principal_unique_id" { + type = string + description = "unique_id of the role (module.nullify_aws_integration.role_unique_id). EKS ties an access entry to the role's ID, so a recreated role with the same ARN is not authorized by the old entry; when this value changes, the entries are replaced" + default = "" +} + +variable "cluster_arns" { + type = list(string) + description = "ARNs of the EKS clusters Nullify scans, all in this module's AWS provider region. AWS provider v5 has no per-resource region, so instantiate the module once per region with providers = { aws = aws. }" + + validation { + condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/" + } +} + +variable "authorization" { + type = string + description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to the list-only nullify-readonly ClusterRole with k8s-resources (enable_managed_scan_rbac), the nullify-k8s-readonly-access Helm chart or manifests/nullify-readonly-rbac.yaml. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + default = "rbac" + + validation { + condition = contains(["rbac", "admin_view_policy"], var.authorization) + error_message = "authorization must be rbac or admin_view_policy. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." + } +} + +variable "kubernetes_group_name" { + type = string + description = "Kubernetes group on the access entry when authorization is rbac" + default = "nullify-readonly" + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$", var.kubernetes_group_name)) + error_message = "Lowercase letters, digits, '.' and '-' only (max 63 characters). ':' is not allowed, which excludes system: groups." + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region, shown on the Nullify configure page. It selects the egress IPs the cluster's public endpoint must admit: ap-southeast-2, eu-central-1 or us-east-2" + + validation { + condition = contains(["ap-southeast-2", "eu-central-1", "us-east-2"], var.nullify_region) + error_message = "nullify_region must be ap-southeast-2, eu-central-1 or us-east-2" + } +} + +variable "tags" { + type = map(string) + description = "Tags to apply to the access entries" + default = {} +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf new file mode 100644 index 0000000..6dd2f65 --- /dev/null +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.33, < 6.0" + } + } +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 8f77aeb..6de986a 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -1,4 +1,6 @@ resource "kubernetes_namespace" "nullify" { + count = var.enable_collector ? 1 : 0 + metadata { name = var.kubernetes_namespace labels = { @@ -9,9 +11,11 @@ resource "kubernetes_namespace" "nullify" { } resource "kubernetes_service_account" "nullify_collector_sa" { + count = var.enable_collector ? 1 : 0 + metadata { name = var.service_account_name - namespace = kubernetes_namespace.nullify.metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name annotations = { "eks.amazonaws.com/role-arn" = var.iam_role_arn @@ -23,9 +27,18 @@ resource "kubernetes_service_account" "nullify_collector_sa" { "app.kubernetes.io/managed-by" = "terraform" } } + + lifecycle { + precondition { + condition = var.iam_role_arn != "" + error_message = "iam_role_arn is required when enable_collector is true: the collector's service account assumes it through IRSA." + } + } } resource "kubernetes_cluster_role" "nullify_readonly_role" { + count = var.enable_collector ? 1 : 0 + metadata { name = "nullify-k8s-collector-role" labels = { @@ -110,6 +123,8 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } resource "kubernetes_cluster_role_binding" "nullify_collector_binding" { + count = var.enable_collector ? 1 : 0 + metadata { name = "nullify-k8s-collector-binding" labels = { @@ -122,20 +137,22 @@ resource "kubernetes_cluster_role_binding" "nullify_collector_binding" { role_ref { api_group = "rbac.authorization.k8s.io" kind = "ClusterRole" - name = kubernetes_cluster_role.nullify_readonly_role.metadata[0].name + name = kubernetes_cluster_role.nullify_readonly_role[0].metadata[0].name } subject { kind = "ServiceAccount" - name = kubernetes_service_account.nullify_collector_sa.metadata[0].name - namespace = kubernetes_namespace.nullify.metadata[0].name + name = kubernetes_service_account.nullify_collector_sa[0].metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name } } resource "kubernetes_cron_job_v1" "k8s_collector" { + count = var.enable_collector ? 1 : 0 + metadata { name = "k8s-info-collector" - namespace = kubernetes_namespace.nullify.metadata[0].name + namespace = kubernetes_namespace.nullify[0].metadata[0].name labels = { "app.kubernetes.io/name" = "nullify" "app.kubernetes.io/component" = "k8s-collector" @@ -168,7 +185,7 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } spec { - service_account_name = kubernetes_service_account.nullify_collector_sa.metadata[0].name + service_account_name = kubernetes_service_account.nullify_collector_sa[0].metadata[0].name restart_policy = "OnFailure" container { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf new file mode 100644 index 0000000..4f6505d --- /dev/null +++ b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf @@ -0,0 +1,82 @@ +locals { + # The kinds Nullify's managed EKS scan lists. Keep in step with + # manifests/nullify-readonly-rbac.yaml and the nullify-k8s-readonly-access chart. + managed_scan_rules = [ + { + api_groups = [""] + resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"] + }, + { + api_groups = ["apps"] + resources = ["deployments", "daemonsets", "statefulsets", "replicasets"] + }, + { + api_groups = ["networking.k8s.io"] + resources = ["ingresses", "networkpolicies"] + }, + { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + }, + { + api_groups = ["rbac.authorization.k8s.io"] + resources = ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] + }, + { + api_groups = ["admissionregistration.k8s.io"] + resources = ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] + }, + ] +} + +resource "kubernetes_cluster_role_v1" "nullify_readonly" { + count = var.enable_managed_scan_rbac ? 1 : 0 + + metadata { + name = var.managed_scan_cluster_role_name + labels = { + "app.kubernetes.io/name" = "nullify" + "app.kubernetes.io/component" = "managed-scan-rbac" + "app.kubernetes.io/managed-by" = "terraform" + } + } + + dynamic "rule" { + for_each = local.managed_scan_rules + content { + api_groups = rule.value.api_groups + resources = rule.value.resources + verbs = ["list"] + } + } + + rule { + non_resource_urls = ["/version"] + verbs = ["get"] + } +} + +resource "kubernetes_cluster_role_binding_v1" "nullify_readonly" { + count = var.enable_managed_scan_rbac ? 1 : 0 + + metadata { + name = var.managed_scan_cluster_role_binding_name + labels = { + "app.kubernetes.io/name" = "nullify" + "app.kubernetes.io/component" = "managed-scan-rbac" + "app.kubernetes.io/managed-by" = "terraform" + } + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name + } + + subject { + api_group = "rbac.authorization.k8s.io" + kind = "Group" + name = var.managed_scan_kubernetes_group + } +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/moved.tf b/aws-integration-setup/terraform/modules/k8s-resources/moved.tf new file mode 100644 index 0000000..fa4b929 --- /dev/null +++ b/aws-integration-setup/terraform/modules/k8s-resources/moved.tf @@ -0,0 +1,24 @@ +moved { + from = kubernetes_namespace.nullify + to = kubernetes_namespace.nullify[0] +} + +moved { + from = kubernetes_service_account.nullify_collector_sa + to = kubernetes_service_account.nullify_collector_sa[0] +} + +moved { + from = kubernetes_cluster_role.nullify_readonly_role + to = kubernetes_cluster_role.nullify_readonly_role[0] +} + +moved { + from = kubernetes_cluster_role_binding.nullify_collector_binding + to = kubernetes_cluster_role_binding.nullify_collector_binding[0] +} + +moved { + from = kubernetes_cron_job_v1.k8s_collector + to = kubernetes_cron_job_v1.k8s_collector[0] +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/outputs.tf b/aws-integration-setup/terraform/modules/k8s-resources/outputs.tf index c25174f..6cc3136 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/outputs.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/outputs.tf @@ -1,24 +1,39 @@ output "namespace_name" { - description = "Name of the created Kubernetes namespace" - value = kubernetes_namespace.nullify.metadata[0].name + description = "Name of the created Kubernetes namespace (null when enable_collector is false)" + value = one(kubernetes_namespace.nullify[*].metadata[0].name) } output "service_account_name" { - description = "Name of the created Kubernetes service account" - value = kubernetes_service_account.nullify_collector_sa.metadata[0].name + description = "Name of the created Kubernetes service account (null when enable_collector is false)" + value = one(kubernetes_service_account.nullify_collector_sa[*].metadata[0].name) } output "cluster_role_name" { - description = "Name of the created cluster role" - value = kubernetes_cluster_role.nullify_readonly_role.metadata[0].name + description = "Name of the collector's cluster role (null when enable_collector is false)" + value = one(kubernetes_cluster_role.nullify_readonly_role[*].metadata[0].name) } output "cluster_role_binding_name" { - description = "Name of the created cluster role binding" - value = kubernetes_cluster_role_binding.nullify_collector_binding.metadata[0].name + description = "Name of the collector's cluster role binding (null when enable_collector is false)" + value = one(kubernetes_cluster_role_binding.nullify_collector_binding[*].metadata[0].name) } output "cronjob_name" { - description = "Name of the created CronJob" - value = kubernetes_cron_job_v1.k8s_collector.metadata[0].name + description = "Name of the created CronJob (null when enable_collector is false)" + value = one(kubernetes_cron_job_v1.k8s_collector[*].metadata[0].name) +} + +output "managed_scan_cluster_role_name" { + description = "Name of the managed-scan ClusterRole (null when enable_managed_scan_rbac is false)" + value = one(kubernetes_cluster_role_v1.nullify_readonly[*].metadata[0].name) +} + +output "managed_scan_cluster_role_binding_name" { + description = "Name of the managed-scan ClusterRoleBinding (null when enable_managed_scan_rbac is false)" + value = one(kubernetes_cluster_role_binding_v1.nullify_readonly[*].metadata[0].name) +} + +output "managed_scan_kubernetes_group" { + description = "Kubernetes group bound to the managed-scan ClusterRole (null when enable_managed_scan_rbac is false)" + value = var.enable_managed_scan_rbac ? var.managed_scan_kubernetes_group : null } diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl new file mode 100644 index 0000000..2b0b487 --- /dev/null +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -0,0 +1,105 @@ +mock_provider "kubernetes" {} + +run "collector_only_is_the_default" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + s3_bucket_name = "nullify-collector-uploads" + } + + assert { + condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 + error_message = "The collector must be deployed by default" + } + + assert { + condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + error_message = "The collector image must default to the pinned Nullify ECR Public tag" + } + + assert { + condition = length(kubernetes_cluster_role_v1.nullify_readonly) == 0 && length(kubernetes_cluster_role_binding_v1.nullify_readonly) == 0 + error_message = "Managed-scan RBAC must be opt-in" + } +} + +run "collector_requires_a_role_arn" { + command = plan + + expect_failures = [kubernetes_service_account.nullify_collector_sa] +} + +run "managed_scan_rbac_only" { + command = plan + + variables { + enable_collector = false + enable_managed_scan_rbac = true + } + + assert { + condition = length(kubernetes_namespace.nullify) == 0 && length(kubernetes_cron_job_v1.k8s_collector) == 0 && length(kubernetes_cluster_role.nullify_readonly_role) == 0 + error_message = "enable_collector = false must remove every collector resource" + } + + assert { + condition = output.namespace_name == null && output.cronjob_name == null + error_message = "Collector outputs must be null when the collector is disabled" + } + + assert { + condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 7 + error_message = "The managed-scan ClusterRole must have six resource rules and one /version rule" + } + + assert { + condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : length(rule.resources)]) == 26 + error_message = "The managed-scan ClusterRole must list exactly the 26 kinds the scanner reads" + } + + assert { + condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : tolist(rule.verbs) == tolist(["list"])]) + error_message = "Resource rules must grant list only" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[6].non_resource_urls) == tolist(["/version"]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[6].verbs) == tolist(["get"]) + error_message = "The only non-resource grant must be get /version" + } + + assert { + condition = contains(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources, "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") + error_message = "The role must list secrets and grant no subresource" + } + + assert { + condition = kubernetes_cluster_role_binding_v1.nullify_readonly[0].subject[0].kind == "Group" && kubernetes_cluster_role_binding_v1.nullify_readonly[0].subject[0].name == "nullify-readonly" + error_message = "The binding must name the nullify-readonly group" + } + + assert { + condition = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name == "nullify-readonly" + error_message = "The ClusterRole name must match the Helm chart and manifest" + } +} + +run "kms_alias_arns_are_accepted" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } +} + +run "invalid_kms_arns_are_rejected" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + kms_key_arn = "arn:aws:s3:::not-a-key" + } + + expect_failures = [var.kms_key_arn] +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf index 9d9c482..40f304e 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf @@ -1,6 +1,42 @@ +variable "enable_collector" { + type = bool + description = "Deploy the in-cluster collector: namespace, IRSA service account, RBAC and CronJob" + default = true +} + +variable "enable_managed_scan_rbac" { + type = bool + description = "Create the list-only ClusterRole and ClusterRoleBinding for Nullify's managed EKS scan, bound to managed_scan_kubernetes_group. Pair it with the eks-managed-scan-access module in rbac mode. Applying it needs rights to grant every listed permission, which in practice means cluster-admin" + default = false +} + +variable "managed_scan_kubernetes_group" { + type = string + description = "Kubernetes group on the Nullify access entry that the managed-scan ClusterRole is bound to" + default = "nullify-readonly" + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$", var.managed_scan_kubernetes_group)) + error_message = "Lowercase letters, digits, '.' and '-' only (max 63 characters). ':' is not allowed, which excludes system: groups." + } +} + +variable "managed_scan_cluster_role_name" { + type = string + description = "Name of the managed-scan ClusterRole. The default matches the nullify-k8s-readonly-access Helm chart and manifests/nullify-readonly-rbac.yaml" + default = "nullify-readonly" +} + +variable "managed_scan_cluster_role_binding_name" { + type = string + description = "Name of the managed-scan ClusterRoleBinding" + default = "nullify-readonly" +} + variable "iam_role_arn" { type = string - description = "The ARN of the IAM role for the service account annotation" + description = "The ARN of the IAM role for the collector service account annotation (required when enable_collector is true)" + default = "" } variable "service_account_name" { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf index 121639e..6552e44 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf @@ -3,6 +3,11 @@ output "role_arn" { value = aws_iam_role.nullify_readonly_role.arn } +output "role_unique_id" { + description = "Stable ID of the IAM role. Pass it to eks-managed-scan-access as principal_unique_id so a recreated role replaces the access entries" + value = aws_iam_role.nullify_readonly_role.unique_id +} + output "role_name" { description = "Name of the IAM Role created for Nullify" value = aws_iam_role.nullify_readonly_role.name diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl new file mode 100644 index 0000000..891ced7 --- /dev/null +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -0,0 +1,134 @@ +mock_provider "aws" { + mock_data "aws_region" { + defaults = { + name = "eu-west-1" + } + } + + mock_data "aws_eks_cluster" { + defaults = { + identity = [{ + oidc = [{ + issuer = "https://oidc.eks.eu-west-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" + }] + }] + } + } +} + +variables { + customer_name = "acme" + external_id = "external-id-123456" + nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" +} + +run "kms_alias_grants_the_alias_and_key_wildcard" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } + + assert { + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads", "arn:aws:kms:eu-central-1:111122223333:key/*"]) + error_message = "An alias ARN must grant the alias and key/* in the same account and region" + } +} + +run "kms_multi_region_key_is_accepted" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab" + } + + assert { + condition = contains(output.kms_policy_resources, "arn:aws:kms:us-east-2:111122223333:key/*") + error_message = "A key ARN must also grant key/* in the same account and region" + } +} + +run "kms_is_optional" { + command = plan + + assert { + condition = length(output.kms_policy_resources) == 0 && length(aws_iam_policy.kms_access_policy) == 0 + error_message = "No KMS policy without kms_key_arn" + } +} + +run "invalid_kms_arns_are_rejected" { + command = plan + + variables { + kms_key_arn = "arn:aws:s3:::not-a-key" + } + + expect_failures = [var.kms_key_arn] +} + +run "s3_access_point_enables_the_s3_policy" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:eu-central-1:111122223333:accesspoint/acme-k8s-collector" + } + + assert { + condition = length(aws_iam_policy.s3_access_policy) == 1 + error_message = "An access point alone must create the S3 policy" + } +} + +run "invalid_access_point_arns_are_rejected" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:::a-bucket" + } + + expect_failures = [var.nullify_s3_access_point_arn] +} + +run "clusters_outside_the_provider_region_need_issuer_urls" { + command = plan + + variables { + enable_kubernetes_integration = true + eks_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.clusters] +} + +run "issuer_urls_skip_the_lookup" { + command = plan + + variables { + enable_kubernetes_integration = true + eks_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + eks_oidc_issuer_urls = ["https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLE0000000000000000000000001"] + } + + assert { + condition = length(data.aws_eks_cluster.clusters) == 0 + error_message = "Clusters must not be looked up when issuer URLs are given" + } + + assert { + condition = output.all_oidc_ids[0] == "EXAMPLE0000000000000000000000001" + error_message = "The OIDC ID must come from the issuer URL" + } +} + +run "issuer_urls_must_match_the_cluster_region" { + command = plan + + variables { + enable_kubernetes_integration = true + eks_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + eks_oidc_issuer_urls = ["https://oidc.eks.eu-west-1.amazonaws.com/id/EXAMPLE0000000000000000000000001"] + } + + expect_failures = [data.aws_iam_policy_document.assume_role_policy] +} From 1d453786daa56b096b24e160e378c58c919ef28d Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 06:07:28 +1000 Subject: [PATCH 13/36] feat(aws): wire managed EKS scan into the root and examples - Root: enable_managed_scan, managed_scan_cluster_arns, nullify_region, managed_scan_authorization (default rbac), managed_scan_kubernetes_group, and managed-scan outputs. v5 root requires AWS provider ~> 5.33. - New examples/managed-scan: one cluster, access entry plus the list-only ClusterRole, manage_rbac = false for GitOps-applied RBAC. - multi-cluster-complete: scan_mode collector (default) | managed | both. v5 uses one access-entry module per regional provider, v6 one instance. Co-Authored-By: Claude Opus 5 (1M context) --- .../examples/managed-scan/main.tf | 72 +++++++++++++++++++ .../examples/managed-scan/outputs.tf | 19 +++++ .../managed-scan/terraform.tfvars.example | 25 +++++++ .../examples/managed-scan/variables.tf | 61 ++++++++++++++++ .../examples/multi-cluster-complete/main.tf | 22 +++++- .../multi-cluster-complete/outputs.tf | 35 ++++++--- .../terraform.tfvars.example | 7 ++ .../multi-cluster-complete/variables.tf | 17 +++++ aws-integration-setup/terraform-v6/main.tf | 13 ++++ aws-integration-setup/terraform-v6/outputs.tf | 22 +++++- .../terraform-v6/terraform.tfvars.example | 8 +++ .../terraform-v6/variables.tf | 30 ++++++++ .../terraform/examples/managed-scan/main.tf | 72 +++++++++++++++++++ .../examples/managed-scan/outputs.tf | 19 +++++ .../managed-scan/terraform.tfvars.example | 25 +++++++ .../examples/managed-scan/variables.tf | 61 ++++++++++++++++ .../examples/multi-cluster-complete/main.tf | 43 ++++++++++- .../multi-cluster-complete/outputs.tf | 33 ++++++--- .../terraform.tfvars.example | 7 ++ .../multi-cluster-complete/variables.tf | 17 +++++ aws-integration-setup/terraform/main.tf | 13 ++++ aws-integration-setup/terraform/outputs.tf | 22 +++++- .../terraform/terraform.tfvars.example | 8 +++ aws-integration-setup/terraform/variables.tf | 30 ++++++++ aws-integration-setup/terraform/versions.tf | 2 +- 25 files changed, 656 insertions(+), 27 deletions(-) create mode 100644 aws-integration-setup/terraform-v6/examples/managed-scan/main.tf create mode 100644 aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf create mode 100644 aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example create mode 100644 aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf create mode 100644 aws-integration-setup/terraform/examples/managed-scan/main.tf create mode 100644 aws-integration-setup/terraform/examples/managed-scan/outputs.tf create mode 100644 aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example create mode 100644 aws-integration-setup/terraform/examples/managed-scan/variables.tf diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf new file mode 100644 index 0000000..e45142a --- /dev/null +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf @@ -0,0 +1,72 @@ +# Managed EKS scan: Nullify lists Kubernetes resources from its own compute +# through an EKS access entry. Nothing runs in the cluster. + +terraform { + required_version = ">= 1.10" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.20" + } + } +} + +locals { + cluster_name = split("/", var.eks_cluster_arn)[1] + cluster_region = split(":", var.eks_cluster_arn)[3] +} + +provider "aws" { + region = local.cluster_region +} + +data "aws_eks_cluster" "this" { + name = local.cluster_name +} + +ephemeral "aws_eks_cluster_auth" "this" { + name = local.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = ephemeral.aws_eks_cluster_auth.this.token +} + +module "nullify_aws_integration" { + source = "../../modules/nullify-aws-integration" + customer_name = var.customer_name + external_id = var.external_id + nullify_role_arn = var.nullify_role_arn + aws_region = local.cluster_region + tags = var.tags +} + +module "eks_managed_scan_access" { + source = "../../modules/eks-managed-scan-access" + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arn] + nullify_region = var.nullify_region + authorization = var.authorization + kubernetes_group_name = var.kubernetes_group_name + tags = var.tags +} + +# Creating the ClusterRole needs rights to grant every permission in it, which +# in practice means cluster-admin. Set manage_rbac = false when Helm, Flux or +# Argo CD applies the nullify-k8s-readonly-access chart instead. +module "k8s_resources" { + source = "../../modules/k8s-resources" + count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 + + enable_collector = false + enable_managed_scan_rbac = true + managed_scan_kubernetes_group = var.kubernetes_group_name +} diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf new file mode 100644 index 0000000..91b6cdc --- /dev/null +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf @@ -0,0 +1,19 @@ +output "role_arn" { + description = "ARN of the Nullify read-only role" + value = module.nullify_aws_integration.role_arn +} + +output "clusters_for_nullify" { + description = "Clusters to register in Nullify" + value = module.eks_managed_scan_access.clusters_for_nullify +} + +output "kubernetes_group_name" { + description = "Kubernetes group the RBAC binding names (null for admin_view_policy)" + value = module.eks_managed_scan_access.kubernetes_group_name +} + +output "endpoint_allowlist" { + description = "Whether Nullify's egress IPs can reach the cluster's public endpoint, and the command that adds any missing CIDRs" + value = module.eks_managed_scan_access.endpoint_allowlist +} diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example new file mode 100644 index 0000000..e17c886 --- /dev/null +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example @@ -0,0 +1,25 @@ +# Managed EKS scan example - no in-cluster agent +# Copy this file to terraform.tfvars and fill in your values. + +# REQUIRED: Customer name (provided by Nullify) +customer_name = "acme-corp" + +# REQUIRED: External ID for cross-account role assumption (provided by Nullify) +external_id = "your-external-id-from-nullify" + +# REQUIRED: Nullify cross-account role ARN (provided by Nullify) +nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" + +# REQUIRED: the cluster to scan. Its authentication mode must be API or API_AND_CONFIG_MAP. +eks_cluster_arn = "arn:aws:eks:eu-west-1:123456789012:cluster/prod" + +# REQUIRED: your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2) +nullify_region = "eu-central-1" + +# rbac (recommended): list-only ClusterRole bound to the access entry's group. +# admin_view_policy: AmazonEKSAdminViewPolicy, no Kubernetes objects, but it reads +# every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. +# authorization = "rbac" + +# Set false if Helm, Flux or Argo CD applies the nullify-k8s-readonly-access chart. +# manage_rbac = true diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf new file mode 100644 index 0000000..f1b6096 --- /dev/null +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf @@ -0,0 +1,61 @@ +variable "customer_name" { + type = string + description = "The name of the customer to create the role for" + + validation { + condition = can(regex("^[a-zA-Z][a-zA-Z0-9_-]*$", var.customer_name)) + error_message = "Customer name must start with a letter and can only contain letters, numbers, underscores, and hyphens" + } +} + +variable "external_id" { + type = string + description = "The external ID for the role (provided by Nullify)" +} + +variable "nullify_role_arn" { + type = string + description = "The Nullify cross-account role ARN" +} + +variable "eks_cluster_arn" { + type = string + description = "ARN of the EKS cluster Nullify scans. The role and access entry are created in this cluster's account and region" + + validation { + condition = can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", var.eks_cluster_arn)) + error_message = "Must be an EKS cluster ARN: arn:aws:eks:::cluster/" + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). It selects the egress IPs the cluster's public endpoint must admit" +} + +variable "authorization" { + type = string + description = "rbac (recommended) or admin_view_policy. admin_view_policy grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" + default = "rbac" +} + +variable "kubernetes_group_name" { + type = string + description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole in rbac mode" + default = "nullify-readonly" +} + +variable "manage_rbac" { + type = bool + description = "In rbac mode, create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" + default = true +} + +variable "tags" { + type = map(string) + description = "Tags to apply to AWS resources" + default = { + ManagedBy = "Terraform" + Purpose = "NullifyIntegration" + } +} diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index 0c386e4..bf93a39 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -20,6 +20,9 @@ locals { secondary_region = split(":", var.eks_cluster_arns[1])[3] collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name + + collector = contains(["collector", "both"], var.scan_mode) + managed = contains(["managed", "both"], var.scan_mode) } provider "aws" { @@ -71,7 +74,7 @@ module "nullify_aws_integration" { nullify_s3_access_point_arn = var.nullify_s3_access_point_arn kms_key_arn = var.kms_key_arn - enable_kubernetes_integration = true + enable_kubernetes_integration = local.collector eks_cluster_arns = var.eks_cluster_arns kubernetes_namespace = var.kubernetes_namespace tags = var.tags @@ -89,6 +92,9 @@ module "k8s_resources_primary" { kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image + + enable_collector = local.collector + enable_managed_scan_rbac = local.managed } module "k8s_resources_secondary" { @@ -103,4 +109,18 @@ module "k8s_resources_secondary" { kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image + + enable_collector = local.collector + enable_managed_scan_rbac = local.managed +} + +module "eks_managed_scan_access" { + source = "../../modules/eks-managed-scan-access" + count = local.managed ? 1 : 0 + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.eks_cluster_arns + nullify_region = var.nullify_region + tags = var.tags } diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf index 1ddbfea..c8524e4 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf @@ -18,21 +18,34 @@ output "cluster_integration_summary" { output "k8s_resources_primary" { description = "Kubernetes resources deployed to primary cluster" value = { - namespace_name = module.k8s_resources_primary.namespace_name - service_account_name = module.k8s_resources_primary.service_account_name - cluster_role_name = module.k8s_resources_primary.cluster_role_name - cluster_role_binding_name = module.k8s_resources_primary.cluster_role_binding_name - cronjob_name = module.k8s_resources_primary.cronjob_name + namespace_name = module.k8s_resources_primary.namespace_name + service_account_name = module.k8s_resources_primary.service_account_name + cluster_role_name = module.k8s_resources_primary.cluster_role_name + cluster_role_binding_name = module.k8s_resources_primary.cluster_role_binding_name + cronjob_name = module.k8s_resources_primary.cronjob_name + managed_scan_cluster_role_name = module.k8s_resources_primary.managed_scan_cluster_role_name } } output "k8s_resources_secondary" { description = "Kubernetes resources deployed to secondary cluster" value = { - namespace_name = module.k8s_resources_secondary.namespace_name - service_account_name = module.k8s_resources_secondary.service_account_name - cluster_role_name = module.k8s_resources_secondary.cluster_role_name - cluster_role_binding_name = module.k8s_resources_secondary.cluster_role_binding_name - cronjob_name = module.k8s_resources_secondary.cronjob_name + namespace_name = module.k8s_resources_secondary.namespace_name + service_account_name = module.k8s_resources_secondary.service_account_name + cluster_role_name = module.k8s_resources_secondary.cluster_role_name + cluster_role_binding_name = module.k8s_resources_secondary.cluster_role_binding_name + cronjob_name = module.k8s_resources_secondary.cronjob_name + managed_scan_cluster_role_name = module.k8s_resources_secondary.managed_scan_cluster_role_name } -} \ No newline at end of file +} + +# Managed EKS scan (null unless scan_mode is managed or both) +output "managed_scan_clusters_for_nullify" { + description = "Clusters to register in Nullify for the managed scan" + value = one(module.eks_managed_scan_access[*].clusters_for_nullify) +} + +output "managed_scan_endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, and a command that adds any missing CIDRs" + value = one(module.eks_managed_scan_access[*].endpoint_allowlist) +} diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example index fc469ac..ebca4aa 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example @@ -16,6 +16,13 @@ eks_cluster_arns = [ "arn:aws:eks:us-east-1:123456789012:cluster/secondary-cluster" ] +# collector (default): in-cluster CronJob. managed: EKS access entry plus the list-only +# nullify-readonly ClusterRole, nothing runs in the cluster. both: deploy both. +# scan_mode = "managed" + +# Your Nullify region from the configure page, required for managed or both +# nullify_region = "eu-central-1" + # Primary AWS region (where IAM resources are created) aws_region = "ap-southeast-2" diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf index b6eb7ad..7131a36 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf @@ -30,6 +30,23 @@ variable "eks_cluster_arns" { } } +variable "scan_mode" { + type = string + description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. both: deploy both" + default = "collector" + + validation { + condition = contains(["collector", "managed", "both"], var.scan_mode) + error_message = "scan_mode must be collector, managed or both" + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed or both" + default = "" +} + variable "aws_region" { type = string description = "The primary AWS region for the integration (where IAM resources are created)" diff --git a/aws-integration-setup/terraform-v6/main.tf b/aws-integration-setup/terraform-v6/main.tf index e6bc376..6cb2142 100644 --- a/aws-integration-setup/terraform-v6/main.tf +++ b/aws-integration-setup/terraform-v6/main.tf @@ -17,3 +17,16 @@ module "nullify_aws_integration" { service_account_name = var.service_account_name tags = var.tags } + +module "eks_managed_scan_access" { + source = "./modules/eks-managed-scan-access" + count = var.enable_managed_scan ? 1 : 0 + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.managed_scan_cluster_arns + nullify_region = var.nullify_region + authorization = var.managed_scan_authorization + kubernetes_group_name = var.managed_scan_kubernetes_group + tags = var.tags +} diff --git a/aws-integration-setup/terraform-v6/outputs.tf b/aws-integration-setup/terraform-v6/outputs.tf index d557ab0..c3f7bf0 100644 --- a/aws-integration-setup/terraform-v6/outputs.tf +++ b/aws-integration-setup/terraform-v6/outputs.tf @@ -50,4 +50,24 @@ output "policy_arns" { output "deployment_summary" { description = "Summary of the Nullify integration deployment" value = module.nullify_aws_integration.deployment_summary -} \ No newline at end of file +} + +output "managed_scan_clusters_for_nullify" { + description = "Clusters to register in Nullify for the managed scan (null when enable_managed_scan is false)" + value = one(module.eks_managed_scan_access[*].clusters_for_nullify) +} + +output "managed_scan_kubernetes_group" { + description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled or for admin_view_policy)" + value = one(module.eks_managed_scan_access[*].kubernetes_group_name) +} + +output "managed_scan_endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, and a command that adds any missing CIDRs" + value = one(module.eks_managed_scan_access[*].endpoint_allowlist) +} + +output "nullify_egress_cidrs" { + description = "Nullify egress CIDRs for nullify_region (null when enable_managed_scan is false)" + value = one(module.eks_managed_scan_access[*].nullify_egress_cidrs) +} diff --git a/aws-integration-setup/terraform-v6/terraform.tfvars.example b/aws-integration-setup/terraform-v6/terraform.tfvars.example index 0cfe9d3..964e8ce 100644 --- a/aws-integration-setup/terraform-v6/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/terraform.tfvars.example @@ -31,6 +31,14 @@ enable_kubernetes_integration = false # "arn:aws:eks:ap-southeast-2:123456789012:cluster/my-cluster" # ] +# Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. +# Clusters can be in any region. In rbac mode, bind the group with the nullify-k8s-readonly-access +# Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. +# enable_managed_scan = true +# managed_scan_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/my-cluster"] +# nullify_region = "eu-central-1" +# managed_scan_authorization = "rbac" + # Kubernetes namespace for Nullify resources # kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index 9f924bb..462ac9a 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -95,3 +95,33 @@ variable "kms_key_arn" { description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" } + +variable "enable_managed_scan" { + type = bool + description = "Create EKS access entries so Nullify's managed scan can list Kubernetes resources with no in-cluster agent" + default = false +} + +variable "managed_scan_cluster_arns" { + type = list(string) + description = "EKS clusters for the managed scan, in any region of this account" + default = [] +} + +variable "managed_scan_authorization" { + type = string + description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to the list-only nullify-readonly ClusterRole with the nullify-k8s-readonly-access Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + default = "rbac" +} + +variable "managed_scan_kubernetes_group" { + type = string + description = "Kubernetes group on the access entries in rbac mode" + default = "nullify-readonly" +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when enable_managed_scan is true" + default = "" +} diff --git a/aws-integration-setup/terraform/examples/managed-scan/main.tf b/aws-integration-setup/terraform/examples/managed-scan/main.tf new file mode 100644 index 0000000..b1870b8 --- /dev/null +++ b/aws-integration-setup/terraform/examples/managed-scan/main.tf @@ -0,0 +1,72 @@ +# Managed EKS scan: Nullify lists Kubernetes resources from its own compute +# through an EKS access entry. Nothing runs in the cluster. + +terraform { + required_version = ">= 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.33" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.20" + } + } +} + +locals { + cluster_name = split("/", var.eks_cluster_arn)[1] + cluster_region = split(":", var.eks_cluster_arn)[3] +} + +provider "aws" { + region = local.cluster_region +} + +data "aws_eks_cluster" "this" { + name = local.cluster_name +} + +data "aws_eks_cluster_auth" "this" { + name = local.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token +} + +module "nullify_aws_integration" { + source = "../../modules/nullify-aws-integration" + customer_name = var.customer_name + external_id = var.external_id + nullify_role_arn = var.nullify_role_arn + aws_region = local.cluster_region + tags = var.tags +} + +module "eks_managed_scan_access" { + source = "../../modules/eks-managed-scan-access" + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arn] + nullify_region = var.nullify_region + authorization = var.authorization + kubernetes_group_name = var.kubernetes_group_name + tags = var.tags +} + +# Creating the ClusterRole needs rights to grant every permission in it, which +# in practice means cluster-admin. Set manage_rbac = false when Helm, Flux or +# Argo CD applies the nullify-k8s-readonly-access chart instead. +module "k8s_resources" { + source = "../../modules/k8s-resources" + count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 + + enable_collector = false + enable_managed_scan_rbac = true + managed_scan_kubernetes_group = var.kubernetes_group_name +} diff --git a/aws-integration-setup/terraform/examples/managed-scan/outputs.tf b/aws-integration-setup/terraform/examples/managed-scan/outputs.tf new file mode 100644 index 0000000..91b6cdc --- /dev/null +++ b/aws-integration-setup/terraform/examples/managed-scan/outputs.tf @@ -0,0 +1,19 @@ +output "role_arn" { + description = "ARN of the Nullify read-only role" + value = module.nullify_aws_integration.role_arn +} + +output "clusters_for_nullify" { + description = "Clusters to register in Nullify" + value = module.eks_managed_scan_access.clusters_for_nullify +} + +output "kubernetes_group_name" { + description = "Kubernetes group the RBAC binding names (null for admin_view_policy)" + value = module.eks_managed_scan_access.kubernetes_group_name +} + +output "endpoint_allowlist" { + description = "Whether Nullify's egress IPs can reach the cluster's public endpoint, and the command that adds any missing CIDRs" + value = module.eks_managed_scan_access.endpoint_allowlist +} diff --git a/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example new file mode 100644 index 0000000..e17c886 --- /dev/null +++ b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example @@ -0,0 +1,25 @@ +# Managed EKS scan example - no in-cluster agent +# Copy this file to terraform.tfvars and fill in your values. + +# REQUIRED: Customer name (provided by Nullify) +customer_name = "acme-corp" + +# REQUIRED: External ID for cross-account role assumption (provided by Nullify) +external_id = "your-external-id-from-nullify" + +# REQUIRED: Nullify cross-account role ARN (provided by Nullify) +nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" + +# REQUIRED: the cluster to scan. Its authentication mode must be API or API_AND_CONFIG_MAP. +eks_cluster_arn = "arn:aws:eks:eu-west-1:123456789012:cluster/prod" + +# REQUIRED: your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2) +nullify_region = "eu-central-1" + +# rbac (recommended): list-only ClusterRole bound to the access entry's group. +# admin_view_policy: AmazonEKSAdminViewPolicy, no Kubernetes objects, but it reads +# every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. +# authorization = "rbac" + +# Set false if Helm, Flux or Argo CD applies the nullify-k8s-readonly-access chart. +# manage_rbac = true diff --git a/aws-integration-setup/terraform/examples/managed-scan/variables.tf b/aws-integration-setup/terraform/examples/managed-scan/variables.tf new file mode 100644 index 0000000..f1b6096 --- /dev/null +++ b/aws-integration-setup/terraform/examples/managed-scan/variables.tf @@ -0,0 +1,61 @@ +variable "customer_name" { + type = string + description = "The name of the customer to create the role for" + + validation { + condition = can(regex("^[a-zA-Z][a-zA-Z0-9_-]*$", var.customer_name)) + error_message = "Customer name must start with a letter and can only contain letters, numbers, underscores, and hyphens" + } +} + +variable "external_id" { + type = string + description = "The external ID for the role (provided by Nullify)" +} + +variable "nullify_role_arn" { + type = string + description = "The Nullify cross-account role ARN" +} + +variable "eks_cluster_arn" { + type = string + description = "ARN of the EKS cluster Nullify scans. The role and access entry are created in this cluster's account and region" + + validation { + condition = can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", var.eks_cluster_arn)) + error_message = "Must be an EKS cluster ARN: arn:aws:eks:::cluster/" + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). It selects the egress IPs the cluster's public endpoint must admit" +} + +variable "authorization" { + type = string + description = "rbac (recommended) or admin_view_policy. admin_view_policy grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" + default = "rbac" +} + +variable "kubernetes_group_name" { + type = string + description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole in rbac mode" + default = "nullify-readonly" +} + +variable "manage_rbac" { + type = bool + description = "In rbac mode, create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" + default = true +} + +variable "tags" { + type = map(string) + description = "Tags to apply to AWS resources" + default = { + ManagedBy = "Terraform" + Purpose = "NullifyIntegration" + } +} diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index 821bebc..7284739 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.0" + version = "~> 5.33" } kubernetes = { source = "hashicorp/kubernetes" @@ -20,6 +20,9 @@ locals { secondary_region = split(":", var.eks_cluster_arns[1])[3] collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name + + collector = contains(["collector", "both"], var.scan_mode) + managed = contains(["managed", "both"], var.scan_mode) } provider "aws" { @@ -80,7 +83,7 @@ module "nullify_aws_integration" { nullify_s3_access_point_arn = var.nullify_s3_access_point_arn kms_key_arn = var.kms_key_arn - enable_kubernetes_integration = true + enable_kubernetes_integration = local.collector eks_cluster_arns = var.eks_cluster_arns kubernetes_namespace = var.kubernetes_namespace tags = var.tags @@ -105,6 +108,9 @@ module "k8s_resources_primary" { kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image + + enable_collector = local.collector + enable_managed_scan_rbac = local.managed } module "k8s_resources_secondary" { @@ -119,4 +125,37 @@ module "k8s_resources_secondary" { kubernetes_namespace = var.kubernetes_namespace cronjob_schedule = var.cronjob_schedule collector_image = var.collector_image + + enable_collector = local.collector + enable_managed_scan_rbac = local.managed +} + +# AWS provider v5 manages access entries only in the provider's region, so +# each cluster gets a module instance on its regional provider. +module "eks_managed_scan_access_primary" { + source = "../../modules/eks-managed-scan-access" + count = local.managed ? 1 : 0 + providers = { + aws = aws.primary + } + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[0]] + nullify_region = var.nullify_region + tags = var.tags +} + +module "eks_managed_scan_access_secondary" { + source = "../../modules/eks-managed-scan-access" + count = local.managed ? 1 : 0 + providers = { + aws = aws.secondary + } + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[1]] + nullify_region = var.nullify_region + tags = var.tags } diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf index 09a0c67..e059fbd 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf @@ -18,21 +18,34 @@ output "cluster_integration_summary" { output "k8s_resources_primary" { description = "Kubernetes resources deployed to primary cluster" value = { - namespace_name = module.k8s_resources_primary.namespace_name - service_account_name = module.k8s_resources_primary.service_account_name - cluster_role_name = module.k8s_resources_primary.cluster_role_name - cluster_role_binding_name = module.k8s_resources_primary.cluster_role_binding_name - cronjob_name = module.k8s_resources_primary.cronjob_name + namespace_name = module.k8s_resources_primary.namespace_name + service_account_name = module.k8s_resources_primary.service_account_name + cluster_role_name = module.k8s_resources_primary.cluster_role_name + cluster_role_binding_name = module.k8s_resources_primary.cluster_role_binding_name + cronjob_name = module.k8s_resources_primary.cronjob_name + managed_scan_cluster_role_name = module.k8s_resources_primary.managed_scan_cluster_role_name } } output "k8s_resources_secondary" { description = "Kubernetes resources deployed to secondary cluster" value = { - namespace_name = module.k8s_resources_secondary.namespace_name - service_account_name = module.k8s_resources_secondary.service_account_name - cluster_role_name = module.k8s_resources_secondary.cluster_role_name - cluster_role_binding_name = module.k8s_resources_secondary.cluster_role_binding_name - cronjob_name = module.k8s_resources_secondary.cronjob_name + namespace_name = module.k8s_resources_secondary.namespace_name + service_account_name = module.k8s_resources_secondary.service_account_name + cluster_role_name = module.k8s_resources_secondary.cluster_role_name + cluster_role_binding_name = module.k8s_resources_secondary.cluster_role_binding_name + cronjob_name = module.k8s_resources_secondary.cronjob_name + managed_scan_cluster_role_name = module.k8s_resources_secondary.managed_scan_cluster_role_name } } + +# Managed EKS scan (empty unless scan_mode is managed or both) +output "managed_scan_clusters_for_nullify" { + description = "Clusters to register in Nullify for the managed scan" + value = flatten(concat(module.eks_managed_scan_access_primary[*].clusters_for_nullify, module.eks_managed_scan_access_secondary[*].clusters_for_nullify)) +} + +output "managed_scan_endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, and a command that adds any missing CIDRs" + value = merge(concat(module.eks_managed_scan_access_primary[*].endpoint_allowlist, module.eks_managed_scan_access_secondary[*].endpoint_allowlist)...) +} diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example index fc469ac..ebca4aa 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example @@ -16,6 +16,13 @@ eks_cluster_arns = [ "arn:aws:eks:us-east-1:123456789012:cluster/secondary-cluster" ] +# collector (default): in-cluster CronJob. managed: EKS access entry plus the list-only +# nullify-readonly ClusterRole, nothing runs in the cluster. both: deploy both. +# scan_mode = "managed" + +# Your Nullify region from the configure page, required for managed or both +# nullify_region = "eu-central-1" + # Primary AWS region (where IAM resources are created) aws_region = "ap-southeast-2" diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf index b6eb7ad..7131a36 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf @@ -30,6 +30,23 @@ variable "eks_cluster_arns" { } } +variable "scan_mode" { + type = string + description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. both: deploy both" + default = "collector" + + validation { + condition = contains(["collector", "managed", "both"], var.scan_mode) + error_message = "scan_mode must be collector, managed or both" + } +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed or both" + default = "" +} + variable "aws_region" { type = string description = "The primary AWS region for the integration (where IAM resources are created)" diff --git a/aws-integration-setup/terraform/main.tf b/aws-integration-setup/terraform/main.tf index 0a52fa7..bbcd6a9 100644 --- a/aws-integration-setup/terraform/main.tf +++ b/aws-integration-setup/terraform/main.tf @@ -18,3 +18,16 @@ module "nullify_aws_integration" { service_account_name = var.service_account_name tags = var.tags } + +module "eks_managed_scan_access" { + source = "./modules/eks-managed-scan-access" + count = var.enable_managed_scan ? 1 : 0 + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.managed_scan_cluster_arns + nullify_region = var.nullify_region + authorization = var.managed_scan_authorization + kubernetes_group_name = var.managed_scan_kubernetes_group + tags = var.tags +} diff --git a/aws-integration-setup/terraform/outputs.tf b/aws-integration-setup/terraform/outputs.tf index 11f2af6..c3f7bf0 100644 --- a/aws-integration-setup/terraform/outputs.tf +++ b/aws-integration-setup/terraform/outputs.tf @@ -50,4 +50,24 @@ output "policy_arns" { output "deployment_summary" { description = "Summary of the Nullify integration deployment" value = module.nullify_aws_integration.deployment_summary -} \ No newline at end of file +} + +output "managed_scan_clusters_for_nullify" { + description = "Clusters to register in Nullify for the managed scan (null when enable_managed_scan is false)" + value = one(module.eks_managed_scan_access[*].clusters_for_nullify) +} + +output "managed_scan_kubernetes_group" { + description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled or for admin_view_policy)" + value = one(module.eks_managed_scan_access[*].kubernetes_group_name) +} + +output "managed_scan_endpoint_allowlist" { + description = "Per cluster: whether Nullify's egress IPs can reach the public endpoint, and a command that adds any missing CIDRs" + value = one(module.eks_managed_scan_access[*].endpoint_allowlist) +} + +output "nullify_egress_cidrs" { + description = "Nullify egress CIDRs for nullify_region (null when enable_managed_scan is false)" + value = one(module.eks_managed_scan_access[*].nullify_egress_cidrs) +} diff --git a/aws-integration-setup/terraform/terraform.tfvars.example b/aws-integration-setup/terraform/terraform.tfvars.example index d97fa0b..3f33b53 100644 --- a/aws-integration-setup/terraform/terraform.tfvars.example +++ b/aws-integration-setup/terraform/terraform.tfvars.example @@ -36,6 +36,14 @@ enable_kubernetes_integration = false # "https://oidc.eks.ap-southeast-2.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" # ] +# Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. +# Clusters must be in aws_region. In rbac mode, bind the group with the nullify-k8s-readonly-access +# Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. +# enable_managed_scan = true +# managed_scan_cluster_arns = ["arn:aws:eks:ap-southeast-2:123456789012:cluster/my-cluster"] +# nullify_region = "ap-southeast-2" +# managed_scan_authorization = "rbac" + # Kubernetes namespace for Nullify resources # kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index b395f47..afcf7cc 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -101,3 +101,33 @@ variable "kms_key_arn" { description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" default = "" } + +variable "enable_managed_scan" { + type = bool + description = "Create EKS access entries so Nullify's managed scan can list Kubernetes resources with no in-cluster agent" + default = false +} + +variable "managed_scan_cluster_arns" { + type = list(string) + description = "EKS clusters for the managed scan, all in aws_region (AWS provider v5 manages access entries only in the provider's region)" + default = [] +} + +variable "managed_scan_authorization" { + type = string + description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to the list-only nullify-readonly ClusterRole with the nullify-k8s-readonly-access Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + default = "rbac" +} + +variable "managed_scan_kubernetes_group" { + type = string + description = "Kubernetes group on the access entries in rbac mode" + default = "nullify-readonly" +} + +variable "nullify_region" { + type = string + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when enable_managed_scan is true" + default = "" +} diff --git a/aws-integration-setup/terraform/versions.tf b/aws-integration-setup/terraform/versions.tf index 946e64b..f7c9efb 100644 --- a/aws-integration-setup/terraform/versions.tf +++ b/aws-integration-setup/terraform/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.0" + version = "~> 5.33" } } } From edf08255574366fa951a2434bf2f8db8e441d920 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 06:07:28 +1000 Subject: [PATCH 14/36] docs(aws): document the managed EKS scan modules, RBAC, endpoint allowlist and upgrade notes Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform-v6/README.md | 61 +++++-- aws-integration-setup/terraform/README.md | 173 ++++++++++++++++--- 2 files changed, 195 insertions(+), 39 deletions(-) diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index 7c61be6..c46424c 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -2,12 +2,14 @@ This directory mirrors `../terraform/` but targets **AWS Terraform provider v6** (`~> 6.0`). Use this if your project has already upgraded to AWS provider v6. -For AWS provider v5, use `../terraform/` instead. `../terraform/README.md` documents the modules, variables and outputs; this file covers what differs. +For AWS provider v5, use `../terraform/` instead. `../terraform/README.md` documents the modules, variables, outputs, the managed EKS scan and its prerequisites; this file covers what differs. ## What Differs from v5 -- **Per-resource `region`.** AWS provider v6 accepts a `region` argument on most resources and data sources. `nullify-aws-integration` reads each EKS cluster in the region from its ARN, so `eks_cluster_arns` can mix regions with one provider. There is no `eks_oidc_issuer_urls` variable here. -- **Ephemeral EKS token.** `examples/multi-cluster-complete` reads the cluster token with `ephemeral "aws_eks_cluster_auth"`, which keeps it out of plan and state and needs Terraform >= 1.10: +- **Per-resource `region`.** AWS provider v6 accepts a `region` argument on most resources and data sources. + - `nullify-aws-integration` reads each EKS cluster in the region from its ARN, so `eks_cluster_arns` can mix regions with one provider. There is no `eks_oidc_issuer_urls` variable here. + - `eks-managed-scan-access` creates each access entry in its cluster's region, so one module instance covers every region. `managed_scan_cluster_arns` in the root can mix regions. +- **Ephemeral EKS token.** The `multi-cluster-complete` and `managed-scan` examples read the cluster token with `ephemeral "aws_eks_cluster_auth"`, which keeps it out of plan and state and needs Terraform >= 1.10: ```hcl ephemeral "aws_eks_cluster_auth" "primary" { @@ -19,9 +21,9 @@ token = ephemeral.aws_eks_cluster_auth.primary.token ## Requirements -- Terraform >= 1.5 (>= 1.10 for `examples/multi-cluster-complete`) +- Terraform >= 1.5 (>= 1.10 for the `multi-cluster-complete` and `managed-scan` examples) - AWS provider ~> 6.0 -- Kubernetes provider ~> 2.20 (only `modules/k8s-resources` and `examples/multi-cluster-complete`) +- Kubernetes provider ~> 2.20 (only `modules/k8s-resources` and the `multi-cluster-complete` and `managed-scan` examples) ## Architecture @@ -29,20 +31,13 @@ token = ephemeral.aws_eks_cluster_auth.primary.token terraform-v6/ ├── modules/ │ ├── nullify-aws-integration/ # AWS IAM resources only -│ │ ├── versions.tf # AWS provider ~> 6.0 -│ │ ├── variables.tf -│ │ ├── locals.tf -│ │ ├── data.tf -│ │ ├── main.tf -│ │ └── outputs.tf -│ └── k8s-resources/ # Kubernetes resources only -│ ├── providers.tf -│ ├── variables.tf -│ ├── main.tf -│ └── outputs.tf +│ ├── eks-managed-scan-access/ # EKS access entries for the managed scan +│ └── k8s-resources/ # Collector and/or managed-scan RBAC +│ (each module has a tests/ terraform test suite) ├── examples/ │ ├── basic/ # AWS IAM only example -│ └── multi-cluster-complete/ # Two-cluster EKS example +│ ├── managed-scan/ # One cluster, managed scan, no in-cluster agent +│ └── multi-cluster-complete/ # Two clusters, collector and/or managed scan ├── versions.tf # AWS ~> 6.0 ├── variables.tf ├── main.tf @@ -63,15 +58,42 @@ cp terraform.tfvars.example terraform.tfvars terraform init && terraform apply ``` -### 2. Multi-Cluster EKS Integration +### 2. Managed EKS Scan (no in-cluster agent) + +```bash +cd examples/managed-scan/ +cp terraform.tfvars.example terraform.tfvars +# Edit with your cluster ARN and Nullify region +terraform init && terraform apply +``` + +### 3. Multi-Cluster EKS Integration ```bash cd examples/multi-cluster-complete/ cp terraform.tfvars.example terraform.tfvars -# Edit with your cluster ARNs and values +# Edit with your cluster ARNs and values; set scan_mode for the managed scan terraform init && terraform apply ``` +## Managed EKS scan in one module instance + +```hcl +module "nullify_eks_access" { + source = "./modules/eks-managed-scan-access" + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + nullify_region = "eu-central-1" + cluster_arns = [ + "arn:aws:eks:eu-west-1:123456789012:cluster/prod", + "arn:aws:eks:us-east-1:123456789012:cluster/us-prod", + ] +} +``` + +The default `authorization = "rbac"` needs the list-only `nullify-readonly` ClusterRole on each cluster (`k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true`, or the `nullify-k8s-readonly-access` Helm chart). `admin_view_policy` is a warned opt-in; see `../terraform/README.md`. + ## Required Variables - `customer_name`: Your company/customer name (used in resource naming) @@ -86,6 +108,7 @@ terraform init && terraform apply - `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page (optional); see "KMS" in `../terraform/README.md` - `enable_kubernetes_integration`: Set to `true` for EKS integration - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with, in any region +- `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: managed EKS scan - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) - `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) - `tags`: Resource tags diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index d7bfd16..0171cd0 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -6,13 +6,13 @@ This directory targets AWS provider v5. If your project uses AWS provider v6, us ## ⚠️ Important Notice -**EKS Integration**: The AWS integration module only sets up IAM roles and trust policies. For full EKS integration, you must also deploy the Kubernetes resources using the separate `k8s-resources` module or Helm charts. +**EKS Integration**: The AWS integration module only sets up IAM roles and trust policies. Kubernetes access needs one of the two scan modes in [Kubernetes scan modes](#kubernetes-scan-modes). ## Requirements - Terraform >= 1.5 -- AWS provider `~> 5.0` -- Kubernetes provider `~> 2.20` (only `modules/k8s-resources` and `examples/multi-cluster-complete`) +- AWS provider `~> 5.33` in the root and examples. `modules/eks-managed-scan-access` needs `>= 5.33` (access entries arrived in 5.33.0); `modules/nullify-aws-integration` still accepts `~> 5.0` +- Kubernetes provider `~> 2.20` (only `modules/k8s-resources` and the `multi-cluster-complete` and `managed-scan` examples) ## Architecture @@ -27,15 +27,23 @@ terraform/ │ │ ├── locals.tf # Module local values │ │ ├── data.tf # Data sources and policies │ │ ├── main.tf # Core IAM resources -│ │ └── outputs.tf # Module outputs +│ │ ├── outputs.tf # Module outputs +│ │ └── tests/ # terraform test suite (mocked providers) +│ ├── eks-managed-scan-access/ # EKS access entries for the managed scan (AWS provider only) +│ │ ├── versions.tf, variables.tf, main.tf, outputs.tf +│ │ └── tests/ │ └── k8s-resources/ # Kubernetes resources only │ ├── providers.tf # Kubernetes provider requirements │ ├── variables.tf # Module input variables -│ ├── main.tf # Kubernetes resources -│ └── outputs.tf # Module outputs +│ ├── main.tf # In-cluster collector +│ ├── managed_scan.tf # List-only ClusterRole for the managed scan +│ ├── moved.tf # Keeps pre-toggle collector state in place +│ ├── outputs.tf # Module outputs +│ └── tests/ ├── examples/ │ ├── basic/ # AWS IAM only example -│ └── multi-cluster-complete/ # Two-cluster EKS example +│ ├── managed-scan/ # One cluster, managed scan, no in-cluster agent +│ └── multi-cluster-complete/ # Two clusters, collector and/or managed scan ├── versions.tf # Root provider requirements ├── providers.tf # AWS provider ├── variables.tf # Root input variables @@ -48,7 +56,8 @@ terraform/ ## Module Separation Benefits - **AWS Module**: Creates IAM roles with multi-cluster OIDC trust policies -- **K8s Module**: Deploys collector cronjob and RBAC to any cluster +- **EKS Access Module**: Creates EKS access entries; needs no connection to the cluster API +- **K8s Module**: Deploys the collector CronJob and/or the managed-scan RBAC to any cluster - **Independent Deployment**: Deploy AWS resources once, K8s resources per cluster - **Multi-Region Support**: Region taken from each cluster ARN - **Simplified Management**: Each module has focused responsibilities @@ -60,10 +69,122 @@ terraform/ - No Kubernetes integration - Minimal configuration for cloud-only scanning +### **Managed Scan** (`examples/managed-scan/`) +- One cluster, no in-cluster agent +- IAM role, access entry, and the list-only `nullify-readonly` ClusterRole + ### **Multi-Cluster Complete** (`examples/multi-cluster-complete/`) -- Full AWS IAM + EKS integration for exactly two clusters +- AWS IAM + EKS integration for exactly two clusters +- `scan_mode`: `collector` (default), `managed` or `both` - Clusters can be in different regions: each is read through an AWS provider in its own region -- Automatic OIDC provider discovery + +## Kubernetes scan modes + +| | In-cluster collector | Managed scan | +|---|---|---| +| Runs in the cluster | CronJob with an IRSA service account | Nothing | +| Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | +| AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | +| Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true`, the `nullify-k8s-readonly-access` Helm chart, or `manifests/nullify-readonly-rbac.yaml` | +| Upgrades | You update the image | None | + +Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. + +## Managed EKS scan + +Nullify lists Kubernetes resources from its own compute, using the read-only role this configuration creates. The scan only calls `list`, plus `get /version`. + +### RBAC granted to Nullify (default, `authorization = "rbac"`) + +The access entry carries the Kubernetes group `nullify-readonly`, bound to the ClusterRole `nullify-readonly`: + +| API group | Resources (verb `list`) | +|---|---| +| core | nodes, namespaces, pods, services, persistentvolumeclaims, persistentvolumes, configmaps, secrets, resourcequotas, limitranges, serviceaccounts | +| apps | deployments, daemonsets, statefulsets, replicasets | +| networking.k8s.io | ingresses, networkpolicies | +| discovery.k8s.io | endpointslices | +| rbac.authorization.k8s.io | roles, rolebindings, clusterroles, clusterrolebindings | +| admissionregistration.k8s.io | validatingwebhookconfigurations, mutatingwebhookconfigurations, validatingadmissionpolicies, validatingadmissionpolicybindings | + +Plus `get` on the non-resource URL `/version`. Kubernetes has no metadata-only permission for Secrets: `list secrets` permits reading values, whichever mode you pick. + +### `authorization = "admin_view_policy"` (opt-in) + +Associates `AmazonEKSAdminViewPolicy` at cluster scope, so no Kubernetes objects are needed. **It grants far more than the scan uses:** `get`, `list` and `watch` on every resource, including Secrets, custom resources and `pods/log`. On EKS 1.34 and earlier, `get pods/exec` is enough to exec into pods. Its grants do not show in `kubectl auth can-i --list`. The module prints a warning on every plan in this mode. `AmazonEKSViewPolicy` is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects. + +### Prerequisites + +- **Authentication mode.** Access entries need `API` or `API_AND_CONFIG_MAP`. Check with `aws eks describe-cluster --name --query cluster.accessConfig.authenticationMode`. Switch (one-way) with `aws eks update-cluster-config --name --access-config authenticationMode=API_AND_CONFIG_MAP`. The module fails at plan for `CONFIG_MAP` clusters. +- **Same account.** Nullify assumes the role in the cluster's own account, so deploy `nullify-aws-integration` in every account that owns a scanned cluster. The module rejects clusters in another account. +- **One region per module instance (AWS provider v5).** Instantiate `eks-managed-scan-access` once per region with `providers = { aws = aws. }`, as `examples/multi-cluster-complete` does. `../terraform-v6/` handles every region in one instance. +- **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`, plus `eks:AssociateAccessPolicy`, `eks:DisassociateAccessPolicy` and `eks:ListAssociatedAccessPolicies` for `admin_view_policy`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the Helm chart instead. +- **Pass `principal_unique_id`.** EKS ties an access entry to the role's ID, so a recreated role with the same ARN is silently not authorized. With `principal_unique_id = module.nullify_aws_integration.role_unique_id`, the entries are replaced with the role. +- **Existing access entry.** If this role is already mapped on a cluster, import it: `terraform import 'module.eks_managed_scan_access.aws_eks_access_entry.nullify[""]' :`. + +### Allowing Nullify to reach the EKS API endpoint + +The managed scan connects to the cluster's public endpoint from Nullify's egress IPs for your Nullify region: + +| Nullify region | Egress IPs | +|---|---| +| ap-southeast-2 | 13.55.32.104, 3.105.146.106, 13.211.99.100 | +| eu-central-1 | 18.198.60.231, 18.157.227.250, 18.185.152.197 | +| us-east-2 | 52.15.146.50, 16.58.40.80, 3.133.15.210 | + +These modules never change `publicAccessCidrs`: the cluster belongs to your own IaC, and the API replaces the whole list. Instead: + +- A `check` block warns on every plan when a cluster's public endpoint is disabled or misses a Nullify IP. It matches exact CIDRs, so a wider range that already covers the IPs still warns. +- The `endpoint_allowlist` output lists the missing CIDRs per cluster and an `aws eks update-cluster-config` command that keeps the existing CIDRs. +- If the cluster is in Terraform, add the CIDRs there: + +```hcl +resource "aws_eks_cluster" "this" { + vpc_config { + endpoint_public_access = true + public_access_cidrs = concat(var.existing_public_access_cidrs, module.nullify_eks_access.nullify_egress_cidrs) + } +} +``` + +Private-only clusters cannot use the managed scan; use the in-cluster collector. + +### CONFIG_MAP clusters (aws-auth) + +These modules do not manage `aws-auth`: writing it from Terraform replaces the whole `mapRoles` string and can drop node roles. If you cannot switch the authentication mode, add the role yourself and apply the RBAC with `k8s-resources` or the Helm chart: + +```yaml +mapRoles: | + - rolearn: arn:aws:iam:::role/AWSIntegration--NullifyReadOnlyRole + username: nullify-managed-scan + groups: + - nullify-readonly +``` + +### Module usage + +```hcl +module "nullify_eks_access" { + source = "./modules/eks-managed-scan-access" + + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = ["arn:aws:eks:us-west-2:123456789012:cluster/my-cluster"] + nullify_region = "us-east-2" +} + +module "nullify_readonly_rbac" { + source = "./modules/k8s-resources" + providers = { kubernetes = kubernetes.my_cluster } + + enable_collector = false + enable_managed_scan_rbac = true +} +``` + +Outputs: `access_entry_arns`, `authorization`, `kubernetes_group_name`, `nullify_egress_cidrs`, `clusters_for_nullify`, `endpoint_allowlist`. + +The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. ## Multi-Cluster Support @@ -91,6 +212,8 @@ eks_oidc_issuer_urls = [ Get an issuer with `aws eks describe-cluster --region --name --query cluster.identity.oidc.issuer --output text`, or read it through a regional provider as `examples/multi-cluster-complete` does. `../terraform-v6/` looks every cluster up in its own region without this variable. +With several clusters in one apply, `data "aws_eks_cluster_auth"` tokens can expire (about 15 minutes) before the Kubernetes resources are created. Use the Kubernetes provider's `exec` block (`aws eks get-token`) for slow applies, or `../terraform-v6/`, whose examples use ephemeral tokens. + ## Quick Start ### 1. AWS-Only Integration @@ -102,7 +225,16 @@ cp terraform.tfvars.example terraform.tfvars terraform init && terraform apply ``` -### 2. Multi-Cluster EKS Integration +### 2. Managed EKS Scan (no in-cluster agent) + +```bash +cd examples/managed-scan/ +cp terraform.tfvars.example terraform.tfvars +# Edit with your cluster ARN and Nullify region +terraform init && terraform apply +``` + +### 3. Multi-Cluster EKS Integration ```bash cd examples/multi-cluster-complete/ @@ -122,6 +254,7 @@ terraform init && terraform apply - `enable_kubernetes_integration`: Set to `true` to trust the in-cluster collector's service account - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with - `eks_oidc_issuer_urls`: OIDC issuer per cluster, only needed for clusters outside `aws_region` +- `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: see [Managed EKS scan](#managed-eks-scan) ## Optional Variables @@ -183,24 +316,22 @@ module "k8s_resources" { } ``` -`k8s-resources` variables include `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. +`k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. `collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0`, a pinned tag of Nullify's ECR Public image. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. ## Outputs ### AWS Integration Module -- `role_arn`: ARN of the created IAM role +- `role_arn`, `role_unique_id`: the created IAM role - `deployment_summary`: Complete deployment information - `cluster_integration_summary`: Multi-cluster setup details - `all_oidc_ids`: List of OIDC provider IDs - `kms_policy_resources`: Resources granted by the KMS policy ### K8s Resources Module -- `namespace_name`: Created namespace name -- `service_account_name`: Service account name -- `cluster_role_name`, `cluster_role_binding_name`: Collector RBAC names -- `cronjob_name`: Data collector cronjob name +- `namespace_name`, `service_account_name`, `cluster_role_name`, `cluster_role_binding_name`, `cronjob_name`: collector resources (null when `enable_collector = false`) +- `managed_scan_cluster_role_name`, `managed_scan_cluster_role_binding_name`, `managed_scan_kubernetes_group`: managed-scan RBAC (null when disabled) ## Using Modules in Other Projects @@ -226,6 +357,8 @@ module "nullify_k8s" { ## Upgrade Notes - Terraform >= 1.5 is now required (the module already used cross-variable validation, which needs 1.9; those checks are now preconditions). +- The root and examples require AWS provider `>= 5.33`. Run `terraform init -upgrade` if your lock file pins an older 5.x. +- `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. - The KMS policy now also grants `key/*` in the account and region of `kms_key_arn`. @@ -245,11 +378,11 @@ module "nullify_k8s" { terraform fmt -recursive terraform validate +# Module tests (mocked providers, Terraform >= 1.7) +(cd modules/eks-managed-scan-access && terraform init -backend=false && terraform test) + # Plan deployment terraform plan - -# Check security -terraform show -json | jq '.values.root_module.resources[].values' ``` ## Support From d3b6852c3ed680224411d75d19bc10b924bc5cd1 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 06:07:28 +1000 Subject: [PATCH 15/36] ci: validate the managed-scan module and examples and run terraform test with mocked providers Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 47 +++++++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index 085cbd7..d8576e5 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -32,8 +32,8 @@ jobs: strategy: fail-fast: false matrix: - # AWS rows run the declared floor (Terraform 1.5); the v6 - # multi-cluster example uses ephemeral resources, which need 1.10. + # AWS rows run the declared floor (Terraform 1.5); examples that use + # ephemeral resources (v6 multi-cluster, v6 managed-scan) need 1.10. include: - directory: gcp-integration-setup/terraform terraform: 1.9.5 @@ -49,12 +49,20 @@ jobs: terraform: 1.5.7 - directory: aws-integration-setup/terraform/examples/multi-cluster-complete terraform: 1.5.7 + - directory: aws-integration-setup/terraform/examples/managed-scan + terraform: 1.5.7 + - directory: aws-integration-setup/terraform/modules/eks-managed-scan-access + terraform: 1.5.7 - directory: aws-integration-setup/terraform-v6 terraform: 1.5.7 - directory: aws-integration-setup/terraform-v6/examples/basic terraform: 1.5.7 - directory: aws-integration-setup/terraform-v6/examples/multi-cluster-complete terraform: 1.10.5 + - directory: aws-integration-setup/terraform-v6/examples/managed-scan + terraform: 1.10.5 + - directory: aws-integration-setup/terraform-v6/modules/eks-managed-scan-access + terraform: 1.5.7 - directory: azure-integration-setup/terraform terraform: 1.9.5 - directory: azure-integration-setup/terraform/examples/management-group @@ -81,6 +89,41 @@ jobs: working-directory: ${{ matrix.directory }} run: terraform validate + # Module logic (preconditions, checks, RBAC rules, KMS grants) against mocked + # providers. mock_provider needs Terraform 1.7+; no cloud credentials are used. + terraform-test: + name: terraform test (${{ matrix.directory }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + directory: + - aws-integration-setup/terraform/modules/eks-managed-scan-access + - aws-integration-setup/terraform/modules/k8s-resources + - aws-integration-setup/terraform/modules/nullify-aws-integration + - aws-integration-setup/terraform-v6/modules/eks-managed-scan-access + - aws-integration-setup/terraform-v6/modules/k8s-resources + - aws-integration-setup/terraform-v6/modules/nullify-aws-integration + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.10.5 + terraform_wrapper: false + + - name: terraform fmt -check (including test files) + working-directory: ${{ matrix.directory }} + run: terraform fmt -check -recursive + + - name: terraform init -backend=false + working-directory: ${{ matrix.directory }} + run: terraform init -backend=false + + - name: terraform test + working-directory: ${{ matrix.directory }} + run: terraform test + shellcheck: runs-on: ubuntu-latest steps: From dca000d85dcaa7dbc99e8ae671bb0ed13fb25880 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 06:10:50 +1000 Subject: [PATCH 16/36] fix(aws): convert publicAccessCidrs to a list before building the update command terraform test caught concat() rejecting the set returned by data.aws_eks_cluster, which would fail every plan where a Nullify egress IP is missing. The test mocks aws_iam_policy_document with a valid policy so provider validation accepts the mocked JSON. Co-Authored-By: Claude Opus 5 (1M context) --- .../terraform-v6/modules/eks-managed-scan-access/main.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 6 ++++++ .../terraform/modules/eks-managed-scan-access/main.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 6 ++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index 5a8cc5e..a0b5033 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -76,7 +76,7 @@ locals { endpoints = { for key, cluster in data.aws_eks_cluster.this : key => { public_access = cluster.vpc_config[0].endpoint_public_access - cidrs = cluster.vpc_config[0].public_access_cidrs + cidrs = tolist(cluster.vpc_config[0].public_access_cidrs) missing_cidrs = [for cidr in local.nullify_egress_cidrs : cidr if !contains(cluster.vpc_config[0].public_access_cidrs, cidr) && !contains(cluster.vpc_config[0].public_access_cidrs, "0.0.0.0/0")] } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index ad693b4..72e1eb1 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -1,4 +1,10 @@ mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + mock_data "aws_eks_cluster" { defaults = { identity = [{ diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 8cff96f..0889969 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -80,7 +80,7 @@ locals { endpoints = { for key, cluster in data.aws_eks_cluster.this : key => { public_access = cluster.vpc_config[0].endpoint_public_access - cidrs = cluster.vpc_config[0].public_access_cidrs + cidrs = tolist(cluster.vpc_config[0].public_access_cidrs) missing_cidrs = [for cidr in local.nullify_egress_cidrs : cidr if !contains(cluster.vpc_config[0].public_access_cidrs, cidr) && !contains(cluster.vpc_config[0].public_access_cidrs, "0.0.0.0/0")] } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 891ced7..fd71d29 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -1,4 +1,10 @@ mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + mock_data "aws_region" { defaults = { name = "eu-west-1" From 59b39457105beeff49ec44b6a9646d7145840644 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 09:59:09 +1000 Subject: [PATCH 17/36] ci: run the terraform test suites at the lowest version that can run them The modules declare required_version >= 1.5, but the validate rows only fmt and validate there, which never evaluate preconditions or check blocks, and the test job pinned 1.10.5 for every row. - terraform-test gains a version dimension: 1.8.5 and 1.10.5 - 1.8.5 is the floor for the suite itself: mock_provider is 1.7+, and the tests use strcontains, which is 1.8+ Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index d8576e5..9cae491 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -90,13 +90,19 @@ jobs: run: terraform validate # Module logic (preconditions, checks, RBAC rules, KMS grants) against mocked - # providers. mock_provider needs Terraform 1.7+; no cloud credentials are used. + # providers. No cloud credentials are used. The modules declare >= 1.5, but + # the validate job only parses them there: 1.8.5 is the lowest version that + # runs this suite (mock_provider is 1.7+, strcontains in the tests is 1.8+), + # so every module is exercised at that floor as well as at 1.10.5. terraform-test: - name: terraform test (${{ matrix.directory }}) + name: terraform test (${{ matrix.directory }}, tf ${{ matrix.terraform }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: + terraform: + - 1.8.5 + - 1.10.5 directory: - aws-integration-setup/terraform/modules/eks-managed-scan-access - aws-integration-setup/terraform/modules/k8s-resources @@ -109,7 +115,7 @@ jobs: - uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.10.5 + terraform_version: ${{ matrix.terraform }} terraform_wrapper: false - name: terraform fmt -check (including test files) From 4996322577069495fd7d5cbcf49fc68e94446f30 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 09:59:10 +1000 Subject: [PATCH 18/36] fix(terraform): name each collector's cluster and narrow the KMS grant Round-1 review of the managed EKS scan stack. - k8s-resources sets CLUSTER_NAME on the CronJob from a new cluster_name variable, so collectors stop sharing the k8s-collector/default-name-data.json upload key; a precondition requires it while enable_collector is true, and the multi-cluster examples pass each cluster's own name - eks-managed-scan-access clears kubernetes_groups with [] rather than null: the attribute is Optional+Computed, so null kept the stale group forever - nullify-aws-integration derives key/* only for an alias ARN; a key ARN is granted alone, as the CloudFormation template already does - collector_image pins k8s-collector-3.46.0, the build the Helm chart's k8s-collector-latest tag resolves to - README documents that the clusters must exist before the first plan, with the -target two-step, and the cluster_arns message says the same - docs describe the RBAC requirement instead of pointing at a chart, a manifest and a tag that exist on no merged branch - the RBAC test pins the exact kind set per API group, not just the counts Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform-v6/README.md | 6 +- .../examples/managed-scan/main.tf | 2 +- .../managed-scan/terraform.tfvars.example | 2 +- .../examples/multi-cluster-complete/main.tf | 2 + .../terraform.tfvars.example | 2 +- .../multi-cluster-complete/variables.tf | 2 +- .../modules/eks-managed-scan-access/main.tf | 2 +- .../tests/eks_managed_scan_access.tftest.hcl | 5 ++ .../eks-managed-scan-access/variables.tf | 6 +- .../modules/k8s-resources/main.tf | 12 ++++ .../modules/k8s-resources/managed_scan.tf | 4 +- .../tests/k8s_resources.tftest.hcl | 60 ++++++++++++++++++- .../modules/k8s-resources/variables.tf | 10 +++- .../modules/nullify-aws-integration/locals.tf | 17 +++--- .../tests/nullify_aws_integration.tftest.hcl | 6 +- .../nullify-aws-integration/variables.tf | 2 +- .../terraform-v6/terraform.tfvars.example | 5 +- .../terraform-v6/variables.tf | 2 +- aws-integration-setup/terraform/README.md | 18 +++--- .../terraform/examples/managed-scan/main.tf | 2 +- .../managed-scan/terraform.tfvars.example | 2 +- .../examples/multi-cluster-complete/main.tf | 2 + .../terraform.tfvars.example | 2 +- .../multi-cluster-complete/variables.tf | 2 +- .../modules/eks-managed-scan-access/main.tf | 2 +- .../tests/eks_managed_scan_access.tftest.hcl | 5 ++ .../eks-managed-scan-access/variables.tf | 6 +- .../terraform/modules/k8s-resources/main.tf | 12 ++++ .../modules/k8s-resources/managed_scan.tf | 4 +- .../tests/k8s_resources.tftest.hcl | 60 ++++++++++++++++++- .../modules/k8s-resources/variables.tf | 10 +++- .../modules/nullify-aws-integration/locals.tf | 17 +++--- .../tests/nullify_aws_integration.tftest.hcl | 6 +- .../nullify-aws-integration/variables.tf | 2 +- .../terraform/terraform.tfvars.example | 5 +- aws-integration-setup/terraform/variables.tf | 2 +- 36 files changed, 237 insertions(+), 69 deletions(-) diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index c46424c..3a1dbfd 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -92,7 +92,7 @@ module "nullify_eks_access" { } ``` -The default `authorization = "rbac"` needs the list-only `nullify-readonly` ClusterRole on each cluster (`k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true`, or the `nullify-k8s-readonly-access` Helm chart). `admin_view_policy` is a warned opt-in; see `../terraform/README.md`. +The default `authorization = "rbac"` needs the list-only `nullify-readonly` ClusterRole on each cluster: apply it with `k8s-resources` (`enable_collector = false, enable_managed_scan_rbac = true`), or with any equivalent manifest of your own. `admin_view_policy` is a warned opt-in. The clusters must already exist when you plan; see `../terraform/README.md`. ## Required Variables @@ -114,7 +114,9 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `tags`: Resource tags - `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` -`k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0`. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. +`k8s-resources` requires `cluster_name` whenever `enable_collector` is true, and it must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, so two collectors sharing a name overwrite each other's inventory. + +`k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. ## Validation diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf index e45142a..86b8041 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf @@ -61,7 +61,7 @@ module "eks_managed_scan_access" { # Creating the ClusterRole needs rights to grant every permission in it, which # in practice means cluster-admin. Set manage_rbac = false when Helm, Flux or -# Argo CD applies the nullify-k8s-readonly-access chart instead. +# Argo CD applies the nullify-readonly ClusterRole instead. module "k8s_resources" { source = "../../modules/k8s-resources" count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example index e17c886..4fe07b7 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example @@ -21,5 +21,5 @@ nullify_region = "eu-central-1" # every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. # authorization = "rbac" -# Set false if Helm, Flux or Argo CD applies the nullify-k8s-readonly-access chart. +# Set false if Helm, Flux or Argo CD applies the nullify-readonly ClusterRole instead. # manage_rbac = true diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index bf93a39..e2e563f 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -86,6 +86,7 @@ module "k8s_resources_primary" { kubernetes = kubernetes.primary } iam_role_arn = module.nullify_aws_integration.role_arn + cluster_name = local.primary_name s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region @@ -103,6 +104,7 @@ module "k8s_resources_secondary" { kubernetes = kubernetes.secondary } iam_role_arn = module.nullify_aws_integration.role_arn + cluster_name = local.secondary_name s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example index ebca4aa..29fcac1 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example @@ -42,7 +42,7 @@ kubernetes_namespace = "nullify" cronjob_schedule = "0 0 * * *" # Collector image (defaults to a pinned tag of Nullify's ECR Public image) -# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" +# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf index 7131a36..af2657b 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf @@ -80,7 +80,7 @@ variable "cronjob_schedule" { variable "collector_image" { type = string description = "Container image for the Kubernetes collector" - default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" } variable "tags" { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index a0b5033..26e07dd 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -42,7 +42,7 @@ resource "aws_eks_access_entry" "nullify" { cluster_name = data.aws_eks_cluster.this[each.key].name principal_arn = var.principal_arn type = "STANDARD" - kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : null + kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : [] tags = var.tags lifecycle { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 24d3da7..0e2d29e 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -172,6 +172,11 @@ run "admin_view_policy_is_cluster_scoped_and_warns" { condition = output.kubernetes_group_name == null error_message = "admin_view_policy binds no Kubernetes group" } + + assert { + condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 + error_message = "admin_view_policy must clear kubernetes_groups on the entry itself: the attribute is Optional+Computed, so null keeps whatever is in state" + } } run "view_policy_is_not_offered" { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf index 2119d71..7ebbb5d 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -16,17 +16,17 @@ variable "principal_unique_id" { variable "cluster_arns" { type = list(string) - description = "ARNs of the EKS clusters Nullify scans. Each cluster is read and granted access in the region from its ARN" + description = "ARNs of the EKS clusters Nullify scans. Each cluster is read and granted access in the region from its ARN. The clusters must already exist and their ARNs must be known at plan time: they drive for_each" validation { condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." } } variable "authorization" { type = string - description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to the list-only nullify-readonly ClusterRole with k8s-resources (enable_managed_scan_rbac), the nullify-k8s-readonly-access Helm chart or manifests/nullify-readonly-rbac.yaml. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" default = "rbac" validation { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 6de986a..7f34e68 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -159,6 +159,13 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + lifecycle { + precondition { + condition = var.cluster_name != "" + error_message = "cluster_name is required when enable_collector is true: the collector uploads to /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + } + } + spec { schedule = var.cronjob_schedule concurrency_policy = "Forbid" @@ -192,6 +199,11 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { name = "k8s-collector" image = var.collector_image + env { + name = "CLUSTER_NAME" + value = var.cluster_name + } + env { name = "NULLIFY_S3_BUCKET_NAME" value = var.s3_bucket_name diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf index 4f6505d..a8d394b 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf @@ -1,6 +1,6 @@ locals { - # The kinds Nullify's managed EKS scan lists. Keep in step with - # manifests/nullify-readonly-rbac.yaml and the nullify-k8s-readonly-access chart. + # The kinds Nullify's managed EKS scan lists. Any other manifest or chart that + # grants this role must list exactly these kinds; the test suite pins the set. managed_scan_rules = [ { api_groups = [""] diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 2b0b487..186e2f9 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -5,16 +5,22 @@ run "collector_only_is_the_default" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" } + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "CLUSTER_NAME" && env.value == "acme-prod"]) == 1 + error_message = "The CronJob must set CLUSTER_NAME: it names the upload, and collectors without one all write k8s-collector/default-name-data.json" + } + assert { condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 error_message = "The collector must be deployed by default" } assert { - condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" error_message = "The collector image must default to the pinned Nullify ECR Public tag" } @@ -27,9 +33,24 @@ run "collector_only_is_the_default" { run "collector_requires_a_role_arn" { command = plan + variables { + cluster_name = "acme-prod" + } + expect_failures = [kubernetes_service_account.nullify_collector_sa] } +run "collector_requires_a_cluster_name" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + s3_bucket_name = "nullify-collector-uploads" + } + + expect_failures = [kubernetes_cron_job_v1.k8s_collector] +} + run "managed_scan_rbac_only" { command = plan @@ -58,6 +79,39 @@ run "managed_scan_rbac_only" { error_message = "The managed-scan ClusterRole must list exactly the 26 kinds the scanner reads" } + # Pins the set, not just the size: swapping one kind for another keeps every + # count assertion green. Anything else that binds this group -- a chart, a + # manifest -- has to grant exactly these kinds. + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"]) + error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[1].api_groups) == tolist(["apps"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[1].resources) == toset(["deployments", "daemonsets", "statefulsets", "replicasets"]) + error_message = "Rule 1 must grant exactly the apps API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[2].api_groups) == tolist(["networking.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[2].resources) == toset(["ingresses", "networkpolicies"]) + error_message = "Rule 2 must grant exactly the networking.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[3].api_groups) == tolist(["discovery.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[3].resources) == toset(["endpointslices"]) + error_message = "Rule 3 must grant exactly the discovery.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[4].api_groups) == tolist(["rbac.authorization.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[4].resources) == toset(["roles", "rolebindings", "clusterroles", "clusterrolebindings"]) + error_message = "Rule 4 must grant exactly the rbac.authorization.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[5].api_groups) == tolist(["admissionregistration.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[5].resources) == toset(["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"]) + error_message = "Rule 5 must grant exactly the admissionregistration.k8s.io API group kinds the scanner lists" + } + assert { condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : tolist(rule.verbs) == tolist(["list"])]) error_message = "Resource rules must grant list only" @@ -80,7 +134,7 @@ run "managed_scan_rbac_only" { assert { condition = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name == "nullify-readonly" - error_message = "The ClusterRole name must match the Helm chart and manifest" + error_message = "The ClusterRole name must stay nullify-readonly: it is the name every other install path for this role uses" } } @@ -89,6 +143,7 @@ run "kms_alias_arns_are_accepted" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } } @@ -98,6 +153,7 @@ run "invalid_kms_arns_are_rejected" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" kms_key_arn = "arn:aws:s3:::not-a-key" } diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf index 40f304e..2660b57 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf @@ -23,7 +23,7 @@ variable "managed_scan_kubernetes_group" { variable "managed_scan_cluster_role_name" { type = string - description = "Name of the managed-scan ClusterRole. The default matches the nullify-k8s-readonly-access Helm chart and manifests/nullify-readonly-rbac.yaml" + description = "Name of the managed-scan ClusterRole. Keep the default unless you also change kubernetes_group_name on the access entry: Nullify only needs the binding to reach a role with these list permissions" default = "nullify-readonly" } @@ -39,6 +39,12 @@ variable "iam_role_arn" { default = "" } +variable "cluster_name" { + type = string + description = "Name of the cluster this collector runs in, required when enable_collector is true. It becomes the collector's CLUSTER_NAME, which names its upload: /k8s-collector/-data.json. Two clusters sharing a name overwrite each other's inventory, so give every cluster its own. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" + default = "" +} + variable "service_account_name" { type = string description = "The name of the Kubernetes service account" @@ -66,7 +72,7 @@ variable "kubernetes_namespace" { variable "collector_image" { type = string description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image" - default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" } variable "cronjob_schedule" { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf index 76f998f..27474ea 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf @@ -20,15 +20,14 @@ locals { enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. IAM ignores alias ARNs in Resource, so key/* in the same - # account and region is granted too; Nullify's key policy is the real gate. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - - kms_policy_resources = local.enable_kms_access ? [ - var.kms_key_arn, - "arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*", - ] : [] + # KMS configuration. IAM ignores an alias ARN in Resource, so an alias also + # grants key/* in its own account and region; Nullify's key policy is the + # real gate. A key ARN resolves in IAM on its own and is granted alone. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_alias = length(local.kms_arn_parts) == 6 && startswith(element(local.kms_arn_parts, 5), "alias/") + kms_alias_key_wildcard = local.kms_arn_is_alias ? ["arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_alias_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 72e1eb1..29bf443 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -35,7 +35,7 @@ run "kms_alias_grants_the_alias_and_key_wildcard" { } } -run "kms_multi_region_key_is_accepted" { +run "kms_key_arn_grants_only_that_key" { command = plan variables { @@ -43,8 +43,8 @@ run "kms_multi_region_key_is_accepted" { } assert { - condition = contains(output.kms_policy_resources, "arn:aws:kms:us-east-2:111122223333:key/*") - error_message = "A key ARN must also grant key/* in the same account and region" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) + error_message = "A key ARN resolves in IAM on its own: only an alias ARN needs key/*" } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf index 049f308..f60158c 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf @@ -97,7 +97,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. IAM ignores alias ARNs, so the KMS policy grants this ARN and key/* in the same Nullify account and region. Nullify's key policy decides which of those keys the role can use" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is granted on its own. An alias ARN also grants key/* in the same Nullify account and region, because IAM does not resolve an alias in a policy Resource. Nullify's key policy decides which of those keys the role can use" default = "" validation { diff --git a/aws-integration-setup/terraform-v6/terraform.tfvars.example b/aws-integration-setup/terraform-v6/terraform.tfvars.example index 964e8ce..0012b28 100644 --- a/aws-integration-setup/terraform-v6/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/terraform.tfvars.example @@ -32,8 +32,9 @@ enable_kubernetes_integration = false # ] # Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. -# Clusters can be in any region. In rbac mode, bind the group with the nullify-k8s-readonly-access -# Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. +# Clusters can be in any region but must already exist when you plan. In rbac mode, bind the group +# to a list-only nullify-readonly ClusterRole: the k8s-resources module applies one with +# enable_managed_scan_rbac = true. # enable_managed_scan = true # managed_scan_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/my-cluster"] # nullify_region = "eu-central-1" diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index 462ac9a..f9999fa 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -110,7 +110,7 @@ variable "managed_scan_cluster_arns" { variable "managed_scan_authorization" { type = string - description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to the list-only nullify-readonly ClusterRole with the nullify-k8s-readonly-access Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" default = "rbac" } diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 0171cd0..ee5350c 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -85,7 +85,7 @@ terraform/ | Runs in the cluster | CronJob with an IRSA service account | Nothing | | Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | | AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | -| Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true`, the `nullify-k8s-readonly-access` Helm chart, or `manifests/nullify-readonly-rbac.yaml` | +| Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | A list-only `nullify-readonly` ClusterRole and ClusterRoleBinding, applied by `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true` | | Upgrades | You update the image | None | Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. @@ -115,10 +115,11 @@ Associates `AmazonEKSAdminViewPolicy` at cluster scope, so no Kubernetes objects ### Prerequisites +- **The clusters must already exist when you plan.** `cluster_arns` drives `for_each` and each cluster is read with `data "aws_eks_cluster"`, so every ARN must be a literal or a value known at plan time. Calling this module in the same apply that creates the cluster fails with *"Invalid for_each argument ... depends on resource attributes that cannot be determined until apply"*. Create the cluster first (`terraform apply -target=module.eks`), then run a normal `terraform apply`; from then on the ARNs are in state and every later apply is a single step. The module is deliberately not restructured to defer the lookup: the preconditions below, and the endpoint `check`, only work against a cluster that can be read at plan. - **Authentication mode.** Access entries need `API` or `API_AND_CONFIG_MAP`. Check with `aws eks describe-cluster --name --query cluster.accessConfig.authenticationMode`. Switch (one-way) with `aws eks update-cluster-config --name --access-config authenticationMode=API_AND_CONFIG_MAP`. The module fails at plan for `CONFIG_MAP` clusters. - **Same account.** Nullify assumes the role in the cluster's own account, so deploy `nullify-aws-integration` in every account that owns a scanned cluster. The module rejects clusters in another account. - **One region per module instance (AWS provider v5).** Instantiate `eks-managed-scan-access` once per region with `providers = { aws = aws. }`, as `examples/multi-cluster-complete` does. `../terraform-v6/` handles every region in one instance. -- **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`, plus `eks:AssociateAccessPolicy`, `eks:DisassociateAccessPolicy` and `eks:ListAssociatedAccessPolicies` for `admin_view_policy`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the Helm chart instead. +- **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`, plus `eks:AssociateAccessPolicy`, `eks:DisassociateAccessPolicy` and `eks:ListAssociatedAccessPolicies` for `admin_view_policy`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the same ClusterRole itself. - **Pass `principal_unique_id`.** EKS ties an access entry to the role's ID, so a recreated role with the same ARN is silently not authorized. With `principal_unique_id = module.nullify_aws_integration.role_unique_id`, the entries are replaced with the role. - **Existing access entry.** If this role is already mapped on a cluster, import it: `terraform import 'module.eks_managed_scan_access.aws_eks_access_entry.nullify[""]' :`. @@ -151,7 +152,7 @@ Private-only clusters cannot use the managed scan; use the in-cluster collector. ### CONFIG_MAP clusters (aws-auth) -These modules do not manage `aws-auth`: writing it from Terraform replaces the whole `mapRoles` string and can drop node roles. If you cannot switch the authentication mode, add the role yourself and apply the RBAC with `k8s-resources` or the Helm chart: +These modules do not manage `aws-auth`: writing it from Terraform replaces the whole `mapRoles` string and can drop node roles. If you cannot switch the authentication mode, add the role yourself and apply the RBAC with `k8s-resources`: ```yaml mapRoles: | @@ -274,7 +275,7 @@ terraform init && terraform apply - a key ARN: `arn:aws:kms:::key/`, including multi-Region `key/mrk-...` keys - an alias ARN: `arn:aws:kms:::alias/` -IAM ignores alias ARNs in a policy's `Resource`, so the KMS policy grants the ARN you pass and `arn:aws:kms:::key/*` in the same Nullify account and region. Nullify's key policy is the real gate: the role can only use Nullify keys whose key policy allows it. The `kms_policy_resources` module output lists both resources. +A key ARN is granted on its own. IAM does not resolve an alias in a policy's `Resource`, so an alias ARN also grants `arn:aws:kms:::key/*` in the same Nullify account and region. Nullify's key policy is the real gate: the role can only use Nullify keys whose key policy allows it. The `kms_policy_resources` module output lists what was granted. ## Module Usage @@ -309,6 +310,7 @@ module "k8s_resources" { } iam_role_arn = module.nullify_aws_integration.role_arn + cluster_name = "cluster-a" s3_bucket_name = "my-scan-results-bucket" # or the S3 access point ARN from Nullify kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" aws_region = "us-west-2" @@ -316,9 +318,11 @@ module "k8s_resources" { } ``` +`cluster_name` is required whenever `enable_collector` is true, and must differ per cluster. The collector uploads to `/k8s-collector/-data.json`, so two collectors sharing a name overwrite each other's inventory and Nullify only ever sees the one that ran last. + `k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. -`collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0`, a pinned tag of Nullify's ECR Public image. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. +`collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. It is the same build as `k8s-collector-latest`, which the `nullify-k8s-collector` Helm chart deploys, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. ## Outputs @@ -361,7 +365,7 @@ module "nullify_k8s" { - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. -- The KMS policy now also grants `key/*` in the account and region of `kms_key_arn`. +- The KMS policy grants `key/*` in the account and region of `kms_key_arn` when that value is an alias ARN. A key ARN grants only itself, as the CloudFormation template does. ## Security Considerations @@ -378,7 +382,7 @@ module "nullify_k8s" { terraform fmt -recursive terraform validate -# Module tests (mocked providers, Terraform >= 1.7) +# Module tests (mocked providers, Terraform >= 1.8; CI runs them at 1.8.5 and 1.10.5) (cd modules/eks-managed-scan-access && terraform init -backend=false && terraform test) # Plan deployment diff --git a/aws-integration-setup/terraform/examples/managed-scan/main.tf b/aws-integration-setup/terraform/examples/managed-scan/main.tf index b1870b8..074fee9 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/main.tf +++ b/aws-integration-setup/terraform/examples/managed-scan/main.tf @@ -61,7 +61,7 @@ module "eks_managed_scan_access" { # Creating the ClusterRole needs rights to grant every permission in it, which # in practice means cluster-admin. Set manage_rbac = false when Helm, Flux or -# Argo CD applies the nullify-k8s-readonly-access chart instead. +# Argo CD applies the nullify-readonly ClusterRole instead. module "k8s_resources" { source = "../../modules/k8s-resources" count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 diff --git a/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example index e17c886..4fe07b7 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example @@ -21,5 +21,5 @@ nullify_region = "eu-central-1" # every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. # authorization = "rbac" -# Set false if Helm, Flux or Argo CD applies the nullify-k8s-readonly-access chart. +# Set false if Helm, Flux or Argo CD applies the nullify-readonly ClusterRole instead. # manage_rbac = true diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index 7284739..5f1f18c 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -102,6 +102,7 @@ module "k8s_resources_primary" { kubernetes = kubernetes.primary } iam_role_arn = module.nullify_aws_integration.role_arn + cluster_name = local.primary_name s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region @@ -119,6 +120,7 @@ module "k8s_resources_secondary" { kubernetes = kubernetes.secondary } iam_role_arn = module.nullify_aws_integration.role_arn + cluster_name = local.secondary_name s3_bucket_name = local.collector_upload_target kms_key_arn = var.kms_key_arn aws_region = var.aws_region diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example index ebca4aa..29fcac1 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example @@ -42,7 +42,7 @@ kubernetes_namespace = "nullify" cronjob_schedule = "0 0 * * *" # Collector image (defaults to a pinned tag of Nullify's ECR Public image) -# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" +# collector_image = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf index 7131a36..af2657b 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf @@ -80,7 +80,7 @@ variable "cronjob_schedule" { variable "collector_image" { type = string description = "Container image for the Kubernetes collector" - default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" } variable "tags" { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 0889969..3af95a0 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -47,7 +47,7 @@ resource "aws_eks_access_entry" "nullify" { cluster_name = data.aws_eks_cluster.this[each.key].name principal_arn = var.principal_arn type = "STANDARD" - kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : null + kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : [] tags = var.tags lifecycle { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 98c6f0b..3b88d30 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -201,6 +201,11 @@ run "admin_view_policy_is_cluster_scoped_and_warns" { condition = output.kubernetes_group_name == null error_message = "admin_view_policy binds no Kubernetes group" } + + assert { + condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 + error_message = "admin_view_policy must clear kubernetes_groups on the entry itself: the attribute is Optional+Computed, so null keeps whatever is in state" + } } run "view_policy_is_not_offered" { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf index 05a4e4c..d6edb9a 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -16,17 +16,17 @@ variable "principal_unique_id" { variable "cluster_arns" { type = list(string) - description = "ARNs of the EKS clusters Nullify scans, all in this module's AWS provider region. AWS provider v5 has no per-resource region, so instantiate the module once per region with providers = { aws = aws. }" + description = "ARNs of the EKS clusters Nullify scans, all in this module's AWS provider region. AWS provider v5 has no per-resource region, so instantiate the module once per region with providers = { aws = aws. }. The clusters must already exist and their ARNs must be known at plan time: they drive for_each" validation { condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." } } variable "authorization" { type = string - description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to the list-only nullify-readonly ClusterRole with k8s-resources (enable_managed_scan_rbac), the nullify-k8s-readonly-access Helm chart or manifests/nullify-readonly-rbac.yaml. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" default = "rbac" validation { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 6de986a..7f34e68 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -159,6 +159,13 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + lifecycle { + precondition { + condition = var.cluster_name != "" + error_message = "cluster_name is required when enable_collector is true: the collector uploads to /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + } + } + spec { schedule = var.cronjob_schedule concurrency_policy = "Forbid" @@ -192,6 +199,11 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { name = "k8s-collector" image = var.collector_image + env { + name = "CLUSTER_NAME" + value = var.cluster_name + } + env { name = "NULLIFY_S3_BUCKET_NAME" value = var.s3_bucket_name diff --git a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf index 4f6505d..a8d394b 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf @@ -1,6 +1,6 @@ locals { - # The kinds Nullify's managed EKS scan lists. Keep in step with - # manifests/nullify-readonly-rbac.yaml and the nullify-k8s-readonly-access chart. + # The kinds Nullify's managed EKS scan lists. Any other manifest or chart that + # grants this role must list exactly these kinds; the test suite pins the set. managed_scan_rules = [ { api_groups = [""] diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 2b0b487..186e2f9 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -5,16 +5,22 @@ run "collector_only_is_the_default" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" } + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "CLUSTER_NAME" && env.value == "acme-prod"]) == 1 + error_message = "The CronJob must set CLUSTER_NAME: it names the upload, and collectors without one all write k8s-collector/default-name-data.json" + } + assert { condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 error_message = "The collector must be deployed by default" } assert { - condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" error_message = "The collector image must default to the pinned Nullify ECR Public tag" } @@ -27,9 +33,24 @@ run "collector_only_is_the_default" { run "collector_requires_a_role_arn" { command = plan + variables { + cluster_name = "acme-prod" + } + expect_failures = [kubernetes_service_account.nullify_collector_sa] } +run "collector_requires_a_cluster_name" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + s3_bucket_name = "nullify-collector-uploads" + } + + expect_failures = [kubernetes_cron_job_v1.k8s_collector] +} + run "managed_scan_rbac_only" { command = plan @@ -58,6 +79,39 @@ run "managed_scan_rbac_only" { error_message = "The managed-scan ClusterRole must list exactly the 26 kinds the scanner reads" } + # Pins the set, not just the size: swapping one kind for another keeps every + # count assertion green. Anything else that binds this group -- a chart, a + # manifest -- has to grant exactly these kinds. + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"]) + error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[1].api_groups) == tolist(["apps"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[1].resources) == toset(["deployments", "daemonsets", "statefulsets", "replicasets"]) + error_message = "Rule 1 must grant exactly the apps API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[2].api_groups) == tolist(["networking.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[2].resources) == toset(["ingresses", "networkpolicies"]) + error_message = "Rule 2 must grant exactly the networking.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[3].api_groups) == tolist(["discovery.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[3].resources) == toset(["endpointslices"]) + error_message = "Rule 3 must grant exactly the discovery.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[4].api_groups) == tolist(["rbac.authorization.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[4].resources) == toset(["roles", "rolebindings", "clusterroles", "clusterrolebindings"]) + error_message = "Rule 4 must grant exactly the rbac.authorization.k8s.io API group kinds the scanner lists" + } + + assert { + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[5].api_groups) == tolist(["admissionregistration.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[5].resources) == toset(["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"]) + error_message = "Rule 5 must grant exactly the admissionregistration.k8s.io API group kinds the scanner lists" + } + assert { condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 6) : tolist(rule.verbs) == tolist(["list"])]) error_message = "Resource rules must grant list only" @@ -80,7 +134,7 @@ run "managed_scan_rbac_only" { assert { condition = kubernetes_cluster_role_v1.nullify_readonly[0].metadata[0].name == "nullify-readonly" - error_message = "The ClusterRole name must match the Helm chart and manifest" + error_message = "The ClusterRole name must stay nullify-readonly: it is the name every other install path for this role uses" } } @@ -89,6 +143,7 @@ run "kms_alias_arns_are_accepted" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } } @@ -98,6 +153,7 @@ run "invalid_kms_arns_are_rejected" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" kms_key_arn = "arn:aws:s3:::not-a-key" } diff --git a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf index 40f304e..2660b57 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf @@ -23,7 +23,7 @@ variable "managed_scan_kubernetes_group" { variable "managed_scan_cluster_role_name" { type = string - description = "Name of the managed-scan ClusterRole. The default matches the nullify-k8s-readonly-access Helm chart and manifests/nullify-readonly-rbac.yaml" + description = "Name of the managed-scan ClusterRole. Keep the default unless you also change kubernetes_group_name on the access entry: Nullify only needs the binding to reach a role with these list permissions" default = "nullify-readonly" } @@ -39,6 +39,12 @@ variable "iam_role_arn" { default = "" } +variable "cluster_name" { + type = string + description = "Name of the cluster this collector runs in, required when enable_collector is true. It becomes the collector's CLUSTER_NAME, which names its upload: /k8s-collector/-data.json. Two clusters sharing a name overwrite each other's inventory, so give every cluster its own. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" + default = "" +} + variable "service_account_name" { type = string description = "The name of the Kubernetes service account" @@ -66,7 +72,7 @@ variable "kubernetes_namespace" { variable "collector_image" { type = string description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image" - default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.45.0" + default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0" } variable "cronjob_schedule" { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf index 76f998f..27474ea 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf @@ -20,15 +20,14 @@ locals { enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. IAM ignores alias ARNs in Resource, so key/* in the same - # account and region is granted too; Nullify's key policy is the real gate. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - - kms_policy_resources = local.enable_kms_access ? [ - var.kms_key_arn, - "arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*", - ] : [] + # KMS configuration. IAM ignores an alias ARN in Resource, so an alias also + # grants key/* in its own account and region; Nullify's key policy is the + # real gate. A key ARN resolves in IAM on its own and is granted alone. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_alias = length(local.kms_arn_parts) == 6 && startswith(element(local.kms_arn_parts, 5), "alias/") + kms_alias_key_wildcard = local.kms_arn_is_alias ? ["arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_alias_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index fd71d29..56e4ff7 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -41,7 +41,7 @@ run "kms_alias_grants_the_alias_and_key_wildcard" { } } -run "kms_multi_region_key_is_accepted" { +run "kms_key_arn_grants_only_that_key" { command = plan variables { @@ -49,8 +49,8 @@ run "kms_multi_region_key_is_accepted" { } assert { - condition = contains(output.kms_policy_resources, "arn:aws:kms:us-east-2:111122223333:key/*") - error_message = "A key ARN must also grant key/* in the same account and region" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) + error_message = "A key ARN resolves in IAM on its own: only an alias ARN needs key/*" } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf index 97f9422..c78eabd 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf @@ -108,7 +108,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. IAM ignores alias ARNs, so the KMS policy grants this ARN and key/* in the same Nullify account and region. Nullify's key policy decides which of those keys the role can use" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is granted on its own. An alias ARN also grants key/* in the same Nullify account and region, because IAM does not resolve an alias in a policy Resource. Nullify's key policy decides which of those keys the role can use" default = "" validation { diff --git a/aws-integration-setup/terraform/terraform.tfvars.example b/aws-integration-setup/terraform/terraform.tfvars.example index 3f33b53..ad64012 100644 --- a/aws-integration-setup/terraform/terraform.tfvars.example +++ b/aws-integration-setup/terraform/terraform.tfvars.example @@ -37,8 +37,9 @@ enable_kubernetes_integration = false # ] # Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. -# Clusters must be in aws_region. In rbac mode, bind the group with the nullify-k8s-readonly-access -# Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. +# Clusters must be in aws_region and must already exist when you plan. In rbac mode, bind the group +# to a list-only nullify-readonly ClusterRole: the k8s-resources module applies one with +# enable_managed_scan_rbac = true. # enable_managed_scan = true # managed_scan_cluster_arns = ["arn:aws:eks:ap-southeast-2:123456789012:cluster/my-cluster"] # nullify_region = "ap-southeast-2" diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index afcf7cc..630bff9 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -116,7 +116,7 @@ variable "managed_scan_cluster_arns" { variable "managed_scan_authorization" { type = string - description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to the list-only nullify-readonly ClusterRole with the nullify-k8s-readonly-access Helm chart, manifests/nullify-readonly-rbac.yaml or the k8s-resources module. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" default = "rbac" } From e3190b90d2f39a6840b2bc0bfacb9af2b2a00fae Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 10:01:09 +1000 Subject: [PATCH 19/36] docs(terraform): note the cluster_name requirement in the upgrade notes Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index ee5350c..0f6df5f 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -362,6 +362,7 @@ module "nullify_k8s" { - Terraform >= 1.5 is now required (the module already used cross-variable validation, which needs 1.9; those checks are now preconditions). - The root and examples require AWS provider `>= 5.33`. Run `terraform init -upgrade` if your lock file pins an older 5.x. +- `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`, and the old object is left behind for you to delete. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. From c251dba74133f25d06452b7e3e539c90f7bbd379 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 10:28:31 +1000 Subject: [PATCH 20/36] fix(terraform): require KMS for the collector and narrow the key/* grant to cross-account Both published collector images refuse to upload without KMS encryption, so a Terraform deployment that left kms_key_arn empty collected the cluster and then failed every upload. cluster_name had two undocumented requirements that fail silently server-side. scan_mode = "both" registered one cluster twice. - k8s-resources: kms_key_arn required while enable_collector is true, with allow_unencrypted_upload as the only opt-out and the only setter of ALLOW_UNENCRYPTED_UPLOAD - cluster_name docs and precondition state the exact-name and registered-cluster requirements; README example uses a real-looking name - nullify-aws-integration: derive key/* for alias and key ARNs alike, only when the ARN's account is not the caller's - examples/multi-cluster-complete: reject scan_mode = "both" - use data.aws_partition for constructed ARNs, and the cluster ARN's own partition for the EKS access policy ARN - declare Terraform >= 1.8 and validate at 1.8.5, the version the test suites run - correct the upgrade note that told customers to delete an object their role is denied s3:DeleteObject* on Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 29 ++++++------ aws-integration-setup/terraform-v6/README.md | 6 +-- .../terraform-v6/examples/basic/main.tf | 2 +- .../examples/multi-cluster-complete/main.tf | 4 +- .../multi-cluster-complete/outputs.tf | 2 +- .../terraform.tfvars.example | 10 +++-- .../multi-cluster-complete/variables.tf | 10 ++--- .../modules/eks-managed-scan-access/main.tf | 9 ++-- .../tests/eks_managed_scan_access.tftest.hcl | 2 +- .../eks-managed-scan-access/versions.tf | 2 +- .../modules/k8s-resources/main.tf | 15 ++++++- .../modules/k8s-resources/providers.tf | 2 +- .../tests/k8s_resources.tftest.hcl | 45 +++++++++++++++++++ .../modules/k8s-resources/variables.tf | 10 ++++- .../modules/nullify-aws-integration/data.tf | 6 ++- .../modules/nullify-aws-integration/locals.tf | 24 ++++++---- .../nullify-aws-integration/outputs.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 35 ++++++++++++--- .../nullify-aws-integration/versions.tf | 2 +- .../terraform-v6/versions.tf | 2 +- aws-integration-setup/terraform/README.md | 28 ++++++++---- .../terraform/examples/basic/main.tf | 2 +- .../terraform/examples/managed-scan/main.tf | 2 +- .../examples/multi-cluster-complete/main.tf | 6 +-- .../multi-cluster-complete/outputs.tf | 2 +- .../terraform.tfvars.example | 10 +++-- .../multi-cluster-complete/variables.tf | 10 ++--- .../modules/eks-managed-scan-access/main.tf | 9 ++-- .../tests/eks_managed_scan_access.tftest.hcl | 2 +- .../eks-managed-scan-access/versions.tf | 2 +- .../terraform/modules/k8s-resources/main.tf | 15 ++++++- .../modules/k8s-resources/providers.tf | 2 +- .../tests/k8s_resources.tftest.hcl | 45 +++++++++++++++++++ .../modules/k8s-resources/variables.tf | 10 ++++- .../modules/nullify-aws-integration/data.tf | 6 ++- .../modules/nullify-aws-integration/locals.tf | 24 ++++++---- .../nullify-aws-integration/outputs.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 35 ++++++++++++--- .../nullify-aws-integration/versions.tf | 2 +- aws-integration-setup/terraform/versions.tf | 2 +- 40 files changed, 323 insertions(+), 112 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index 9cae491..001afd0 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -32,8 +32,9 @@ jobs: strategy: fail-fast: false matrix: - # AWS rows run the declared floor (Terraform 1.5); examples that use - # ephemeral resources (v6 multi-cluster, v6 managed-scan) need 1.10. + # AWS rows run the declared floor (Terraform 1.8, the version the + # terraform-test job also runs); examples that use ephemeral resources + # (v6 multi-cluster, v6 managed-scan) need 1.10. include: - directory: gcp-integration-setup/terraform terraform: 1.9.5 @@ -44,25 +45,25 @@ jobs: - directory: gcp-integration-setup/terraform/examples/single-project terraform: 1.9.5 - directory: aws-integration-setup/terraform - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform/examples/basic - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform/examples/multi-cluster-complete - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform/examples/managed-scan - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform/modules/eks-managed-scan-access - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6 - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6/examples/basic - terraform: 1.5.7 + terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6/examples/multi-cluster-complete terraform: 1.10.5 - directory: aws-integration-setup/terraform-v6/examples/managed-scan terraform: 1.10.5 - directory: aws-integration-setup/terraform-v6/modules/eks-managed-scan-access - terraform: 1.5.7 + terraform: 1.8.5 - directory: azure-integration-setup/terraform terraform: 1.9.5 - directory: azure-integration-setup/terraform/examples/management-group @@ -90,10 +91,10 @@ jobs: run: terraform validate # Module logic (preconditions, checks, RBAC rules, KMS grants) against mocked - # providers. No cloud credentials are used. The modules declare >= 1.5, but - # the validate job only parses them there: 1.8.5 is the lowest version that - # runs this suite (mock_provider is 1.7+, strcontains in the tests is 1.8+), - # so every module is exercised at that floor as well as at 1.10.5. + # providers. No cloud credentials are used. 1.8.5 is the declared floor and + # the lowest version that runs this suite (mock_provider is 1.7+, strcontains + # in the tests is 1.8+), so every module is exercised at the floor it + # advertises as well as at 1.10.5. terraform-test: name: terraform test (${{ matrix.directory }}, tf ${{ matrix.terraform }}) runs-on: ubuntu-latest diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index 3a1dbfd..cd4d16c 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -21,7 +21,7 @@ token = ephemeral.aws_eks_cluster_auth.primary.token ## Requirements -- Terraform >= 1.5 (>= 1.10 for the `multi-cluster-complete` and `managed-scan` examples) +- Terraform >= 1.8 (>= 1.10 for the `multi-cluster-complete` and `managed-scan` examples) - AWS provider ~> 6.0 - Kubernetes provider ~> 2.20 (only `modules/k8s-resources` and the `multi-cluster-complete` and `managed-scan` examples) @@ -105,7 +105,7 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `aws_region`: AWS region for IAM resources (default: ap-southeast-2) - `s3_bucket_name`: S3 bucket for scan results (optional) - `nullify_s3_access_point_arn`: S3 access point for collector uploads (optional) -- `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page (optional); see "KMS" in `../terraform/README.md` +- `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page; see "KMS" in `../terraform/README.md`. Optional for this module; required by `k8s-resources` whenever `enable_collector` is true - `enable_kubernetes_integration`: Set to `true` for EKS integration - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with, in any region - `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: managed EKS scan @@ -114,7 +114,7 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `tags`: Resource tags - `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` -`k8s-resources` requires `cluster_name` whenever `enable_collector` is true, and it must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, so two collectors sharing a name overwrite each other's inventory. +`k8s-resources` requires `cluster_name` and `kms_key_arn` whenever `enable_collector` is true. `cluster_name` must match your actual cluster name exactly, must match a cluster registered on the Nullify configure page, and must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, and Nullify joins that upload on the name. See "Kubernetes Resources" and "KMS" in `../terraform/README.md`. `k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. diff --git a/aws-integration-setup/terraform-v6/examples/basic/main.tf b/aws-integration-setup/terraform-v6/examples/basic/main.tf index 168d981..7883a8c 100644 --- a/aws-integration-setup/terraform-v6/examples/basic/main.tf +++ b/aws-integration-setup/terraform-v6/examples/basic/main.tf @@ -2,7 +2,7 @@ # This example shows the minimal setup for AWS integration without Kubernetes terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index e2e563f..738dc49 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -21,8 +21,8 @@ locals { collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name - collector = contains(["collector", "both"], var.scan_mode) - managed = contains(["managed", "both"], var.scan_mode) + collector = var.scan_mode == "collector" + managed = var.scan_mode == "managed" } provider "aws" { diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf index c8524e4..c7d6df5 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/outputs.tf @@ -39,7 +39,7 @@ output "k8s_resources_secondary" { } } -# Managed EKS scan (null unless scan_mode is managed or both) +# Managed EKS scan (null unless scan_mode is managed) output "managed_scan_clusters_for_nullify" { description = "Clusters to register in Nullify for the managed scan" value = one(module.eks_managed_scan_access[*].clusters_for_nullify) diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example index 29fcac1..06d4bfc 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example @@ -17,10 +17,11 @@ eks_cluster_arns = [ ] # collector (default): in-cluster CronJob. managed: EKS access entry plus the list-only -# nullify-readonly ClusterRole, nothing runs in the cluster. both: deploy both. +# nullify-readonly ClusterRole, nothing runs in the cluster. Pick one: the two paths +# register the same cluster twice, under different identities. # scan_mode = "managed" -# Your Nullify region from the configure page, required for managed or both +# Your Nullify region from the configure page, required for managed # nullify_region = "eu-central-1" # Primary AWS region (where IAM resources are created) @@ -32,8 +33,9 @@ aws_region = "ap-southeast-2" # S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" -# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# KMS ARN from the Nullify configure page. A key ARN or an alias ARN both work. +# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS encryption. +kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Kubernetes namespace for Nullify resources kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf index af2657b..0f3fde5 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/variables.tf @@ -32,18 +32,18 @@ variable "eks_cluster_arns" { variable "scan_mode" { type = string - description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. both: deploy both" + description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. Pick one: the two paths register the same cluster under different identities, so running both duplicates its inventory" default = "collector" validation { - condition = contains(["collector", "managed", "both"], var.scan_mode) - error_message = "scan_mode must be collector, managed or both" + condition = contains(["collector", "managed"], var.scan_mode) + error_message = "scan_mode must be collector or managed. Running both against one cluster is not supported: the collector's upload is registered as an on-prem cluster keyed on cluster_name, the managed scan as the EKS cluster keyed on its ARN, and nothing joins the two -- the cluster appears twice in inventory with its pods and containers duplicated. To change mode, apply the new one and ask Nullify to remove the old cluster registration." } } variable "nullify_region" { type = string - description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed or both" + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed" default = "" } @@ -95,6 +95,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" + description = "The KMS ARN shown on the Nullify configure page: a key ARN or an alias ARN. Required when scan_mode is collector -- the collector refuses to upload without KMS encryption" default = "" } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index 26e07dd..49de74a 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -10,9 +10,10 @@ locals { clusters = { for arn in var.cluster_arns : arn => { - name = split("/", arn)[1] - region = split(":", arn)[3] - account = split(":", arn)[4] + name = split("/", arn)[1] + partition = split(":", arn)[1] + region = split(":", arn)[3] + account = split(":", arn)[4] } } } @@ -61,7 +62,7 @@ resource "aws_eks_access_policy_association" "admin_view" { region = each.value.region cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn - policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + policy_arn = "arn:${each.value.partition}:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" access_scope { type = "cluster" diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 0e2d29e..d7da1ea 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -175,7 +175,7 @@ run "admin_view_policy_is_cluster_scoped_and_warns" { assert { condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 - error_message = "admin_view_policy must clear kubernetes_groups on the entry itself: the attribute is Optional+Computed, so null keeps whatever is in state" + error_message = "admin_view_policy must plan kubernetes_groups as an empty set on the entry: the attribute is Optional+Computed, so null would keep whatever is in state. This pins the planned configuration under a mocked provider, not the applied result" } } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf index f13e466..18b2404 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 7f34e68..0637664 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -162,7 +162,12 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { lifecycle { precondition { condition = var.cluster_name != "" - error_message = "cluster_name is required when enable_collector is true: the collector uploads to /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + error_message = "cluster_name is required when enable_collector is true. It must match your actual cluster name exactly, as AWS reports it, and it must match a cluster registered on the Nullify configure page: Nullify joins the upload on this name and silently drops or skips it otherwise. It also names the upload, /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + } + + precondition { + condition = var.kms_key_arn != "" || var.allow_unencrypted_upload + error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. Take the value from the Nullify configure page. Set allow_unencrypted_upload = true only if Nullify issued you no key." } } @@ -227,6 +232,14 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + dynamic "env" { + for_each = var.kms_key_arn == "" && var.allow_unencrypted_upload ? [1] : [] + content { + name = "ALLOW_UNENCRYPTED_UPLOAD" + value = "true" + } + } + dynamic "env" { for_each = var.enable_debug ? [1] : [] content { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf index 895c880..d01afe6 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { kubernetes = { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 186e2f9..005004d 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -7,6 +7,7 @@ run "collector_only_is_the_default" { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } assert { @@ -14,6 +15,11 @@ run "collector_only_is_the_default" { error_message = "The CronJob must set CLUSTER_NAME: it names the upload, and collectors without one all write k8s-collector/default-name-data.json" } + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD"]) == 0 + error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set unless allow_unencrypted_upload is true" + } + assert { condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 error_message = "The collector must be deployed by default" @@ -35,6 +41,7 @@ run "collector_requires_a_role_arn" { variables { cluster_name = "acme-prod" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } expect_failures = [kubernetes_service_account.nullify_collector_sa] @@ -46,11 +53,49 @@ run "collector_requires_a_cluster_name" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } expect_failures = [kubernetes_cron_job_v1.k8s_collector] } +# The published collector images exit 1 on upload with "upload blocked: no KMS +# encryption and unencrypted uploads not explicitly allowed" when neither is +# set, after a successful collection. Failing at plan is the only signal the +# customer gets before that. +run "collector_requires_a_kms_key" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + } + + expect_failures = [kubernetes_cron_job_v1.k8s_collector] +} + +run "unencrypted_upload_is_an_explicit_opt_out" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + allow_unencrypted_upload = true + } + + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD" && env.value == "true"]) == 1 + error_message = "allow_unencrypted_upload must be the only thing that sets ALLOW_UNENCRYPTED_UPLOAD" + } + + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN"]) == 0 + error_message = "NULLIFY_KMS_KEY_ARN must not be set when no key was given" + } +} + run "managed_scan_rbac_only" { command = plan diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf index 2660b57..7dbea72 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf @@ -41,7 +41,7 @@ variable "iam_role_arn" { variable "cluster_name" { type = string - description = "Name of the cluster this collector runs in, required when enable_collector is true. It becomes the collector's CLUSTER_NAME, which names its upload: /k8s-collector/-data.json. Two clusters sharing a name overwrite each other's inventory, so give every cluster its own. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" + description = "Name of the cluster this collector runs in, required when enable_collector is true. It must match your actual cluster name exactly, as AWS reports it, and it must match a cluster registered on the Nullify configure page: Nullify joins the upload on this name and silently drops or skips it otherwise. It becomes the collector's CLUSTER_NAME and names the upload, /k8s-collector/-data.json, so two clusters sharing a name overwrite each other's inventory. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" default = "" } @@ -83,7 +83,7 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" + description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" default = "" validation { @@ -92,6 +92,12 @@ variable "kms_key_arn" { } } +variable "allow_unencrypted_upload" { + type = bool + description = "WARNING: send the cluster inventory to Nullify's S3 area with no KMS encryption. Sets ALLOW_UNENCRYPTED_UPLOAD on the collector, which otherwise exits 1 rather than upload unencrypted. This is the only thing that sets that variable. Use it only if Nullify issued you no KMS key; otherwise leave it false and set kms_key_arn" + default = false +} + variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 5614ed4..90aba6e 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -1,5 +1,7 @@ data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + data "aws_eks_cluster" "clusters" { count = var.enable_kubernetes_integration ? length(var.eks_cluster_arns) : 0 region = split(":", var.eks_cluster_arns[count.index])[3] @@ -17,7 +19,7 @@ locals { all_oidc_ids = [for cluster in local.all_clusters_info : cluster.oidc_id] eks_oidc_provider_arns = var.enable_kubernetes_integration ? [ for cluster in local.all_clusters_info : - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${cluster.region}.amazonaws.com/id/${cluster.oidc_id}" + "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${cluster.region}.amazonaws.com/id/${cluster.oidc_id}" ] : [] } @@ -49,7 +51,7 @@ data "aws_iam_policy_document" "assume_role_policy" { effect = "Allow" principals { type = "Federated" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${statement.value.region}.amazonaws.com/id/${statement.value.oidc_id}"] + identifiers = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${statement.value.region}.amazonaws.com/id/${statement.value.oidc_id}"] } actions = ["sts:AssumeRoleWithWebIdentity"] condition { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf index 27474ea..f88960e 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf @@ -18,16 +18,22 @@ locals { # S3 configuration enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" - s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" + s3_bucket_arn = var.s3_bucket_name != "" ? "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. IAM ignores an alias ARN in Resource, so an alias also - # grants key/* in its own account and region; Nullify's key policy is the - # real gate. A key ARN resolves in IAM on its own and is granted alone. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - kms_arn_is_alias = length(local.kms_arn_parts) == 6 && startswith(element(local.kms_arn_parts, 5), "alias/") - kms_alias_key_wildcard = local.kms_arn_is_alias ? ["arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*"] : [] - kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_alias_key_wildcard) : [] + # KMS configuration. The key Nullify hands out lives in Nullify's account, so + # key/* is granted next to the ARN: IAM never resolves an alias ARN in a + # Resource, and the key policy on a key Nullify owns is the real gate on what + # this role can use. The wildcard is derived only when the ARN's account + # differs from this one, so a key pasted from the customer's own account + # grants that key alone and reaches nothing else in the account. For a + # Nullify-supplied key this grants what the CloudFormation template grants. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_full = length(local.kms_arn_parts) == 6 + kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" + kms_arn_is_cross_account = local.kms_arn_is_full && local.kms_arn_account != data.aws_caller_identity.current.account_id + kms_key_wildcard = local.kms_arn_is_cross_account ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf index 6552e44..db5825b 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf @@ -81,7 +81,7 @@ output "policy_arns" { } output "kms_policy_resources" { - description = "Resources the KMS policy grants: kms_key_arn and key/* in its account and region (empty when kms_key_arn is not set)" + description = "Resources the KMS policy grants: kms_key_arn, plus key/* in its account and region when that account is not this one (empty when kms_key_arn is not set)" value = local.kms_policy_resources } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 29bf443..bb192e4 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -1,4 +1,16 @@ mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_partition" { + defaults = { + partition = "aws" + } + } + mock_data "aws_iam_policy_document" { defaults = { json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" @@ -22,7 +34,7 @@ variables { nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" } -run "kms_alias_grants_the_alias_and_key_wildcard" { +run "cross_account_alias_grants_the_alias_and_key_wildcard" { command = plan variables { @@ -31,11 +43,11 @@ run "kms_alias_grants_the_alias_and_key_wildcard" { assert { condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads", "arn:aws:kms:eu-central-1:111122223333:key/*"]) - error_message = "An alias ARN must grant the alias and key/* in the same account and region" + error_message = "An alias ARN in Nullify's account must grant the alias and key/* in that account and region" } } -run "kms_key_arn_grants_only_that_key" { +run "cross_account_key_arn_grants_the_key_wildcard_too" { command = plan variables { @@ -43,8 +55,21 @@ run "kms_key_arn_grants_only_that_key" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) - error_message = "A key ARN resolves in IAM on its own: only an alias ARN needs key/*" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab", "arn:aws:kms:us-east-2:111122223333:key/*"]) + error_message = "A key ARN in Nullify's account must grant key/* too: this is what the CloudFormation template grants for the same input" + } +} + +run "a_key_in_this_account_grants_only_that_key" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:us-east-2:123456789012:alias/my-own-key" + } + + assert { + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key"]) + error_message = "An ARN in the caller's own account must never derive key/*: that would reach every key in the account whose policy delegates to the root" } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf index f13e466..18b2404 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform-v6/versions.tf b/aws-integration-setup/terraform-v6/versions.tf index f13e466..18b2404 100644 --- a/aws-integration-setup/terraform-v6/versions.tf +++ b/aws-integration-setup/terraform-v6/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 0f6df5f..b7bb9b0 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -10,7 +10,7 @@ This directory targets AWS provider v5. If your project uses AWS provider v6, us ## Requirements -- Terraform >= 1.5 +- Terraform >= 1.8 (the version CI validates and runs the module test suites at) - AWS provider `~> 5.33` in the root and examples. `modules/eks-managed-scan-access` needs `>= 5.33` (access entries arrived in 5.33.0); `modules/nullify-aws-integration` still accepts `~> 5.0` - Kubernetes provider `~> 2.20` (only `modules/k8s-resources` and the `multi-cluster-complete` and `managed-scan` examples) @@ -75,7 +75,7 @@ terraform/ ### **Multi-Cluster Complete** (`examples/multi-cluster-complete/`) - AWS IAM + EKS integration for exactly two clusters -- `scan_mode`: `collector` (default), `managed` or `both` +- `scan_mode`: `collector` (default) or `managed`, one per cluster - Clusters can be in different regions: each is read through an AWS provider in its own region ## Kubernetes scan modes @@ -262,7 +262,7 @@ terraform init && terraform apply - `aws_region`: AWS region for IAM resources (default: ap-southeast-2) - `s3_bucket_name`: S3 bucket for scan results (optional) - `nullify_s3_access_point_arn`: S3 access point for collector uploads (optional). Grants `s3:PutObject` and `s3:PutObjectAcl` on `/object/k8s-collector/*` next to the bucket grants -- `kms_key_arn`: KMS ARN from the Nullify configure page (optional), see [KMS](#kms) +- `kms_key_arn`: KMS ARN from the Nullify configure page, see [KMS](#kms). Optional for this module; required by `k8s-resources` whenever `enable_collector` is true - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) - `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) - `tags`: Resource tags @@ -275,7 +275,9 @@ terraform init && terraform apply - a key ARN: `arn:aws:kms:::key/`, including multi-Region `key/mrk-...` keys - an alias ARN: `arn:aws:kms:::alias/` -A key ARN is granted on its own. IAM does not resolve an alias in a policy's `Resource`, so an alias ARN also grants `arn:aws:kms:::key/*` in the same Nullify account and region. Nullify's key policy is the real gate: the role can only use Nullify keys whose key policy allows it. The `kms_policy_resources` module output lists what was granted. +The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and `arn::kms:::key/*` in that same account and region: IAM does not resolve an alias ARN in a policy's `Resource`, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The `kms_policy_resources` module output lists what was granted. + +`k8s-resources` takes the same variable and writes no IAM policy: it forwards the value to the collector as `NULLIFY_KMS_KEY_ARN`, which S3 takes as `SSEKMSKeyId`. **The collector refuses to upload without it** — it walks the cluster, then exits 1 with `upload blocked: no KMS encryption and unencrypted uploads not explicitly allowed` — so `kms_key_arn` is required whenever `enable_collector` is true, and the module fails at plan without it. `allow_unencrypted_upload = true` is the only way to run the collector without a key; use it only if Nullify issued you none. ## Module Usage @@ -310,7 +312,7 @@ module "k8s_resources" { } iam_role_arn = module.nullify_aws_integration.role_arn - cluster_name = "cluster-a" + cluster_name = "acme-prod-use1" s3_bucket_name = "my-scan-results-bucket" # or the S3 access point ARN from Nullify kms_key_arn = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" aws_region = "us-west-2" @@ -318,7 +320,13 @@ module "k8s_resources" { } ``` -`cluster_name` is required whenever `enable_collector` is true, and must differ per cluster. The collector uploads to `/k8s-collector/-data.json`, so two collectors sharing a name overwrite each other's inventory and Nullify only ever sees the one that ran last. +`cluster_name` is required whenever `enable_collector` is true, and has three requirements: + +- **It must match your actual cluster name exactly**, as AWS reports it (`aws eks describe-cluster --name --query cluster.name`). Nullify joins the upload to the cluster it discovered in your AWS inventory on this name; a mismatch drops the whole payload, so no pods, services or ingresses reach the graph and nothing says so. +- **It must match a cluster registered on the Nullify configure page.** An upload for an unregistered name is skipped on ingest, again with no customer-visible signal. +- **It must differ per cluster.** The collector uploads to `/k8s-collector/-data.json`, so two collectors sharing a name overwrite each other's inventory and Nullify only ever sees the one that ran last. + +This is the same value as `collector.clusterName` in the `nullify-k8s-collector` Helm chart, which states the same requirement. `k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. @@ -360,13 +368,15 @@ module "nullify_k8s" { ## Upgrade Notes -- Terraform >= 1.5 is now required (the module already used cross-variable validation, which needs 1.9; those checks are now preconditions). +- Terraform >= 1.8 is now required. It is the floor CI validates at and the floor the module test suites run at, so the declared version is the tested one. - The root and examples require AWS provider `>= 5.33`. Run `terraform init -upgrade` if your lock file pins an older 5.x. -- `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`, and the old object is left behind for you to delete. +- `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`. The old object is in Nullify's bucket, which your role cannot read or delete: ask Nullify to remove the stale `default-name` cluster, and register the new name on the configure page before the next run. +- `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page, or set `allow_unencrypted_upload = true` if Nullify issued you no key. +- `scan_mode = "both"` is no longer accepted in `examples/multi-cluster-complete`. The collector registers a cluster as on-prem, keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins them, so "both" listed one cluster twice with its pods and containers duplicated. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. -- The KMS policy grants `key/*` in the account and region of `kms_key_arn` when that value is an alias ARN. A key ARN grants only itself, as the CloudFormation template does. +- The KMS policy grants `key/*` in the account and region of `kms_key_arn` whenever that account is not your own, for alias ARNs and key ARNs alike — the same resources the CloudFormation template grants. An ARN in your own account grants only that key. ## Security Considerations diff --git a/aws-integration-setup/terraform/examples/basic/main.tf b/aws-integration-setup/terraform/examples/basic/main.tf index 12b49af..14c3629 100644 --- a/aws-integration-setup/terraform/examples/basic/main.tf +++ b/aws-integration-setup/terraform/examples/basic/main.tf @@ -2,7 +2,7 @@ # This example shows the minimal setup for AWS integration without Kubernetes terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform/examples/managed-scan/main.tf b/aws-integration-setup/terraform/examples/managed-scan/main.tf index 074fee9..97b8144 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/main.tf +++ b/aws-integration-setup/terraform/examples/managed-scan/main.tf @@ -2,7 +2,7 @@ # through an EKS access entry. Nothing runs in the cluster. terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index 5f1f18c..63b05fe 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { @@ -21,8 +21,8 @@ locals { collector_upload_target = var.nullify_s3_access_point_arn != "" ? var.nullify_s3_access_point_arn : var.s3_bucket_name - collector = contains(["collector", "both"], var.scan_mode) - managed = contains(["managed", "both"], var.scan_mode) + collector = var.scan_mode == "collector" + managed = var.scan_mode == "managed" } provider "aws" { diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf index e059fbd..f2631fd 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/outputs.tf @@ -39,7 +39,7 @@ output "k8s_resources_secondary" { } } -# Managed EKS scan (empty unless scan_mode is managed or both) +# Managed EKS scan (empty unless scan_mode is managed) output "managed_scan_clusters_for_nullify" { description = "Clusters to register in Nullify for the managed scan" value = flatten(concat(module.eks_managed_scan_access_primary[*].clusters_for_nullify, module.eks_managed_scan_access_secondary[*].clusters_for_nullify)) diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example index 29fcac1..06d4bfc 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example @@ -17,10 +17,11 @@ eks_cluster_arns = [ ] # collector (default): in-cluster CronJob. managed: EKS access entry plus the list-only -# nullify-readonly ClusterRole, nothing runs in the cluster. both: deploy both. +# nullify-readonly ClusterRole, nothing runs in the cluster. Pick one: the two paths +# register the same cluster twice, under different identities. # scan_mode = "managed" -# Your Nullify region from the configure page, required for managed or both +# Your Nullify region from the configure page, required for managed # nullify_region = "eu-central-1" # Primary AWS region (where IAM resources are created) @@ -32,8 +33,9 @@ aws_region = "ap-southeast-2" # S3 access point ARN for collector uploads (optional, shown on the Nullify configure page if used) # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" -# KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# KMS ARN from the Nullify configure page. A key ARN or an alias ARN both work. +# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS encryption. +kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" # Kubernetes namespace for Nullify resources kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf index af2657b..0f3fde5 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/variables.tf @@ -32,18 +32,18 @@ variable "eks_cluster_arns" { variable "scan_mode" { type = string - description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. both: deploy both" + description = "collector: in-cluster CronJob with IRSA (default). managed: Nullify lists resources through an EKS access entry and the list-only nullify-readonly ClusterRole, with nothing running in the cluster. Pick one: the two paths register the same cluster under different identities, so running both duplicates its inventory" default = "collector" validation { - condition = contains(["collector", "managed", "both"], var.scan_mode) - error_message = "scan_mode must be collector, managed or both" + condition = contains(["collector", "managed"], var.scan_mode) + error_message = "scan_mode must be collector or managed. Running both against one cluster is not supported: the collector's upload is registered as an on-prem cluster keyed on cluster_name, the managed scan as the EKS cluster keyed on its ARN, and nothing joins the two -- the cluster appears twice in inventory with its pods and containers duplicated. To change mode, apply the new one and ask Nullify to remove the old cluster registration." } } variable "nullify_region" { type = string - description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed or both" + description = "Your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2). Required when scan_mode is managed" default = "" } @@ -95,6 +95,6 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" + description = "The KMS ARN shown on the Nullify configure page: a key ARN or an alias ARN. Required when scan_mode is collector -- the collector refuses to upload without KMS encryption" default = "" } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 3af95a0..3459c84 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -10,9 +10,10 @@ locals { clusters = { for arn in var.cluster_arns : arn => { - name = split("/", arn)[1] - region = split(":", arn)[3] - account = split(":", arn)[4] + name = split("/", arn)[1] + partition = split(":", arn)[1] + region = split(":", arn)[3] + account = split(":", arn)[4] } } } @@ -65,7 +66,7 @@ resource "aws_eks_access_policy_association" "admin_view" { cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn - policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + policy_arn = "arn:${each.value.partition}:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" access_scope { type = "cluster" diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 3b88d30..b9b9924 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -204,7 +204,7 @@ run "admin_view_policy_is_cluster_scoped_and_warns" { assert { condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 - error_message = "admin_view_policy must clear kubernetes_groups on the entry itself: the attribute is Optional+Computed, so null keeps whatever is in state" + error_message = "admin_view_policy must plan kubernetes_groups as an empty set on the entry: the attribute is Optional+Computed, so null would keep whatever is in state. This pins the planned configuration under a mocked provider, not the applied result" } } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf index 6dd2f65..4b13899 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 7f34e68..0637664 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -162,7 +162,12 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { lifecycle { precondition { condition = var.cluster_name != "" - error_message = "cluster_name is required when enable_collector is true: the collector uploads to /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + error_message = "cluster_name is required when enable_collector is true. It must match your actual cluster name exactly, as AWS reports it, and it must match a cluster registered on the Nullify configure page: Nullify joins the upload on this name and silently drops or skips it otherwise. It also names the upload, /k8s-collector/-data.json, so collectors without a name all overwrite the same k8s-collector/default-name-data.json object." + } + + precondition { + condition = var.kms_key_arn != "" || var.allow_unencrypted_upload + error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. Take the value from the Nullify configure page. Set allow_unencrypted_upload = true only if Nullify issued you no key." } } @@ -227,6 +232,14 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } + dynamic "env" { + for_each = var.kms_key_arn == "" && var.allow_unencrypted_upload ? [1] : [] + content { + name = "ALLOW_UNENCRYPTED_UPLOAD" + value = "true" + } + } + dynamic "env" { for_each = var.enable_debug ? [1] : [] content { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/providers.tf b/aws-integration-setup/terraform/modules/k8s-resources/providers.tf index 895c880..d01afe6 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/providers.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { kubernetes = { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 186e2f9..005004d 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -7,6 +7,7 @@ run "collector_only_is_the_default" { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } assert { @@ -14,6 +15,11 @@ run "collector_only_is_the_default" { error_message = "The CronJob must set CLUSTER_NAME: it names the upload, and collectors without one all write k8s-collector/default-name-data.json" } + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD"]) == 0 + error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set unless allow_unencrypted_upload is true" + } + assert { condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_service_account.nullify_collector_sa) == 1 error_message = "The collector must be deployed by default" @@ -35,6 +41,7 @@ run "collector_requires_a_role_arn" { variables { cluster_name = "acme-prod" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } expect_failures = [kubernetes_service_account.nullify_collector_sa] @@ -46,11 +53,49 @@ run "collector_requires_a_cluster_name" { variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } expect_failures = [kubernetes_cron_job_v1.k8s_collector] } +# The published collector images exit 1 on upload with "upload blocked: no KMS +# encryption and unencrypted uploads not explicitly allowed" when neither is +# set, after a successful collection. Failing at plan is the only signal the +# customer gets before that. +run "collector_requires_a_kms_key" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + } + + expect_failures = [kubernetes_cron_job_v1.k8s_collector] +} + +run "unencrypted_upload_is_an_explicit_opt_out" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + allow_unencrypted_upload = true + } + + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD" && env.value == "true"]) == 1 + error_message = "allow_unencrypted_upload must be the only thing that sets ALLOW_UNENCRYPTED_UPLOAD" + } + + assert { + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN"]) == 0 + error_message = "NULLIFY_KMS_KEY_ARN must not be set when no key was given" + } +} + run "managed_scan_rbac_only" { command = plan diff --git a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf index 2660b57..7dbea72 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf @@ -41,7 +41,7 @@ variable "iam_role_arn" { variable "cluster_name" { type = string - description = "Name of the cluster this collector runs in, required when enable_collector is true. It becomes the collector's CLUSTER_NAME, which names its upload: /k8s-collector/-data.json. Two clusters sharing a name overwrite each other's inventory, so give every cluster its own. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" + description = "Name of the cluster this collector runs in, required when enable_collector is true. It must match your actual cluster name exactly, as AWS reports it, and it must match a cluster registered on the Nullify configure page: Nullify joins the upload on this name and silently drops or skips it otherwise. It becomes the collector's CLUSTER_NAME and names the upload, /k8s-collector/-data.json, so two clusters sharing a name overwrite each other's inventory. Same value as collector.clusterName in the nullify-k8s-collector Helm chart" default = "" } @@ -83,7 +83,7 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN or an alias ARN" + description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" default = "" validation { @@ -92,6 +92,12 @@ variable "kms_key_arn" { } } +variable "allow_unencrypted_upload" { + type = bool + description = "WARNING: send the cluster inventory to Nullify's S3 area with no KMS encryption. Sets ALLOW_UNENCRYPTED_UPLOAD on the collector, which otherwise exits 1 rather than upload unencrypted. This is the only thing that sets that variable. Use it only if Nullify issued you no KMS key; otherwise leave it false and set kms_key_arn" + default = false +} + variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index ae04233..52732a6 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -1,5 +1,7 @@ data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + data "aws_region" "current" {} locals { @@ -31,7 +33,7 @@ locals { all_oidc_ids = [for cluster in local.all_clusters_info : cluster.oidc_id] eks_oidc_provider_arns = var.enable_kubernetes_integration ? [ for cluster in local.all_clusters_info : - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${cluster.region}.amazonaws.com/id/${cluster.oidc_id}" + "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${cluster.region}.amazonaws.com/id/${cluster.oidc_id}" ] : [] } @@ -73,7 +75,7 @@ data "aws_iam_policy_document" "assume_role_policy" { effect = "Allow" principals { type = "Federated" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${statement.value.region}.amazonaws.com/id/${statement.value.oidc_id}"] + identifiers = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${statement.value.region}.amazonaws.com/id/${statement.value.oidc_id}"] } actions = ["sts:AssumeRoleWithWebIdentity"] condition { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf index 27474ea..f88960e 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf @@ -18,16 +18,22 @@ locals { # S3 configuration enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" - s3_bucket_arn = var.s3_bucket_name != "" ? "arn:aws:s3:::${var.s3_bucket_name}" : "" + s3_bucket_arn = var.s3_bucket_name != "" ? "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. IAM ignores an alias ARN in Resource, so an alias also - # grants key/* in its own account and region; Nullify's key policy is the - # real gate. A key ARN resolves in IAM on its own and is granted alone. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - kms_arn_is_alias = length(local.kms_arn_parts) == 6 && startswith(element(local.kms_arn_parts, 5), "alias/") - kms_alias_key_wildcard = local.kms_arn_is_alias ? ["arn:aws:kms:${element(local.kms_arn_parts, 3)}:${element(local.kms_arn_parts, 4)}:key/*"] : [] - kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_alias_key_wildcard) : [] + # KMS configuration. The key Nullify hands out lives in Nullify's account, so + # key/* is granted next to the ARN: IAM never resolves an alias ARN in a + # Resource, and the key policy on a key Nullify owns is the real gate on what + # this role can use. The wildcard is derived only when the ARN's account + # differs from this one, so a key pasted from the customer's own account + # grants that key alone and reaches nothing else in the account. For a + # Nullify-supplied key this grants what the CloudFormation template grants. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_full = length(local.kms_arn_parts) == 6 + kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" + kms_arn_is_cross_account = local.kms_arn_is_full && local.kms_arn_account != data.aws_caller_identity.current.account_id + kms_key_wildcard = local.kms_arn_is_cross_account ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf index 6552e44..db5825b 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf @@ -81,7 +81,7 @@ output "policy_arns" { } output "kms_policy_resources" { - description = "Resources the KMS policy grants: kms_key_arn and key/* in its account and region (empty when kms_key_arn is not set)" + description = "Resources the KMS policy grants: kms_key_arn, plus key/* in its account and region when that account is not this one (empty when kms_key_arn is not set)" value = local.kms_policy_resources } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 56e4ff7..362a38d 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -1,4 +1,16 @@ mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_partition" { + defaults = { + partition = "aws" + } + } + mock_data "aws_iam_policy_document" { defaults = { json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" @@ -28,7 +40,7 @@ variables { nullify_role_arn = "arn:aws:iam::123456789012:role/NullifyRole" } -run "kms_alias_grants_the_alias_and_key_wildcard" { +run "cross_account_alias_grants_the_alias_and_key_wildcard" { command = plan variables { @@ -37,11 +49,11 @@ run "kms_alias_grants_the_alias_and_key_wildcard" { assert { condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads", "arn:aws:kms:eu-central-1:111122223333:key/*"]) - error_message = "An alias ARN must grant the alias and key/* in the same account and region" + error_message = "An alias ARN in Nullify's account must grant the alias and key/* in that account and region" } } -run "kms_key_arn_grants_only_that_key" { +run "cross_account_key_arn_grants_the_key_wildcard_too" { command = plan variables { @@ -49,8 +61,21 @@ run "kms_key_arn_grants_only_that_key" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) - error_message = "A key ARN resolves in IAM on its own: only an alias ARN needs key/*" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab", "arn:aws:kms:us-east-2:111122223333:key/*"]) + error_message = "A key ARN in Nullify's account must grant key/* too: this is what the CloudFormation template grants for the same input" + } +} + +run "a_key_in_this_account_grants_only_that_key" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:us-east-2:123456789012:alias/my-own-key" + } + + assert { + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key"]) + error_message = "An ARN in the caller's own account must never derive key/*: that would reach every key in the account whose policy delegates to the root" } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf index 946e64b..6a1c978 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { diff --git a/aws-integration-setup/terraform/versions.tf b/aws-integration-setup/terraform/versions.tf index f7c9efb..c4ff8a8 100644 --- a/aws-integration-setup/terraform/versions.tf +++ b/aws-integration-setup/terraform/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.5" + required_version = ">= 1.8" required_providers { aws = { From a90bf29dfefc23f0f8b18f46831f0ad7be708c8f Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 10:48:44 +1000 Subject: [PATCH 21/36] fix(terraform): drop the unencrypted-upload opt-out and reject collector + managed scan together allow_unencrypted_upload could not work against Nullify's bucket: an empty kms_key_arn also sets enable_kms_access = false, so the role gets no KMS statement, and the bucket's SSE-KMS default encryption still requires kms:GenerateDataKey on Nullify's key. The variable traded the collector's self-explaining error for an opaque S3 AccessDenied, so it is removed and kms_key_arn is unconditionally required when enable_collector is true. - k8s-resources: remove allow_unencrypted_upload and the ALLOW_UNENCRYPTED_UPLOAD env block; the KMS precondition is now kms_key_arn != "" alone - k8s-resources: precondition rejecting enable_collector together with enable_managed_scan_rbac, matching the example's scan_mode = "both" rejection - nullify-aws-integration: kms_key_arn description now states the cross-account rule the locals actually implement - terraform/README.md: fold #88's bare-key-form note into the rewritten KMS section so merging both PRs cannot leave two near-duplicate paragraphs - multi-cluster-complete: comment out the placeholder KMS ARN and shape it so the variable regex rejects it if pasted as-is - CI: validate rows for the four remaining modules/ directories Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 13 +++++++- aws-integration-setup/terraform-v6/README.md | 2 +- .../terraform.tfvars.example | 6 ++-- .../modules/k8s-resources/main.tf | 12 ++----- .../modules/k8s-resources/managed_scan.tf | 7 ++++ .../tests/k8s_resources.tftest.hcl | 32 ++++++++++++------- .../modules/k8s-resources/variables.tf | 8 +---- .../nullify-aws-integration/variables.tf | 2 +- aws-integration-setup/terraform/README.md | 12 +++++-- .../terraform.tfvars.example | 6 ++-- .../terraform/modules/k8s-resources/main.tf | 12 ++----- .../modules/k8s-resources/managed_scan.tf | 7 ++++ .../tests/k8s_resources.tftest.hcl | 32 ++++++++++++------- .../modules/k8s-resources/variables.tf | 8 +---- .../nullify-aws-integration/variables.tf | 2 +- 15 files changed, 94 insertions(+), 67 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index 001afd0..b0644a4 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -34,7 +34,10 @@ jobs: matrix: # AWS rows run the declared floor (Terraform 1.8, the version the # terraform-test job also runs); examples that use ephemeral resources - # (v6 multi-cluster, v6 managed-scan) need 1.10. + # (v6 multi-cluster, v6 managed-scan) need 1.10. Every modules/ directory + # is listed so a module added without a test suite is still parsed and + # fmt-checked; the validation blocks and preconditions themselves are + # only reached by the terraform-test job below, which supplies values. include: - directory: gcp-integration-setup/terraform terraform: 1.9.5 @@ -54,6 +57,10 @@ jobs: terraform: 1.8.5 - directory: aws-integration-setup/terraform/modules/eks-managed-scan-access terraform: 1.8.5 + - directory: aws-integration-setup/terraform/modules/k8s-resources + terraform: 1.8.5 + - directory: aws-integration-setup/terraform/modules/nullify-aws-integration + terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6 terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6/examples/basic @@ -64,6 +71,10 @@ jobs: terraform: 1.10.5 - directory: aws-integration-setup/terraform-v6/modules/eks-managed-scan-access terraform: 1.8.5 + - directory: aws-integration-setup/terraform-v6/modules/k8s-resources + terraform: 1.8.5 + - directory: aws-integration-setup/terraform-v6/modules/nullify-aws-integration + terraform: 1.8.5 - directory: azure-integration-setup/terraform terraform: 1.9.5 - directory: azure-integration-setup/terraform/examples/management-group diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index cd4d16c..e32bf63 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -114,7 +114,7 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `tags`: Resource tags - `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` -`k8s-resources` requires `cluster_name` and `kms_key_arn` whenever `enable_collector` is true. `cluster_name` must match your actual cluster name exactly, must match a cluster registered on the Nullify configure page, and must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, and Nullify joins that upload on the name. See "Kubernetes Resources" and "KMS" in `../terraform/README.md`. +`k8s-resources` requires `cluster_name` and `kms_key_arn` whenever `enable_collector` is true, with no unencrypted opt-out, and rejects `enable_collector` together with `enable_managed_scan_rbac`. `cluster_name` must match your actual cluster name exactly, must match a cluster registered on the Nullify configure page, and must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, and Nullify joins that upload on the name. See "Kubernetes Resources" and "KMS" in `../terraform/README.md`. `k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example index 06d4bfc..85c8d76 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/terraform.tfvars.example @@ -34,8 +34,10 @@ aws_region = "ap-southeast-2" # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" # KMS ARN from the Nullify configure page. A key ARN or an alias ARN both work. -# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS encryption. -kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS +# encryption, and there is no opt-out. The key lives in Nullify's account, not yours, +# so it is not the account id used for nullify_role_arn or eks_cluster_arns above. +# kms_key_arn = "arn:aws:kms:::key/" # Kubernetes namespace for Nullify resources kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 0637664..efa8271 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -166,8 +166,8 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } precondition { - condition = var.kms_key_arn != "" || var.allow_unencrypted_upload - error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. Take the value from the Nullify configure page. Set allow_unencrypted_upload = true only if Nullify issued you no key." + condition = var.kms_key_arn != "" + error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. There is no opt-out. Nullify's upload bucket sets SSE-KMS as its default encryption, so an upload sent with no encryption headers is still encrypted with Nullify's key, and S3 rejects it unless the caller holds kms:GenerateDataKey on that key -- which this role only gets from kms_key_arn. Take the value from the Nullify configure page." } } @@ -232,14 +232,6 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } - dynamic "env" { - for_each = var.kms_key_arn == "" && var.allow_unencrypted_upload ? [1] : [] - content { - name = "ALLOW_UNENCRYPTED_UPLOAD" - value = "true" - } - } - dynamic "env" { for_each = var.enable_debug ? [1] : [] content { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf index a8d394b..6676e38 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf @@ -32,6 +32,13 @@ locals { resource "kubernetes_cluster_role_v1" "nullify_readonly" { count = var.enable_managed_scan_rbac ? 1 : 0 + lifecycle { + precondition { + condition = !var.enable_collector + error_message = "enable_collector and enable_managed_scan_rbac cannot both be true for one cluster. The collector's upload is registered as an on-prem cluster keyed on cluster_name; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster appears twice in inventory with its pods and containers duplicated. Pick one mode: apply it, then ask Nullify to remove the old cluster registration." + } + } + metadata { name = var.managed_scan_cluster_role_name labels = { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 005004d..9022f6d 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -17,7 +17,7 @@ run "collector_only_is_the_default" { assert { condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD"]) == 0 - error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set unless allow_unencrypted_upload is true" + error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set: the module has no unencrypted path, and Nullify's bucket defaults to SSE-KMS so an unencrypted upload is rejected by KMS anyway" } assert { @@ -60,9 +60,9 @@ run "collector_requires_a_cluster_name" { } # The published collector images exit 1 on upload with "upload blocked: no KMS -# encryption and unencrypted uploads not explicitly allowed" when neither is -# set, after a successful collection. Failing at plan is the only signal the -# customer gets before that. +# encryption and unencrypted uploads not explicitly allowed" when the key is +# missing, after a successful collection. Failing at plan is the only signal +# the customer gets before that. run "collector_requires_a_kms_key" { command = plan @@ -75,24 +75,34 @@ run "collector_requires_a_kms_key" { expect_failures = [kubernetes_cron_job_v1.k8s_collector] } -run "unencrypted_upload_is_an_explicit_opt_out" { +run "collector_and_managed_scan_rbac_cannot_both_run" { command = plan variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" - allow_unencrypted_upload = true + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + enable_collector = true + enable_managed_scan_rbac = true } - assert { - condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD" && env.value == "true"]) == 1 - error_message = "allow_unencrypted_upload must be the only thing that sets ALLOW_UNENCRYPTED_UPLOAD" + expect_failures = [kubernetes_cluster_role_v1.nullify_readonly] +} + +run "the_kms_key_reaches_the_collector" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } assert { - condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN"]) == 0 - error_message = "NULLIFY_KMS_KEY_ARN must not be set when no key was given" + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN" && env.value == "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads"]) == 1 + error_message = "kms_key_arn must reach the collector as NULLIFY_KMS_KEY_ARN, which S3 takes as SSEKMSKeyId" } } diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf index 7dbea72..61d5cd9 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/variables.tf @@ -83,7 +83,7 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" + description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it, and there is no unencrypted path -- Nullify's bucket defaults to SSE-KMS, so an unencrypted upload needs kms:GenerateDataKey on Nullify's key anyway. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" default = "" validation { @@ -92,12 +92,6 @@ variable "kms_key_arn" { } } -variable "allow_unencrypted_upload" { - type = bool - description = "WARNING: send the cluster inventory to Nullify's S3 area with no KMS encryption. Sets ALLOW_UNENCRYPTED_UPLOAD on the collector, which otherwise exits 1 rather than upload unencrypted. This is the only thing that sets that variable. Use it only if Nullify issued you no KMS key; otherwise leave it false and set kms_key_arn" - default = false -} - variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf index f60158c..95c8808 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf @@ -97,7 +97,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is granted on its own. An alias ARN also grants key/* in the same Nullify account and region, because IAM does not resolve an alias in a policy Resource. Nullify's key policy decides which of those keys the role can use" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and key/* in that same account and region, for key ARNs and alias ARNs alike: IAM does not resolve an alias in a policy Resource, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The kms_policy_resources output lists what was granted" default = "" validation { diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index b7bb9b0..bd88919 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -277,7 +277,9 @@ terraform init && terraform apply The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and `arn::kms:::key/*` in that same account and region: IAM does not resolve an alias ARN in a policy's `Resource`, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The `kms_policy_resources` module output lists what was granted. -`k8s-resources` takes the same variable and writes no IAM policy: it forwards the value to the collector as `NULLIFY_KMS_KEY_ARN`, which S3 takes as `SSEKMSKeyId`. **The collector refuses to upload without it** — it walks the cluster, then exits 1 with `upload blocked: no KMS encryption and unencrypted uploads not explicitly allowed` — so `kms_key_arn` is required whenever `enable_collector` is true, and the module fails at plan without it. `allow_unencrypted_upload = true` is the only way to run the collector without a key; use it only if Nullify issued you none. +`k8s-resources` takes the same variable and writes no IAM policy: it forwards the value to the collector as `NULLIFY_KMS_KEY_ARN`, which S3 takes as `SSEKMSKeyId`. S3 resolves a bare key ID or a bare `alias/` against the calling account, and the key is in Nullify's, so only the two ARN forms above work there too; both modules validate for them. + +**The collector refuses to upload without a key** — it walks the cluster, then exits 1 with `upload blocked: no KMS encryption and unencrypted uploads not explicitly allowed` — so `kms_key_arn` is required whenever `enable_collector` is true, and the module fails at plan without it. There is no opt-out, and one would not help: Nullify's upload bucket sets SSE-KMS as its default encryption, so an object written with no encryption headers is still encrypted with Nullify's key, and S3 refuses the `PutObject` unless the caller holds `kms:GenerateDataKey` on it. `kms_key_arn` is the only thing that grants the role that. ## Module Usage @@ -330,6 +332,8 @@ This is the same value as `collector.clusterName` in the `nullify-k8s-collector` `k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. +`enable_collector` and `enable_managed_scan_rbac` are mutually exclusive for one cluster, and the module fails at plan if both are true. The collector's upload registers the cluster as on-prem keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so enabling both lists the cluster twice with its pods and containers duplicated. To change mode, apply the new one and ask Nullify to remove the old cluster registration. + `collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. It is the same build as `k8s-collector-latest`, which the `nullify-k8s-collector` Helm chart deploys, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. ## Outputs @@ -371,8 +375,10 @@ module "nullify_k8s" { - Terraform >= 1.8 is now required. It is the floor CI validates at and the floor the module test suites run at, so the declared version is the tested one. - The root and examples require AWS provider `>= 5.33`. Run `terraform init -upgrade` if your lock file pins an older 5.x. - `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`. The old object is in Nullify's bucket, which your role cannot read or delete: ask Nullify to remove the stale `default-name` cluster, and register the new name on the configure page before the next run. -- `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page, or set `allow_unencrypted_upload = true` if Nullify issued you no key. -- `scan_mode = "both"` is no longer accepted in `examples/multi-cluster-complete`. The collector registers a cluster as on-prem, keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins them, so "both" listed one cluster twice with its pods and containers duplicated. +- `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true, with no opt-out. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page; ask Nullify for a key if you have none. +- `k8s-resources` no longer declares `allow_unencrypted_upload`. Passing it is now a plan error. It never worked against Nullify's bucket: leaving `kms_key_arn` empty also leaves the IAM role with no KMS statement, and the bucket's SSE-KMS default encryption makes `kms:GenerateDataKey` on Nullify's key a requirement of every upload, encrypted headers or not. The variable turned a self-explaining collector error into an S3 `AccessDenied`. +- `k8s-resources` now rejects `enable_collector` and `enable_managed_scan_rbac` together, which `examples/multi-cluster-complete` already rejected as `scan_mode = "both"`. The two paths register the same cluster under different identities and nothing joins them. +- `scan_mode = "both"` is no longer accepted in `examples/multi-cluster-complete`, and the module enforces the same rule. The collector registers a cluster as on-prem, keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins them, so "both" listed one cluster twice with its pods and containers duplicated. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example index 06d4bfc..85c8d76 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/terraform.tfvars.example @@ -34,8 +34,10 @@ aws_region = "ap-southeast-2" # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" # KMS ARN from the Nullify configure page. A key ARN or an alias ARN both work. -# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS encryption. -kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# REQUIRED when scan_mode is collector: the collector refuses to upload without KMS +# encryption, and there is no opt-out. The key lives in Nullify's account, not yours, +# so it is not the account id used for nullify_role_arn or eks_cluster_arns above. +# kms_key_arn = "arn:aws:kms:::key/" # Kubernetes namespace for Nullify resources kubernetes_namespace = "nullify" diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 0637664..efa8271 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -166,8 +166,8 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } precondition { - condition = var.kms_key_arn != "" || var.allow_unencrypted_upload - error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. Take the value from the Nullify configure page. Set allow_unencrypted_upload = true only if Nullify issued you no key." + condition = var.kms_key_arn != "" + error_message = "kms_key_arn is required when enable_collector is true: the collector walks the cluster, then refuses to upload without KMS encryption and exits 1, so every CronJob run fails and Nullify receives nothing. There is no opt-out. Nullify's upload bucket sets SSE-KMS as its default encryption, so an upload sent with no encryption headers is still encrypted with Nullify's key, and S3 rejects it unless the caller holds kms:GenerateDataKey on that key -- which this role only gets from kms_key_arn. Take the value from the Nullify configure page." } } @@ -232,14 +232,6 @@ resource "kubernetes_cron_job_v1" "k8s_collector" { } } - dynamic "env" { - for_each = var.kms_key_arn == "" && var.allow_unencrypted_upload ? [1] : [] - content { - name = "ALLOW_UNENCRYPTED_UPLOAD" - value = "true" - } - } - dynamic "env" { for_each = var.enable_debug ? [1] : [] content { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf index a8d394b..6676e38 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf @@ -32,6 +32,13 @@ locals { resource "kubernetes_cluster_role_v1" "nullify_readonly" { count = var.enable_managed_scan_rbac ? 1 : 0 + lifecycle { + precondition { + condition = !var.enable_collector + error_message = "enable_collector and enable_managed_scan_rbac cannot both be true for one cluster. The collector's upload is registered as an on-prem cluster keyed on cluster_name; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster appears twice in inventory with its pods and containers duplicated. Pick one mode: apply it, then ask Nullify to remove the old cluster registration." + } + } + metadata { name = var.managed_scan_cluster_role_name labels = { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 005004d..9022f6d 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -17,7 +17,7 @@ run "collector_only_is_the_default" { assert { condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD"]) == 0 - error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set unless allow_unencrypted_upload is true" + error_message = "ALLOW_UNENCRYPTED_UPLOAD must never be set: the module has no unencrypted path, and Nullify's bucket defaults to SSE-KMS so an unencrypted upload is rejected by KMS anyway" } assert { @@ -60,9 +60,9 @@ run "collector_requires_a_cluster_name" { } # The published collector images exit 1 on upload with "upload blocked: no KMS -# encryption and unencrypted uploads not explicitly allowed" when neither is -# set, after a successful collection. Failing at plan is the only signal the -# customer gets before that. +# encryption and unencrypted uploads not explicitly allowed" when the key is +# missing, after a successful collection. Failing at plan is the only signal +# the customer gets before that. run "collector_requires_a_kms_key" { command = plan @@ -75,24 +75,34 @@ run "collector_requires_a_kms_key" { expect_failures = [kubernetes_cron_job_v1.k8s_collector] } -run "unencrypted_upload_is_an_explicit_opt_out" { +run "collector_and_managed_scan_rbac_cannot_both_run" { command = plan variables { iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" cluster_name = "acme-prod" s3_bucket_name = "nullify-collector-uploads" - allow_unencrypted_upload = true + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + enable_collector = true + enable_managed_scan_rbac = true } - assert { - condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "ALLOW_UNENCRYPTED_UPLOAD" && env.value == "true"]) == 1 - error_message = "allow_unencrypted_upload must be the only thing that sets ALLOW_UNENCRYPTED_UPLOAD" + expect_failures = [kubernetes_cluster_role_v1.nullify_readonly] +} + +run "the_kms_key_reaches_the_collector" { + command = plan + + variables { + iam_role_arn = "arn:aws:iam::123456789012:role/AWSIntegration-acme-NullifyReadOnlyRole" + cluster_name = "acme-prod" + s3_bucket_name = "nullify-collector-uploads" + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" } assert { - condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN"]) == 0 - error_message = "NULLIFY_KMS_KEY_ARN must not be set when no key was given" + condition = length([for env in kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].env : env if env.name == "NULLIFY_KMS_KEY_ARN" && env.value == "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads"]) == 1 + error_message = "kms_key_arn must reach the collector as NULLIFY_KMS_KEY_ARN, which S3 takes as SSEKMSKeyId" } } diff --git a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf index 7dbea72..61d5cd9 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/variables.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/variables.tf @@ -83,7 +83,7 @@ variable "cronjob_schedule" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" + description = "The KMS ARN shown on the Nullify configure page, required when enable_collector is true: a key ARN or an alias ARN. The collector sends it to S3 as SSEKMSKeyId and refuses to upload without it, and there is no unencrypted path -- Nullify's bucket defaults to SSE-KMS, so an unencrypted upload needs kms:GenerateDataKey on Nullify's key anyway. The key is in Nullify's account, so a bare key ID or a bare alias/ would resolve against your own account and is rejected here" default = "" validation { @@ -92,12 +92,6 @@ variable "kms_key_arn" { } } -variable "allow_unencrypted_upload" { - type = bool - description = "WARNING: send the cluster inventory to Nullify's S3 area with no KMS encryption. Sets ALLOW_UNENCRYPTED_UPLOAD on the collector, which otherwise exits 1 rather than upload unencrypted. This is the only thing that sets that variable. Use it only if Nullify issued you no KMS key; otherwise leave it false and set kms_key_arn" - default = false -} - variable "enable_debug" { type = bool description = "Enable debug logging for troubleshooting" diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf index c78eabd..88f5b95 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf @@ -108,7 +108,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is granted on its own. An alias ARN also grants key/* in the same Nullify account and region, because IAM does not resolve an alias in a policy Resource. Nullify's key policy decides which of those keys the role can use" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and key/* in that same account and region, for key ARNs and alias ARNs alike: IAM does not resolve an alias in a policy Resource, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The kms_policy_resources output lists what was granted" default = "" validation { From a7dac03288b2deec58367bb44fe02f8a87fac82c Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 11:13:45 +1000 Subject: [PATCH 22/36] fix(terraform): move the collector/managed-scan duplicate guard to the access entry and drop phantom upgrade notes The precondition guarding against duplicate cluster registration lived on k8s-resources' ClusterRole, so authorization = admin_view_policy (which creates no Kubernetes objects) never triggered it: a direct caller running the collector and a managed scan on one cluster in that mode still plans clean. The same guard also rejected a legitimate configuration -- staging the ClusterRole ahead of a cutover with no access entry behind it yet. - Move the guard to eks-managed-scan-access's access entry (created in every authorization mode) via a new collector_cluster_arns input, keyed on cluster name; remove the now-redundant precondition from k8s-resources' ClusterRole so staging is allowed again. - Wire collector_cluster_arns through the root configs and the multi-cluster-complete examples in both terraform/ and terraform-v6/. - Add terraform test rows for both directions: rejected under admin_view_policy, and unaffected when the collector cluster differs. - Drop three Upgrade Notes bullets that describe variables (allow_unencrypted_upload, enable_managed_scan_rbac, scan_mode) absent from origin/main and both release tags -- there is no upgrade path they describe. - Add the missing Upgrade Notes section to terraform-v6/README.md with the real (non-phantom) bullets, mirrored from v5. - Fix the KMS placeholder ARN in the three remaining tfvars.example files to match the corrected multi-cluster-complete one (R3-8): the key is in Nullify's account, not the customer's. - Remove the four terraform-validate rows for module directories already covered end-to-end by the terraform-test job, and correct the comment justifying them (both matrices are hand-maintained, not auto-discovered). Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 20 +++----------- aws-integration-setup/terraform-v6/README.md | 7 +++++ .../examples/basic/terraform.tfvars.example | 3 ++- .../examples/multi-cluster-complete/main.tf | 11 ++++---- aws-integration-setup/terraform-v6/main.tf | 19 ++++++++----- .../modules/eks-managed-scan-access/main.tf | 7 +++++ .../tests/eks_managed_scan_access.tftest.hcl | 27 +++++++++++++++++++ .../eks-managed-scan-access/variables.tf | 6 +++++ .../modules/k8s-resources/managed_scan.tf | 7 ----- .../tests/k8s_resources.tftest.hcl | 14 ++++++++-- .../terraform-v6/terraform.tfvars.example | 3 ++- aws-integration-setup/terraform/README.md | 9 +++---- .../examples/basic/terraform.tfvars.example | 3 ++- .../examples/multi-cluster-complete/main.tf | 22 ++++++++------- aws-integration-setup/terraform/main.tf | 19 ++++++++----- .../modules/eks-managed-scan-access/main.tf | 7 +++++ .../tests/eks_managed_scan_access.tftest.hcl | 27 +++++++++++++++++++ .../eks-managed-scan-access/variables.tf | 6 +++++ .../modules/k8s-resources/managed_scan.tf | 7 ----- .../tests/k8s_resources.tftest.hcl | 14 ++++++++-- .../terraform/terraform.tfvars.example | 3 ++- 21 files changed, 169 insertions(+), 72 deletions(-) diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index b0644a4..882d52c 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -34,10 +34,10 @@ jobs: matrix: # AWS rows run the declared floor (Terraform 1.8, the version the # terraform-test job also runs); examples that use ephemeral resources - # (v6 multi-cluster, v6 managed-scan) need 1.10. Every modules/ directory - # is listed so a module added without a test suite is still parsed and - # fmt-checked; the validation blocks and preconditions themselves are - # only reached by the terraform-test job below, which supplies values. + # (v6 multi-cluster, v6 managed-scan) need 1.10. The three modules/ + # directories each tree has are covered by the terraform-test job + # below instead, which already runs fmt-check, init and validate (via + # terraform test) against both Terraform versions with mocked values. include: - directory: gcp-integration-setup/terraform terraform: 1.9.5 @@ -55,12 +55,6 @@ jobs: terraform: 1.8.5 - directory: aws-integration-setup/terraform/examples/managed-scan terraform: 1.8.5 - - directory: aws-integration-setup/terraform/modules/eks-managed-scan-access - terraform: 1.8.5 - - directory: aws-integration-setup/terraform/modules/k8s-resources - terraform: 1.8.5 - - directory: aws-integration-setup/terraform/modules/nullify-aws-integration - terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6 terraform: 1.8.5 - directory: aws-integration-setup/terraform-v6/examples/basic @@ -69,12 +63,6 @@ jobs: terraform: 1.10.5 - directory: aws-integration-setup/terraform-v6/examples/managed-scan terraform: 1.10.5 - - directory: aws-integration-setup/terraform-v6/modules/eks-managed-scan-access - terraform: 1.8.5 - - directory: aws-integration-setup/terraform-v6/modules/k8s-resources - terraform: 1.8.5 - - directory: aws-integration-setup/terraform-v6/modules/nullify-aws-integration - terraform: 1.8.5 - directory: azure-integration-setup/terraform terraform: 1.9.5 - directory: azure-integration-setup/terraform/examples/management-group diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index e32bf63..49d90d0 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -118,6 +118,13 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus `k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. +## Upgrade Notes + +- `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`. The old object is in Nullify's bucket, which your role cannot read or delete: ask Nullify to remove the stale `default-name` cluster, and register the new name on the configure page before the next run. +- `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true, with no opt-out. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page; ask Nullify for a key if you have none. +- `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. +- The KMS policy grants `key/*` in the account and region of `kms_key_arn` whenever that account is not your own, for alias ARNs and key ARNs alike. An ARN in your own account grants only that key. + ## Validation ```bash diff --git a/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example index 5ec8c91..d4a26e0 100644 --- a/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/basic/terraform.tfvars.example @@ -17,7 +17,8 @@ aws_region = "ap-southeast-2" # s3_bucket_name = "nullify-scan-results-acme-corp" # KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# The key lives in Nullify's account, not yours. +# kms_key_arn = "arn:aws:kms:::key/" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index 738dc49..f3e77b7 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -120,9 +120,10 @@ module "eks_managed_scan_access" { source = "../../modules/eks-managed-scan-access" count = local.managed ? 1 : 0 - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = var.eks_cluster_arns - nullify_region = var.nullify_region - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.eks_cluster_arns + nullify_region = var.nullify_region + collector_cluster_arns = local.collector ? var.eks_cluster_arns : [] + tags = var.tags } diff --git a/aws-integration-setup/terraform-v6/main.tf b/aws-integration-setup/terraform-v6/main.tf index 6cb2142..777323b 100644 --- a/aws-integration-setup/terraform-v6/main.tf +++ b/aws-integration-setup/terraform-v6/main.tf @@ -1,3 +1,7 @@ +locals { + collector_cluster_arns = var.enable_kubernetes_integration ? var.eks_cluster_arns : [] +} + module "nullify_aws_integration" { source = "./modules/nullify-aws-integration" @@ -22,11 +26,12 @@ module "eks_managed_scan_access" { source = "./modules/eks-managed-scan-access" count = var.enable_managed_scan ? 1 : 0 - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = var.managed_scan_cluster_arns - nullify_region = var.nullify_region - authorization = var.managed_scan_authorization - kubernetes_group_name = var.managed_scan_kubernetes_group - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.managed_scan_cluster_arns + nullify_region = var.nullify_region + authorization = var.managed_scan_authorization + kubernetes_group_name = var.managed_scan_kubernetes_group + collector_cluster_arns = local.collector_cluster_arns + tags = var.tags } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index 49de74a..ee80f47 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -16,6 +16,8 @@ locals { account = split(":", arn)[4] } } + + collector_cluster_names = toset([for arn in var.collector_cluster_arns : split("/", arn)[1]]) } data "aws_eks_cluster" "this" { @@ -29,6 +31,11 @@ data "aws_eks_cluster" "this" { condition = each.value.account == local.principal_account error_message = "Cluster ${each.key} is in account ${each.value.account}, but principal_arn is in account ${local.principal_account}. Nullify assumes the integration role in the cluster's own account: deploy nullify-aws-integration there and pass that role." } + + precondition { + condition = !contains(local.collector_cluster_names, each.value.name) + error_message = "Cluster ${each.value.name} is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop it from collector_cluster_arns and cluster_arns here, or disable enable_collector on that cluster's k8s-resources instance." + } } } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index d7da1ea..a468379 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -208,3 +208,30 @@ run "cross_account_clusters_are_rejected" { expect_failures = [data.aws_eks_cluster.this] } + +run "a_collector_cluster_is_rejected_even_under_admin_view_policy" { + # admin_view_policy creates no Kubernetes objects, so the k8s-resources + # ClusterRole's own precondition can never see this combination. The guard + # has to live here, on the access entry every authorization mode creates. + command = plan + + variables { + authorization = "admin_view_policy" + collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.this, check.admin_view_policy_is_broader_than_required] +} + +run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { + command = plan + + variables { + collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/some-other-cluster"] + } + + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "A collector_cluster_arns entry for a different cluster must not block this one" + } +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf index 7ebbb5d..83ee4cd 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -61,3 +61,9 @@ variable "tags" { description = "Tags to apply to the access entries" default = {} } + +variable "collector_cluster_arns" { + type = list(string) + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources" + default = [] +} diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf index 6676e38..a8d394b 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf @@ -32,13 +32,6 @@ locals { resource "kubernetes_cluster_role_v1" "nullify_readonly" { count = var.enable_managed_scan_rbac ? 1 : 0 - lifecycle { - precondition { - condition = !var.enable_collector - error_message = "enable_collector and enable_managed_scan_rbac cannot both be true for one cluster. The collector's upload is registered as an on-prem cluster keyed on cluster_name; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster appears twice in inventory with its pods and containers duplicated. Pick one mode: apply it, then ask Nullify to remove the old cluster registration." - } - } - metadata { name = var.managed_scan_cluster_role_name labels = { diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 9022f6d..85380c8 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -75,7 +75,14 @@ run "collector_requires_a_kms_key" { expect_failures = [kubernetes_cron_job_v1.k8s_collector] } -run "collector_and_managed_scan_rbac_cannot_both_run" { +run "staging_the_rbac_role_alongside_the_running_collector_plans_clean" { + # The duplicate-registration hazard is caused by the EKS access entry + # (enforced in eks-managed-scan-access, see its test suite), not by this + # ClusterRole: with no access entry mapping Nullify's principal into + # managed_scan_kubernetes_group, nothing joins the group and no managed + # scan runs. Staging the RBAC ahead of a cutover -- apply the ClusterRole + # now while the collector still runs, wire the access entry and disable + # the collector at the cutover -- must not be a plan error. command = plan variables { @@ -87,7 +94,10 @@ run "collector_and_managed_scan_rbac_cannot_both_run" { enable_managed_scan_rbac = true } - expect_failures = [kubernetes_cluster_role_v1.nullify_readonly] + assert { + condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_cluster_role_v1.nullify_readonly) == 1 + error_message = "Both the collector CronJob and the managed-scan ClusterRole must plan when no access entry exists yet" + } } run "the_kms_key_reaches_the_collector" { diff --git a/aws-integration-setup/terraform-v6/terraform.tfvars.example b/aws-integration-setup/terraform-v6/terraform.tfvars.example index 0012b28..0fd1d2f 100644 --- a/aws-integration-setup/terraform-v6/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/terraform.tfvars.example @@ -21,7 +21,8 @@ aws_region = "ap-southeast-2" # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" # KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# The key lives in Nullify's account, not yours. +# kms_key_arn = "arn:aws:kms:::key/" # Enable Kubernetes integration (set to true if you have EKS clusters) enable_kubernetes_integration = false diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index bd88919..51dcbf1 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -183,9 +183,11 @@ module "nullify_readonly_rbac" { } ``` +If any cluster in `cluster_arns` also runs the collector (a `k8s-resources` instance with `enable_collector = true`, an equivalent manifest, or the `nullify-k8s-collector` Helm chart), list its ARN in `collector_cluster_arns` too — the module fails at plan rather than double-registering it, in every `authorization` mode. + Outputs: `access_entry_arns`, `authorization`, `kubernetes_group_name`, `nullify_egress_cidrs`, `clusters_for_nullify`, `endpoint_allowlist`. -The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. +The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true. ## Multi-Cluster Support @@ -332,7 +334,7 @@ This is the same value as `collector.clusterName` in the `nullify-k8s-collector` `k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. -`enable_collector` and `enable_managed_scan_rbac` are mutually exclusive for one cluster, and the module fails at plan if both are true. The collector's upload registers the cluster as on-prem keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so enabling both lists the cluster twice with its pods and containers duplicated. To change mode, apply the new one and ask Nullify to remove the old cluster registration. +A cluster cannot run the collector and the managed scan at once: the collector's upload registers it as on-prem keyed on `cluster_name`, the managed scan registers the same cluster as its EKS ARN, and nothing joins the two, so running both lists the cluster twice with its pods and containers duplicated. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination: pass it the cluster ARNs where `enable_collector` is `true` and the module fails at plan for any of them, in every `authorization` mode (`admin_view_policy` creates no Kubernetes objects, so a rule inside `k8s-resources` could never see it). To change mode, apply the new one and ask Nullify to remove the old cluster registration. `collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. It is the same build as `k8s-collector-latest`, which the `nullify-k8s-collector` Helm chart deploys, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. @@ -376,9 +378,6 @@ module "nullify_k8s" { - The root and examples require AWS provider `>= 5.33`. Run `terraform init -upgrade` if your lock file pins an older 5.x. - `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`. The old object is in Nullify's bucket, which your role cannot read or delete: ask Nullify to remove the stale `default-name` cluster, and register the new name on the configure page before the next run. - `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true, with no opt-out. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page; ask Nullify for a key if you have none. -- `k8s-resources` no longer declares `allow_unencrypted_upload`. Passing it is now a plan error. It never worked against Nullify's bucket: leaving `kms_key_arn` empty also leaves the IAM role with no KMS statement, and the bucket's SSE-KMS default encryption makes `kms:GenerateDataKey` on Nullify's key a requirement of every upload, encrypted headers or not. The variable turned a self-explaining collector error into an S3 `AccessDenied`. -- `k8s-resources` now rejects `enable_collector` and `enable_managed_scan_rbac` together, which `examples/multi-cluster-complete` already rejected as `scan_mode = "both"`. The two paths register the same cluster under different identities and nothing joins them. -- `scan_mode = "both"` is no longer accepted in `examples/multi-cluster-complete`, and the module enforces the same rule. The collector registers a cluster as on-prem, keyed on `cluster_name`; the managed scan registers the same cluster as its EKS ARN. Nothing joins them, so "both" listed one cluster twice with its pods and containers duplicated. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. diff --git a/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example b/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example index 5ec8c91..d4a26e0 100644 --- a/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/basic/terraform.tfvars.example @@ -17,7 +17,8 @@ aws_region = "ap-southeast-2" # s3_bucket_name = "nullify-scan-results-acme-corp" # KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# The key lives in Nullify's account, not yours. +# kms_key_arn = "arn:aws:kms:::key/" # Tags to apply to all AWS resources tags = { diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index 63b05fe..39f1a70 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -141,11 +141,12 @@ module "eks_managed_scan_access_primary" { aws = aws.primary } - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = [var.eks_cluster_arns[0]] - nullify_region = var.nullify_region - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[0]] + nullify_region = var.nullify_region + collector_cluster_arns = local.collector ? [var.eks_cluster_arns[0]] : [] + tags = var.tags } module "eks_managed_scan_access_secondary" { @@ -155,9 +156,10 @@ module "eks_managed_scan_access_secondary" { aws = aws.secondary } - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = [var.eks_cluster_arns[1]] - nullify_region = var.nullify_region - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[1]] + nullify_region = var.nullify_region + collector_cluster_arns = local.collector ? [var.eks_cluster_arns[1]] : [] + tags = var.tags } diff --git a/aws-integration-setup/terraform/main.tf b/aws-integration-setup/terraform/main.tf index bbcd6a9..659c5af 100644 --- a/aws-integration-setup/terraform/main.tf +++ b/aws-integration-setup/terraform/main.tf @@ -1,3 +1,7 @@ +locals { + collector_cluster_arns = var.enable_kubernetes_integration ? var.eks_cluster_arns : [] +} + module "nullify_aws_integration" { source = "./modules/nullify-aws-integration" @@ -23,11 +27,12 @@ module "eks_managed_scan_access" { source = "./modules/eks-managed-scan-access" count = var.enable_managed_scan ? 1 : 0 - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = var.managed_scan_cluster_arns - nullify_region = var.nullify_region - authorization = var.managed_scan_authorization - kubernetes_group_name = var.managed_scan_kubernetes_group - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.managed_scan_cluster_arns + nullify_region = var.nullify_region + authorization = var.managed_scan_authorization + kubernetes_group_name = var.managed_scan_kubernetes_group + collector_cluster_arns = local.collector_cluster_arns + tags = var.tags } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 3459c84..64f9817 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -16,6 +16,8 @@ locals { account = split(":", arn)[4] } } + + collector_cluster_names = toset([for arn in var.collector_cluster_arns : split("/", arn)[1]]) } data "aws_region" "current" {} @@ -35,6 +37,11 @@ data "aws_eks_cluster" "this" { condition = each.value.account == local.principal_account error_message = "Cluster ${each.key} is in account ${each.value.account}, but principal_arn is in account ${local.principal_account}. Nullify assumes the integration role in the cluster's own account: deploy nullify-aws-integration there and pass that role." } + + precondition { + condition = !contains(local.collector_cluster_names, each.value.name) + error_message = "Cluster ${each.value.name} is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop it from collector_cluster_arns and cluster_arns here, or disable enable_collector on that cluster's k8s-resources instance." + } } } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index b9b9924..c6110c4 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -237,3 +237,30 @@ run "cross_account_clusters_are_rejected" { expect_failures = [data.aws_eks_cluster.this] } + +run "a_collector_cluster_is_rejected_even_under_admin_view_policy" { + # admin_view_policy creates no Kubernetes objects, so the k8s-resources + # ClusterRole's own precondition can never see this combination. The guard + # has to live here, on the access entry every authorization mode creates. + command = plan + + variables { + authorization = "admin_view_policy" + collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] + } + + expect_failures = [data.aws_eks_cluster.this, check.admin_view_policy_is_broader_than_required] +} + +run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { + command = plan + + variables { + collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/some-other-cluster"] + } + + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "A collector_cluster_arns entry for a different cluster must not block this one" + } +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf index d6edb9a..4e7e616 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -61,3 +61,9 @@ variable "tags" { description = "Tags to apply to the access entries" default = {} } + +variable "collector_cluster_arns" { + type = list(string) + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources" + default = [] +} diff --git a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf index 6676e38..a8d394b 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf @@ -32,13 +32,6 @@ locals { resource "kubernetes_cluster_role_v1" "nullify_readonly" { count = var.enable_managed_scan_rbac ? 1 : 0 - lifecycle { - precondition { - condition = !var.enable_collector - error_message = "enable_collector and enable_managed_scan_rbac cannot both be true for one cluster. The collector's upload is registered as an on-prem cluster keyed on cluster_name; the managed scan registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster appears twice in inventory with its pods and containers duplicated. Pick one mode: apply it, then ask Nullify to remove the old cluster registration." - } - } - metadata { name = var.managed_scan_cluster_role_name labels = { diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl index 9022f6d..85380c8 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -75,7 +75,14 @@ run "collector_requires_a_kms_key" { expect_failures = [kubernetes_cron_job_v1.k8s_collector] } -run "collector_and_managed_scan_rbac_cannot_both_run" { +run "staging_the_rbac_role_alongside_the_running_collector_plans_clean" { + # The duplicate-registration hazard is caused by the EKS access entry + # (enforced in eks-managed-scan-access, see its test suite), not by this + # ClusterRole: with no access entry mapping Nullify's principal into + # managed_scan_kubernetes_group, nothing joins the group and no managed + # scan runs. Staging the RBAC ahead of a cutover -- apply the ClusterRole + # now while the collector still runs, wire the access entry and disable + # the collector at the cutover -- must not be a plan error. command = plan variables { @@ -87,7 +94,10 @@ run "collector_and_managed_scan_rbac_cannot_both_run" { enable_managed_scan_rbac = true } - expect_failures = [kubernetes_cluster_role_v1.nullify_readonly] + assert { + condition = length(kubernetes_cron_job_v1.k8s_collector) == 1 && length(kubernetes_cluster_role_v1.nullify_readonly) == 1 + error_message = "Both the collector CronJob and the managed-scan ClusterRole must plan when no access entry exists yet" + } } run "the_kms_key_reaches_the_collector" { diff --git a/aws-integration-setup/terraform/terraform.tfvars.example b/aws-integration-setup/terraform/terraform.tfvars.example index ad64012..05970ae 100644 --- a/aws-integration-setup/terraform/terraform.tfvars.example +++ b/aws-integration-setup/terraform/terraform.tfvars.example @@ -21,7 +21,8 @@ aws_region = "ap-southeast-2" # nullify_s3_access_point_arn = "arn:aws:s3:ap-southeast-2:123456789012:accesspoint/acme-corp-k8s-collector" # KMS ARN from the Nullify configure page (optional). A key ARN or an alias ARN both work. -# kms_key_arn = "arn:aws:kms:ap-southeast-2:123456789012:key/12345678-1234-1234-1234-123456789012" +# The key lives in Nullify's account, not yours. +# kms_key_arn = "arn:aws:kms:::key/" # Enable Kubernetes integration (set to true if you have EKS clusters) enable_kubernetes_integration = false From c20f94853e6519efac012c0ceb71e902528b818c Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 11:31:04 +1000 Subject: [PATCH 23/36] fix(terraform): compare full cluster ARNs in the collector duplicate guard and root-derive it correctly The guard added in the previous round compared bare cluster names, so two clusters sharing a name in different regions/accounts collided with no override in the v6 root. It also derived collector_cluster_arns from the IRSA trust list, which is a prerequisite for a collector, not evidence one is deployed. - Compare full cluster ARNs instead of names in eks-managed-scan-access (both trees); add a same-name-different-region test row in both. - Add a root collector_cluster_arns override variable (both trees) so root users can correct the IRSA-trust-list proxy and decouple a collector-to-managed-scan cutover from revoking IRSA trust. - Reword the guard's error message to cover the Helm/manifest paths its own variable description already names. - Add ARN-shape validation to collector_cluster_arns, matching cluster_arns. - Remove the dead collector_cluster_arns wiring in both multi-cluster examples: scan_mode's mutual exclusivity makes it always []. - Fix terraform-v6/README.md still documenting the enable_collector + enable_managed_scan_rbac rejection this PR's base commit removed; add the missing collector_cluster_arns doc and Kubernetes-provider-removal upgrade note to match the v5 tree. Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform-v6/README.md | 6 ++++- .../examples/multi-cluster-complete/main.tf | 11 +++++----- aws-integration-setup/terraform-v6/main.tf | 2 +- .../modules/eks-managed-scan-access/main.tf | 6 ++--- .../tests/eks_managed_scan_access.tftest.hcl | 18 +++++++++++++++ .../eks-managed-scan-access/variables.tf | 7 +++++- .../terraform-v6/variables.tf | 6 +++++ aws-integration-setup/terraform/README.md | 3 ++- .../examples/multi-cluster-complete/main.tf | 22 +++++++++---------- aws-integration-setup/terraform/main.tf | 2 +- .../modules/eks-managed-scan-access/main.tf | 6 ++--- .../tests/eks_managed_scan_access.tftest.hcl | 18 +++++++++++++++ .../eks-managed-scan-access/variables.tf | 7 +++++- aws-integration-setup/terraform/variables.tf | 6 +++++ 14 files changed, 90 insertions(+), 30 deletions(-) diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index 49d90d0..f867f19 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -108,13 +108,16 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page; see "KMS" in `../terraform/README.md`. Optional for this module; required by `k8s-resources` whenever `enable_collector` is true - `enable_kubernetes_integration`: Set to `true` for EKS integration - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with, in any region +- `collector_cluster_arns`: Override for which of those clusters `eks_managed_scan_access` treats as already running the collector; defaults to `eks_cluster_arns` when `enable_kubernetes_integration` is `true`. Set it explicitly when trust is granted ahead of the collector's actual deployment, or to decouple a collector-to-managed-scan cutover from revoking IRSA trust - `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: managed EKS scan - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) - `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) - `tags`: Resource tags - `cronjob_schedule`: Deprecated and unused here; set the schedule on `k8s-resources` -`k8s-resources` requires `cluster_name` and `kms_key_arn` whenever `enable_collector` is true, with no unencrypted opt-out, and rejects `enable_collector` together with `enable_managed_scan_rbac`. `cluster_name` must match your actual cluster name exactly, must match a cluster registered on the Nullify configure page, and must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, and Nullify joins that upload on the name. See "Kubernetes Resources" and "KMS" in `../terraform/README.md`. +`k8s-resources` requires `cluster_name` and `kms_key_arn` whenever `enable_collector` is true, with no unencrypted opt-out. `cluster_name` must match your actual cluster name exactly, must match a cluster registered on the Nullify configure page, and must differ per cluster: the collector uploads to `/k8s-collector/-data.json`, and Nullify joins that upload on the name. See "Kubernetes Resources" and "KMS" in `../terraform/README.md`. + +A cluster cannot run the collector and the managed scan at once: the two paths register it under different identities, duplicating its inventory. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination -- `enable_collector` and `enable_managed_scan_rbac` may be set together on one `k8s-resources` instance to stage a cutover, since the ClusterRole it applies is inert until an access entry uses it. See "Kubernetes Resources" in `../terraform/README.md`. `k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. @@ -124,6 +127,7 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true, with no opt-out. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page; ask Nullify for a key if you have none. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The KMS policy grants `key/*` in the account and region of `kms_key_arn` whenever that account is not your own, for alias ARNs and key ARNs alike. An ARN in your own account grants only that key. +- The root no longer declares a `kubernetes` provider or `hashicorp/kubernetes` requirement: this tree's root never applied Kubernetes objects, so remove any provider configuration you added to satisfy it. ## Validation diff --git a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf index f3e77b7..738dc49 100644 --- a/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform-v6/examples/multi-cluster-complete/main.tf @@ -120,10 +120,9 @@ module "eks_managed_scan_access" { source = "../../modules/eks-managed-scan-access" count = local.managed ? 1 : 0 - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = var.eks_cluster_arns - nullify_region = var.nullify_region - collector_cluster_arns = local.collector ? var.eks_cluster_arns : [] - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = var.eks_cluster_arns + nullify_region = var.nullify_region + tags = var.tags } diff --git a/aws-integration-setup/terraform-v6/main.tf b/aws-integration-setup/terraform-v6/main.tf index 777323b..f7da46c 100644 --- a/aws-integration-setup/terraform-v6/main.tf +++ b/aws-integration-setup/terraform-v6/main.tf @@ -1,5 +1,5 @@ locals { - collector_cluster_arns = var.enable_kubernetes_integration ? var.eks_cluster_arns : [] + collector_cluster_arns = var.collector_cluster_arns != null ? var.collector_cluster_arns : (var.enable_kubernetes_integration ? var.eks_cluster_arns : []) } module "nullify_aws_integration" { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index ee80f47..73344cd 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -17,7 +17,7 @@ locals { } } - collector_cluster_names = toset([for arn in var.collector_cluster_arns : split("/", arn)[1]]) + collector_cluster_arns = toset(var.collector_cluster_arns) } data "aws_eks_cluster" "this" { @@ -33,8 +33,8 @@ data "aws_eks_cluster" "this" { } precondition { - condition = !contains(local.collector_cluster_names, each.value.name) - error_message = "Cluster ${each.value.name} is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop it from collector_cluster_arns and cluster_arns here, or disable enable_collector on that cluster's k8s-resources instance." + condition = !contains(local.collector_cluster_arns, each.key) + error_message = "Cluster ${each.value.name} (${each.key}) is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop its ARN from collector_cluster_arns and cluster_arns here, or stop treating it as a collector target (disable enable_collector on its k8s-resources instance, or remove it from the Helm chart or manifest that registers it)." } } } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index a468379..351d2ce 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -235,3 +235,21 @@ run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { error_message = "A collector_cluster_arns entry for a different cluster must not block this one" } } + +run "a_same_named_cluster_in_another_region_is_unaffected" { + # The guard compares full ARNs, not bare cluster names. This tree supports + # collector_cluster_arns spanning multiple regions in one instance (see the + # README's multi-region example), so a cluster called "prod" in eu-west-1 + # (cluster_arns) must not collide with an unrelated cluster also called + # "prod" in us-east-1 (collector_cluster_arns). + command = plan + + variables { + collector_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + } + + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "A same-named cluster in a different region must not be mistaken for the same cluster" + } +} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf index 83ee4cd..d7f566d 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -64,6 +64,11 @@ variable "tags" { variable "collector_cluster_arns" { type = list(string) - description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources" + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match -- this tree supports collector_cluster_arns spanning multiple regions in one instance, same as cluster_arns" default = [] + + validation { + condition = alltrue([for arn in var.collector_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + } } diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index f9999fa..031df2d 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -108,6 +108,12 @@ variable "managed_scan_cluster_arns" { default = [] } +variable "collector_cluster_arns" { + type = list(string) + description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" + default = null +} + variable "managed_scan_authorization" { type = string description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 51dcbf1..4e226c9 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -187,7 +187,7 @@ If any cluster in `cluster_arns` also runs the collector (a `k8s-resources` inst Outputs: `access_entry_arns`, `authorization`, `kubernetes_group_name`, `nullify_egress_cidrs`, `clusters_for_nullify`, `endpoint_allowlist`. -The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true. +The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true — that is only a proxy (granting the collector's IRSA trust is not the same as deploying it), so override it with the root's own `collector_cluster_arns` variable when trust is granted ahead of deployment, or when migrating a cluster from collector to managed scan and you want to stop the guard blocking it before you also remove the cluster from `eks_cluster_arns` (which revokes its IRSA trust). ## Multi-Cluster Support @@ -257,6 +257,7 @@ terraform init && terraform apply - `enable_kubernetes_integration`: Set to `true` to trust the in-cluster collector's service account - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with - `eks_oidc_issuer_urls`: OIDC issuer per cluster, only needed for clusters outside `aws_region` +- `collector_cluster_arns`: Override for which clusters the managed-scan duplicate guard treats as collector targets; defaults to `eks_cluster_arns` when `enable_kubernetes_integration` is `true` - `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: see [Managed EKS scan](#managed-eks-scan) ## Optional Variables diff --git a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf index 39f1a70..63b05fe 100644 --- a/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf +++ b/aws-integration-setup/terraform/examples/multi-cluster-complete/main.tf @@ -141,12 +141,11 @@ module "eks_managed_scan_access_primary" { aws = aws.primary } - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = [var.eks_cluster_arns[0]] - nullify_region = var.nullify_region - collector_cluster_arns = local.collector ? [var.eks_cluster_arns[0]] : [] - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[0]] + nullify_region = var.nullify_region + tags = var.tags } module "eks_managed_scan_access_secondary" { @@ -156,10 +155,9 @@ module "eks_managed_scan_access_secondary" { aws = aws.secondary } - principal_arn = module.nullify_aws_integration.role_arn - principal_unique_id = module.nullify_aws_integration.role_unique_id - cluster_arns = [var.eks_cluster_arns[1]] - nullify_region = var.nullify_region - collector_cluster_arns = local.collector ? [var.eks_cluster_arns[1]] : [] - tags = var.tags + principal_arn = module.nullify_aws_integration.role_arn + principal_unique_id = module.nullify_aws_integration.role_unique_id + cluster_arns = [var.eks_cluster_arns[1]] + nullify_region = var.nullify_region + tags = var.tags } diff --git a/aws-integration-setup/terraform/main.tf b/aws-integration-setup/terraform/main.tf index 659c5af..12535da 100644 --- a/aws-integration-setup/terraform/main.tf +++ b/aws-integration-setup/terraform/main.tf @@ -1,5 +1,5 @@ locals { - collector_cluster_arns = var.enable_kubernetes_integration ? var.eks_cluster_arns : [] + collector_cluster_arns = var.collector_cluster_arns != null ? var.collector_cluster_arns : (var.enable_kubernetes_integration ? var.eks_cluster_arns : []) } module "nullify_aws_integration" { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 64f9817..5371d0c 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -17,7 +17,7 @@ locals { } } - collector_cluster_names = toset([for arn in var.collector_cluster_arns : split("/", arn)[1]]) + collector_cluster_arns = toset(var.collector_cluster_arns) } data "aws_region" "current" {} @@ -39,8 +39,8 @@ data "aws_eks_cluster" "this" { } precondition { - condition = !contains(local.collector_cluster_names, each.value.name) - error_message = "Cluster ${each.value.name} is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop it from collector_cluster_arns and cluster_arns here, or disable enable_collector on that cluster's k8s-resources instance." + condition = !contains(local.collector_cluster_arns, each.key) + error_message = "Cluster ${each.value.name} (${each.key}) is also in collector_cluster_arns. The collector's upload registers it as an on-prem cluster keyed on cluster_name; this module's access entry registers the same cluster as its EKS ARN. Nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Pick one mode per cluster: drop its ARN from collector_cluster_arns and cluster_arns here, or stop treating it as a collector target (disable enable_collector on its k8s-resources instance, or remove it from the Helm chart or manifest that registers it)." } } } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index c6110c4..2cc657a 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -264,3 +264,21 @@ run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { error_message = "A collector_cluster_arns entry for a different cluster must not block this one" } } + +run "a_same_named_cluster_in_another_region_is_unaffected" { + # The guard compares full ARNs, not bare cluster names: cluster_arns names + # a cluster called "prod" in eu-west-1, and this collector_cluster_arns + # entry names a different cluster that happens to share the name "prod" in + # us-east-1. Comparing names alone would collide the two and reject the + # entry, even though nothing is duplicated. + command = plan + + variables { + collector_cluster_arns = ["arn:aws:eks:us-east-1:123456789012:cluster/prod"] + } + + assert { + condition = length(aws_eks_access_entry.nullify) == 1 + error_message = "A same-named cluster in a different region must not be mistaken for the same cluster" + } +} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf index 4e7e616..3927989 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -64,6 +64,11 @@ variable "tags" { variable "collector_cluster_arns" { type = list(string) - description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources" + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match" default = [] + + validation { + condition = alltrue([for arn in var.collector_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + } } diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index 630bff9..8dc3fa3 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -114,6 +114,12 @@ variable "managed_scan_cluster_arns" { default = [] } +variable "collector_cluster_arns" { + type = list(string) + description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" + default = null +} + variable "managed_scan_authorization" { type = string description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" From c74571bd1cf43ef92018c8d802de7db7939327d7 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 11:42:50 +1000 Subject: [PATCH 24/36] docs(terraform): tell collector-cutover instructions to stop the collector first collector_cluster_arns docs told users to override the guard during a collector-to-managed-scan cutover but never said to disable the collector first, so following them literally leaves the collector uploading while the cluster also gets managed-scan access -- the duplicate registration the guard exists to prevent. Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform-v6/README.md | 2 +- aws-integration-setup/terraform-v6/variables.tf | 2 +- aws-integration-setup/terraform/README.md | 2 +- aws-integration-setup/terraform/variables.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index f867f19..9ac3524 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -108,7 +108,7 @@ The default `authorization = "rbac"` needs the list-only `nullify-readonly` Clus - `kms_key_arn`: KMS key ARN or alias ARN from the Nullify configure page; see "KMS" in `../terraform/README.md`. Optional for this module; required by `k8s-resources` whenever `enable_collector` is true - `enable_kubernetes_integration`: Set to `true` for EKS integration - `eks_cluster_arns`: List of EKS cluster ARNs to integrate with, in any region -- `collector_cluster_arns`: Override for which of those clusters `eks_managed_scan_access` treats as already running the collector; defaults to `eks_cluster_arns` when `enable_kubernetes_integration` is `true`. Set it explicitly when trust is granted ahead of the collector's actual deployment, or to decouple a collector-to-managed-scan cutover from revoking IRSA trust +- `collector_cluster_arns`: Override for which of those clusters `eks_managed_scan_access` treats as already running the collector; defaults to `eks_cluster_arns` when `enable_kubernetes_integration` is `true`. Set it explicitly when trust is granted ahead of the collector's actual deployment, or to decouple a collector-to-managed-scan cutover from revoking IRSA trust -- disable or remove the collector on that cluster first, before setting the override - `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization`, `managed_scan_kubernetes_group`: managed EKS scan - `kubernetes_namespace`: Kubernetes namespace name (default: nullify) - `service_account_name`: Collector service account name (default: nullify-k8s-collector-sa) diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index 031df2d..c0210a5 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -110,7 +110,7 @@ variable "managed_scan_cluster_arns" { variable "collector_cluster_arns" { type = list(string) - description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" + description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: disable or remove the collector on that cluster first, then set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" default = null } diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 4e226c9..617a1ee 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -187,7 +187,7 @@ If any cluster in `cluster_arns` also runs the collector (a `k8s-resources` inst Outputs: `access_entry_arns`, `authorization`, `kubernetes_group_name`, `nullify_egress_cidrs`, `clusters_for_nullify`, `endpoint_allowlist`. -The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true — that is only a proxy (granting the collector's IRSA trust is not the same as deploying it), so override it with the root's own `collector_cluster_arns` variable when trust is granted ahead of deployment, or when migrating a cluster from collector to managed scan and you want to stop the guard blocking it before you also remove the cluster from `eks_cluster_arns` (which revokes its IRSA trust). +The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true — that is only a proxy (granting the collector's IRSA trust is not the same as deploying it), so override it with the root's own `collector_cluster_arns` variable when trust is granted ahead of deployment, or when migrating a cluster from collector to managed scan: disable or remove the collector on that cluster first, then set the override so the guard stops blocking it before you also remove the cluster from `eks_cluster_arns` (which revokes its IRSA trust). ## Multi-Cluster Support diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index 8dc3fa3..83bbc15 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -116,7 +116,7 @@ variable "managed_scan_cluster_arns" { variable "collector_cluster_arns" { type = list(string) - description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" + description = "Override for which clusters eks_managed_scan_access treats as already running the collector, for its duplicate-registration guard. Defaults to eks_cluster_arns when enable_kubernetes_integration is true, since granting the collector's IRSA trust is the closest available signal that a collector is deployed there -- but trust and deployment can diverge: a staged rollout may grant trust ahead of deploying k8s-resources' CronJob (or the Helm chart, or an equivalent manifest) to those clusters, incorrectly blocking their managed scan. It also decouples a collector-to-managed-scan cutover from revoking IRSA trust: disable or remove the collector on that cluster first, then set this to eks_cluster_arns minus the cluster being migrated so the guard stops blocking it, and drop the cluster from eks_cluster_arns itself only once the cutover is confirmed. null keeps the default derivation from eks_cluster_arns" default = null } From cabde870027e4aaf93f64294a18d05c7fb0374c7 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 14:17:11 +1000 Subject: [PATCH 25/36] fix(aws): drop collector ClusterRole grants the collector never lists whiskeylotus's review on #58 (W58-2) flagged that the collector ClusterRole grants 12 kinds main's collect.go never lists: jobs is kept since the monorepo's fix/k8s-deployment-matching-review stack (#12807, which #12830 sits on) starts collecting Jobs owned by a CronJob; cronjobs, horizontalpodautoscalers, poddisruptionbudgets, ingressclasses, storageclasses, csidrivers, runtimeclasses, priorityclasses, flowschemas, certificatesigningrequests and customresourcedefinitions are dropped -- no open collector change reads any of them. Verified against context/internal/cloudintegrations/k8sintegration/ collect/collect.go on both the monorepo's main and fix/k8s-deployment-matching-review branches. Co-Authored-By: Claude Opus 5 (1M context) --- .../templates/clusterrole.yaml | 37 +++++-------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index db9f326..08a4226 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -6,7 +6,14 @@ metadata: labels: {{- include "k8s-collector.labels" . | nindent 4 }} rules: - # Access for collecting cluster information + # Kinds the collector actually lists (context/internal/cloudintegrations/ + # k8sintegration/collect/collect.go in the platform monorepo), plus "jobs": + # the fix/k8s-deployment-matching-review stack (monorepo #12807, which + # #12830 sits on) starts collecting Jobs owned by a CronJob to resolve + # scheduled-run pod ownership, so that grant is added ahead of the + # collector code landing rather than dropped and re-added. No other kind + # here is collected today; each one removed a grant this role held with no + # reader. - apiGroups: [""] resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "secrets", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] verbs: ["list"] @@ -14,16 +21,10 @@ rules: resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] verbs: ["list"] - apiGroups: ["batch"] - resources: ["jobs", "cronjobs"] - verbs: ["list"] - - apiGroups: ["autoscaling"] - resources: ["horizontalpodautoscalers"] - verbs: ["list"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] + resources: ["jobs"] verbs: ["list"] - apiGroups: ["networking.k8s.io"] - resources: ["ingresses", "ingressclasses", "networkpolicies"] + resources: ["ingresses", "networkpolicies"] verbs: ["list"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] @@ -31,27 +32,9 @@ rules: - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] verbs: ["list"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csidrivers"] - verbs: ["list"] - - apiGroups: ["node.k8s.io"] - resources: ["runtimeclasses"] - verbs: ["list"] - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["list"] - - apiGroups: ["flowcontrol.apiserver.k8s.io"] - resources: ["flowschemas"] - verbs: ["list"] - - apiGroups: ["certificates.k8s.io"] - resources: ["certificatesigningrequests"] - verbs: ["list"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] verbs: ["list"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["list"] - nonResourceURLs: ["/version"] verbs: ["get"] {{- end }} \ No newline at end of file From 52c291cbe13c1998a183aa3cdc87b973213c7c86 Mon Sep 17 00:00:00 2001 From: Tim Thacker <123459940+tim-thacker-nullify@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:09:26 +1000 Subject: [PATCH 26/36] fix(aws): drop collector ClusterRole grants the collector never lists (#99) whiskeylotus's review on #58 (W58-2) flagged that the collector ClusterRole grants 12 kinds main's collect.go never lists: jobs is kept since the monorepo's fix/k8s-deployment-matching-review stack (#12807, which #12830 sits on) starts collecting Jobs owned by a CronJob; cronjobs, horizontalpodautoscalers, poddisruptionbudgets, ingressclasses, storageclasses, csidrivers, runtimeclasses, priorityclasses, flowschemas, certificatesigningrequests and customresourcedefinitions are dropped -- no open collector change reads any of them. Verified against context/internal/cloudintegrations/k8sintegration/ collect/collect.go on both the monorepo's main and fix/k8s-deployment-matching-review branches. Co-authored-by: Claude Opus 5 (1M context) --- .../templates/clusterrole.yaml | 37 +++++-------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index db9f326..08a4226 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -6,7 +6,14 @@ metadata: labels: {{- include "k8s-collector.labels" . | nindent 4 }} rules: - # Access for collecting cluster information + # Kinds the collector actually lists (context/internal/cloudintegrations/ + # k8sintegration/collect/collect.go in the platform monorepo), plus "jobs": + # the fix/k8s-deployment-matching-review stack (monorepo #12807, which + # #12830 sits on) starts collecting Jobs owned by a CronJob to resolve + # scheduled-run pod ownership, so that grant is added ahead of the + # collector code landing rather than dropped and re-added. No other kind + # here is collected today; each one removed a grant this role held with no + # reader. - apiGroups: [""] resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "secrets", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] verbs: ["list"] @@ -14,16 +21,10 @@ rules: resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] verbs: ["list"] - apiGroups: ["batch"] - resources: ["jobs", "cronjobs"] - verbs: ["list"] - - apiGroups: ["autoscaling"] - resources: ["horizontalpodautoscalers"] - verbs: ["list"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] + resources: ["jobs"] verbs: ["list"] - apiGroups: ["networking.k8s.io"] - resources: ["ingresses", "ingressclasses", "networkpolicies"] + resources: ["ingresses", "networkpolicies"] verbs: ["list"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] @@ -31,27 +32,9 @@ rules: - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] verbs: ["list"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csidrivers"] - verbs: ["list"] - - apiGroups: ["node.k8s.io"] - resources: ["runtimeclasses"] - verbs: ["list"] - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["list"] - - apiGroups: ["flowcontrol.apiserver.k8s.io"] - resources: ["flowschemas"] - verbs: ["list"] - - apiGroups: ["certificates.k8s.io"] - resources: ["certificatesigningrequests"] - verbs: ["list"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] verbs: ["list"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["list"] - nonResourceURLs: ["/version"] verbs: ["get"] {{- end }} \ No newline at end of file From 1d5cd1749e17903049c09439b3984922a3c2e856 Mon Sep 17 00:00:00 2001 From: Tim Thacker <123459940+tim-thacker-nullify@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:09:55 +1000 Subject: [PATCH 27/36] fix(terraform): reconcile K8s scan-mode RBAC and address #81 review feedback (#98) * fix(terraform): reconcile K8s scan-mode RBAC and fix review nits from #81 Applies dnjg's review feedback from ncc-cloud-connector #81 to #78's collapsed branch. - Align the in-cluster collector's Kubernetes RBAC (Helm chart and the Terraform k8s-resources module, v5 and v6) with the managed scan's ClusterRole and with collect.go, so switching scan mode no longer silently changes which kinds get scanned (N78-2, N78-3). - Document why both scan modes list Secrets without reading values: the managed scan reads Kubernetes' server-side Table representation, the in-cluster collector redacts before upload (N78-2). - Fix the documented terraform import address for the EKS access entry, covering the root module's count index, a direct module call and the multi-cluster example (N78-4). - Cite the platform monorepo's iac/network/vpc module as the source of the hardcoded Nullify egress IPs, in both the module comment and the README (N78-5). Serving the list from an API is left as a product follow-up, not built here. - Reword the ARN partition validation errors to say only the commercial aws partition is supported, instead of reading like a malformed-ARN typo (N78-6). - Add terraform test assertions on the KMS, S3 access point and readonly aws_iam_policy_document statement blocks: the existing mock_provider fakes every document's computed json, so no test previously asserted an actual IAM action or resource (N78-8). Co-Authored-By: Claude Opus 5 (1M context) * ci(terraform): add an IAM policy drift check across CFN and Terraform whiskeylotus's review on ncc #58 (W58-3) asks for a job that verifies the CloudFormation template and both Terraform trees keep granting the same Allow/Deny action sets for the customer readonly role -- nothing currently checks that the three independently maintained templates stay identical. Adds scripts/check_iam_policy_drift.py, comparing the CloudFormation template's ReadOnlyAccessPolicy(+2)/DenyActionsPolicy statements against readonly_policy_part1/part2/deny_actions_policy in both aws-integration-setup/terraform and terraform-v6, and wires it as a new job beside the existing terraform-validate matrix. Verified locally: passes against the current templates (437 Allow, 45 Deny actions matching on both trees) and correctly fails when an action is injected into one side only. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/terraform-validate.yml | 13 +++ .../modules/eks-managed-scan-access/main.tf | 7 ++ .../eks-managed-scan-access/variables.tf | 6 +- .../modules/k8s-resources/main.tf | 49 +++++----- .../tests/nullify_aws_integration.tftest.hcl | 65 ++++++++++++++ .../terraform-v6/variables.tf | 4 +- aws-integration-setup/terraform/README.md | 12 ++- .../modules/eks-managed-scan-access/main.tf | 7 ++ .../eks-managed-scan-access/variables.tf | 6 +- .../terraform/modules/k8s-resources/main.tf | 49 +++++----- .../tests/nullify_aws_integration.tftest.hcl | 65 ++++++++++++++ aws-integration-setup/terraform/variables.tf | 4 +- .../templates/clusterrole.yaml | 16 ++-- scripts/check_iam_policy_drift.py | 89 +++++++++++++++++++ 14 files changed, 317 insertions(+), 75 deletions(-) create mode 100755 scripts/check_iam_policy_drift.py diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml index 882d52c..42d3acf 100644 --- a/.github/workflows/terraform-validate.yml +++ b/.github/workflows/terraform-validate.yml @@ -130,6 +130,19 @@ jobs: working-directory: ${{ matrix.directory }} run: terraform test + # The CloudFormation template and both Terraform trees grant the same + # customer role from three independently maintained templates. Nothing + # else checks that their Allow/Deny action sets stay identical (ncc #58, + # W58-3). + iam-policy-drift: + name: IAM policy drift (CloudFormation vs Terraform) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: compare Allow/Deny action sets + run: python3 scripts/check_iam_policy_drift.py + shellcheck: runs-on: ubuntu-latest steps: diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index 73344cd..bd32446 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -1,4 +1,11 @@ locals { + # These are the NAT gateway Elastic IPs Nullify's scan compute egresses + # through in each region, sourced from the platform monorepo's + # `iac/network/vpc` module (`aws_eip.nat`, imported per prod workspace in + # `imports.tf`) -- not allocated or owned by this repo. They change only if + # Nullify recreates a NAT gateway; if a cluster starts failing the + # `nullify_endpoint_allowlist` check with no config change on your side, + # re-fetch this list rather than assuming a local misconfiguration. nullify_egress_cidrs_by_region = { "ap-southeast-2" = ["13.55.32.104/32", "3.105.146.106/32", "13.211.99.100/32"] "eu-central-1" = ["18.198.60.231/32", "18.157.227.250/32", "18.185.152.197/32"] diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf index d7f566d..8d156d4 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -4,7 +4,7 @@ variable "principal_arn" { validation { condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.principal_arn)) - error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/" + error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } @@ -20,7 +20,7 @@ variable "cluster_arns" { validation { condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." } } @@ -69,6 +69,6 @@ variable "collector_cluster_arns" { validation { condition = alltrue([for arn in var.collector_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index efa8271..4203295 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -48,16 +48,22 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } } + # Kinds the collector lists, kept identical to the managed-scan ClusterRole + # (managed_scan.tf) and to collect.go in the platform monorepo + # (context/internal/cloudintegrations/k8sintegration/collect). rule { api_groups = [""] resources = [ "pods", "services", - "endpoints", "namespaces", "nodes", "persistentvolumes", "persistentvolumeclaims", + "configmaps", + "secrets", + "resourcequotas", + "limitranges", "serviceaccounts", ] verbs = ["get", "list"] @@ -69,6 +75,12 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["get", "list"] } + rule { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + verbs = ["get", "list"] + } + rule { api_groups = ["apps"] resources = [ @@ -92,33 +104,14 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } rule { - api_groups = ["storage.k8s.io"] - resources = ["storageclasses"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["batch"] - resources = ["jobs", "cronjobs"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["apiextensions.k8s.io"] - resources = ["customresourcedefinitions"] - verbs = ["get", "list"] + api_groups = ["admissionregistration.k8s.io"] + resources = [ + "validatingwebhookconfigurations", + "mutatingwebhookconfigurations", + "validatingadmissionpolicies", + "validatingadmissionpolicybindings", + ] + verbs = ["get", "list"] } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index bb192e4..f0eede8 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -143,3 +143,68 @@ run "kubernetes_integration_needs_clusters" { expect_failures = [data.aws_iam_policy_document.assume_role_policy] } + +# The mock above fakes every aws_iam_policy_document's computed `json`, so a +# passing test only proves the module plans, not that a policy grants any +# action. `statement` blocks are the data source's own configuration -- they +# are plan-time known even when the resulting document is mocked -- so assert +# on those instead. Only the KMS, S3 access point and readonly policies are +# covered here: the trust and deny-list policies aren't in W58/N78's scope. +run "kms_policy_grants_the_expected_actions_and_resources" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } + + assert { + condition = toset(data.aws_iam_policy_document.kms_access_policy[0].statement[0].actions) == toset([ + "kms:DescribeKey", + "kms:GenerateDataKey", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + ]) + error_message = "The KMS policy must grant exactly these actions, no more and no less" + } + + assert { + condition = tolist(data.aws_iam_policy_document.kms_access_policy[0].statement[0].resources) == tolist(output.kms_policy_resources) + error_message = "The KMS policy statement's resources must be exactly the derived kms_policy_resources output" + } +} + +run "s3_access_point_policy_grants_put_only_under_the_collector_prefix" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:eu-central-1:111122223333:accesspoint/nullify-uploads" + } + + assert { + condition = toset(data.aws_iam_policy_document.s3_access_policy[0].statement[0].actions) == toset([ + "s3:PutObject", + "s3:PutObjectAcl", + ]) + error_message = "The access point statement must grant only PutObject and PutObjectAcl" + } + + assert { + condition = tolist(data.aws_iam_policy_document.s3_access_policy[0].statement[0].resources) == tolist(["arn:aws:s3:eu-central-1:111122223333:accesspoint/nullify-uploads/object/k8s-collector/*"]) + error_message = "The access point statement must be scoped to the k8s-collector object prefix, not the whole access point" + } +} + +run "readonly_policy_grants_a_read_only_action" { + command = plan + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "access-analyzer:GetFinding") + error_message = "The readonly policy's first statement must contain the expected access-analyzer read action" + } + + assert { + condition = data.aws_iam_policy_document.readonly_policy_part1.statement[0].effect == "Allow" + error_message = "The readonly policy statement must Allow, not Deny" + } +} diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index c0210a5..8b12167 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -24,7 +24,7 @@ variable "nullify_role_arn" { validation { condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.nullify_role_arn)) - error_message = "Must be a valid ARN for an IAM role in the format arn:aws:iam:::role/" + error_message = "Must be a valid ARN for an IAM role in the format arn:aws:iam:::role/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } @@ -41,7 +41,7 @@ variable "eks_cluster_arns" { validation { condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 617a1ee..105f743 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -86,6 +86,7 @@ terraform/ | Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | | AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | | Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | A list-only `nullify-readonly` ClusterRole and ClusterRoleBinding, applied by `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true` | +| Kinds read | Same kind list as the managed scan (both derive from the platform's collector) | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | | Upgrades | You update the image | None | Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. @@ -107,7 +108,9 @@ The access entry carries the Kubernetes group `nullify-readonly`, bound to the C | rbac.authorization.k8s.io | roles, rolebindings, clusterroles, clusterrolebindings | | admissionregistration.k8s.io | validatingwebhookconfigurations, mutatingwebhookconfigurations, validatingadmissionpolicies, validatingadmissionpolicybindings | -Plus `get` on the non-resource URL `/version`. Kubernetes has no metadata-only permission for Secrets: `list secrets` permits reading values, whichever mode you pick. +Plus `get` on the non-resource URL `/version`. + +Both modes list Secrets, but neither reads values. The managed scan requests Kubernetes' server-side [Table representation](https://kubernetes.io/docs/reference/using-api/api-concepts/#alternate-representations-of-resources) for Secrets, so only metadata (name, type, key count) crosses the API server boundary; values never leave the cluster. The in-cluster collector fetches full Secret objects but redacts every value before it uploads collected data. The `list` verb technically permits reading values in either mode -- Kubernetes has no metadata-only RBAC verb for Secrets -- but Nullify's collection code never exercises that. ### `authorization = "admin_view_policy"` (opt-in) @@ -121,7 +124,10 @@ Associates `AmazonEKSAdminViewPolicy` at cluster scope, so no Kubernetes objects - **One region per module instance (AWS provider v5).** Instantiate `eks-managed-scan-access` once per region with `providers = { aws = aws. }`, as `examples/multi-cluster-complete` does. `../terraform-v6/` handles every region in one instance. - **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`, plus `eks:AssociateAccessPolicy`, `eks:DisassociateAccessPolicy` and `eks:ListAssociatedAccessPolicies` for `admin_view_policy`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the same ClusterRole itself. - **Pass `principal_unique_id`.** EKS ties an access entry to the role's ID, so a recreated role with the same ARN is silently not authorized. With `principal_unique_id = module.nullify_aws_integration.role_unique_id`, the entries are replaced with the role. -- **Existing access entry.** If this role is already mapped on a cluster, import it: `terraform import 'module.eks_managed_scan_access.aws_eks_access_entry.nullify[""]' :`. +- **Existing access entry.** If this role is already mapped on a cluster, import it. The address depends on how you call this module: + - Through the root module (`enable_managed_scan = true`, which wraps `eks_managed_scan_access` in `count`): `terraform import 'module.eks_managed_scan_access[0].aws_eks_access_entry.nullify[""]' :`. + - Calling `modules/eks-managed-scan-access` directly with no `count`/`for_each` on the module block: `terraform import 'module.eks_managed_scan_access.aws_eks_access_entry.nullify[""]' :`. + - From `examples/multi-cluster-complete`, which names two instances: `terraform import 'module.eks_managed_scan_access_primary[0].aws_eks_access_entry.nullify[""]' :` (and `_secondary[0]` for the second cluster). ### Allowing Nullify to reach the EKS API endpoint @@ -133,6 +139,8 @@ The managed scan connects to the cluster's public endpoint from Nullify's egress | eu-central-1 | 18.198.60.231, 18.157.227.250, 18.185.152.197 | | us-east-2 | 52.15.146.50, 16.58.40.80, 3.133.15.210 | +These are the NAT gateway Elastic IPs Nullify's scan compute egresses through, owned by the platform's `iac/network/vpc` module, not by this repo. They rotate only if Nullify recreates a NAT gateway in that region; if a cluster starts failing the check below with no change on your side, contact Nullify for the current list rather than assuming stale local config. Serving this list from an API instead of hardcoding it here is an open follow-up, not yet built. + These modules never change `publicAccessCidrs`: the cluster belongs to your own IaC, and the API replaces the whole list. Instead: - A `check` block warns on every plan when a cluster's public endpoint is disabled or misses a Nullify IP. It matches exact CIDRs, so a wider range that already covers the IPs still warns. diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 5371d0c..00df32f 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -1,4 +1,11 @@ locals { + # These are the NAT gateway Elastic IPs Nullify's scan compute egresses + # through in each region, sourced from the platform monorepo's + # `iac/network/vpc` module (`aws_eip.nat`, imported per prod workspace in + # `imports.tf`) -- not allocated or owned by this repo. They change only if + # Nullify recreates a NAT gateway; if a cluster starts failing the + # `nullify_endpoint_allowlist` check with no config change on your side, + # re-fetch this list rather than assuming a local misconfiguration. nullify_egress_cidrs_by_region = { "ap-southeast-2" = ["13.55.32.104/32", "3.105.146.106/32", "13.211.99.100/32"] "eu-central-1" = ["18.198.60.231/32", "18.157.227.250/32", "18.185.152.197/32"] diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf index 3927989..cdb3c6b 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -4,7 +4,7 @@ variable "principal_arn" { validation { condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.principal_arn)) - error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/" + error_message = "principal_arn must be an IAM role ARN: arn:aws:iam:::role/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } @@ -20,7 +20,7 @@ variable "cluster_arns" { validation { condition = length(var.cluster_arns) > 0 && alltrue([for arn in var.cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." + error_message = "Provide at least one EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected. Every ARN must be known at plan time and its cluster must already exist: they drive for_each and a data source lookup, so a cluster created in the same apply has to be applied first (see 'Ordering' in the README)." } } @@ -69,6 +69,6 @@ variable "collector_cluster_arns" { validation { condition = alltrue([for arn in var.collector_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index efa8271..4203295 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -48,16 +48,22 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } } + # Kinds the collector lists, kept identical to the managed-scan ClusterRole + # (managed_scan.tf) and to collect.go in the platform monorepo + # (context/internal/cloudintegrations/k8sintegration/collect). rule { api_groups = [""] resources = [ "pods", "services", - "endpoints", "namespaces", "nodes", "persistentvolumes", "persistentvolumeclaims", + "configmaps", + "secrets", + "resourcequotas", + "limitranges", "serviceaccounts", ] verbs = ["get", "list"] @@ -69,6 +75,12 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["get", "list"] } + rule { + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + verbs = ["get", "list"] + } + rule { api_groups = ["apps"] resources = [ @@ -92,33 +104,14 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } rule { - api_groups = ["storage.k8s.io"] - resources = ["storageclasses"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["batch"] - resources = ["jobs", "cronjobs"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] - verbs = ["get", "list"] - } - - rule { - api_groups = ["apiextensions.k8s.io"] - resources = ["customresourcedefinitions"] - verbs = ["get", "list"] + api_groups = ["admissionregistration.k8s.io"] + resources = [ + "validatingwebhookconfigurations", + "mutatingwebhookconfigurations", + "validatingadmissionpolicies", + "validatingadmissionpolicybindings", + ] + verbs = ["get", "list"] } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 362a38d..9a2d9bf 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -163,3 +163,68 @@ run "issuer_urls_must_match_the_cluster_region" { expect_failures = [data.aws_iam_policy_document.assume_role_policy] } + +# The mock above fakes every aws_iam_policy_document's computed `json`, so a +# passing test only proves the module plans, not that a policy grants any +# action. `statement` blocks are the data source's own configuration -- they +# are plan-time known even when the resulting document is mocked -- so assert +# on those instead. Only the KMS, S3 access point and readonly policies are +# covered here: the trust and deny-list policies aren't in W58/N78's scope. +run "kms_policy_grants_the_expected_actions_and_resources" { + command = plan + + variables { + kms_key_arn = "arn:aws:kms:eu-central-1:111122223333:alias/nullify-customer-uploads" + } + + assert { + condition = toset(data.aws_iam_policy_document.kms_access_policy[0].statement[0].actions) == toset([ + "kms:DescribeKey", + "kms:GenerateDataKey", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + ]) + error_message = "The KMS policy must grant exactly these actions, no more and no less" + } + + assert { + condition = tolist(data.aws_iam_policy_document.kms_access_policy[0].statement[0].resources) == tolist(output.kms_policy_resources) + error_message = "The KMS policy statement's resources must be exactly the derived kms_policy_resources output" + } +} + +run "s3_access_point_policy_grants_put_only_under_the_collector_prefix" { + command = plan + + variables { + nullify_s3_access_point_arn = "arn:aws:s3:eu-central-1:111122223333:accesspoint/nullify-uploads" + } + + assert { + condition = toset(data.aws_iam_policy_document.s3_access_policy[0].statement[0].actions) == toset([ + "s3:PutObject", + "s3:PutObjectAcl", + ]) + error_message = "The access point statement must grant only PutObject and PutObjectAcl" + } + + assert { + condition = tolist(data.aws_iam_policy_document.s3_access_policy[0].statement[0].resources) == tolist(["arn:aws:s3:eu-central-1:111122223333:accesspoint/nullify-uploads/object/k8s-collector/*"]) + error_message = "The access point statement must be scoped to the k8s-collector object prefix, not the whole access point" + } +} + +run "readonly_policy_grants_a_read_only_action" { + command = plan + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "access-analyzer:GetFinding") + error_message = "The readonly policy's first statement must contain the expected access-analyzer read action" + } + + assert { + condition = data.aws_iam_policy_document.readonly_policy_part1.statement[0].effect == "Allow" + error_message = "The readonly policy statement must Allow, not Deny" + } +} diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index 83bbc15..beac5ff 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -24,7 +24,7 @@ variable "nullify_role_arn" { validation { condition = can(regex("^arn:aws:iam::[0-9]{12}:role/.+$", var.nullify_role_arn)) - error_message = "Must be a valid ARN for an IAM role in the format arn:aws:iam:::role/" + error_message = "Must be a valid ARN for an IAM role in the format arn:aws:iam:::role/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } @@ -41,7 +41,7 @@ variable "eks_cluster_arns" { validation { condition = alltrue([for arn in var.eks_cluster_arns : can(regex("^arn:aws:eks:[a-z0-9-]+:[0-9]{12}:cluster/[A-Za-z0-9][A-Za-z0-9_-]*$", arn))]) - error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/" + error_message = "Each entry must be an EKS cluster ARN: arn:aws:eks:::cluster/. Only the commercial aws partition is supported; arn:aws-us-gov: and arn:aws-cn: ARNs are rejected." } } diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index 30c2b99..5e5fab1 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -6,23 +6,25 @@ metadata: labels: {{- include "k8s-collector.labels" . | nindent 4 }} rules: - # Access for collecting cluster information + # Kinds the collector lists, kept identical to the managed-scan ClusterRole + # (modules/k8s-resources/managed_scan.tf) and to collect.go in the platform + # monorepo (context/internal/cloudintegrations/k8sintegration/collect). - apiGroups: [""] - resources: ["nodes", "namespaces", "pods", "services", "endpoints", "persistentvolumes", "persistentvolumeclaims", "serviceaccounts"] + resources: ["nodes", "namespaces", "pods", "services", "persistentvolumes", "persistentvolumeclaims", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"] verbs: ["get", "list"] - apiGroups: ["apps"] resources: ["deployments", "daemonsets", "statefulsets", "replicasets"] verbs: ["get", "list"] - - apiGroups: ["batch"] - resources: ["jobs", "cronjobs"] - verbs: ["get", "list"] - apiGroups: ["networking.k8s.io"] resources: ["ingresses", "networkpolicies"] verbs: ["get", "list"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] verbs: ["get", "list"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] verbs: ["get", "list"] {{- end }} \ No newline at end of file diff --git a/scripts/check_iam_policy_drift.py b/scripts/check_iam_policy_drift.py new file mode 100755 index 0000000..ca93fc4 --- /dev/null +++ b/scripts/check_iam_policy_drift.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Compares the Allow/Deny action sets granted by the CloudFormation template +against both Terraform trees (v5 and v6), so the three customer-facing +templates for the same IAM role cannot silently drift apart (ncc #58, +whiskeylotus, W58-3). + +Terraform's readonly grant is split across readonly_policy_part1 and +readonly_policy_part2 (an IAM managed policy document size limit); the +CloudFormation template splits the same grant across ReadOnlyAccessPolicy and +ReadOnlyAccessPolicy2. Both splits are combined before comparing. + +Exits non-zero and prints the symmetric difference on any mismatch. +""" + +import json +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CFN_TEMPLATE = REPO_ROOT / "aws-integration-setup/cloudformation/nullify-cloudformation-template.json" +TERRAFORM_TREES = ["aws-integration-setup/terraform", "aws-integration-setup/terraform-v6"] +DATA_TF_RELATIVE = "modules/nullify-aws-integration/data.tf" + +# data source name -> Effect it must carry +READONLY_SOURCES = ["readonly_policy_part1", "readonly_policy_part2"] +DENY_SOURCE = "deny_actions_policy" + + +def cfn_actions(effect_resources: list) -> set: + actions: set = set() + for resource_name in effect_resources: + template = json.loads(CFN_TEMPLATE.read_text()) + statements = template["Resources"][resource_name]["Properties"]["PolicyDocument"]["Statement"] + for statement in statements: + action = statement.get("Action", []) + actions.update(action if isinstance(action, list) else [action]) + return actions + + +def terraform_actions(data_tf: Path, data_source_names: list) -> set: + text = data_tf.read_text() + actions: set = set() + for name in data_source_names: + block_match = re.search( + r'data\s+"aws_iam_policy_document"\s+"%s"\s*\{(.*?)\n\}(?:\n|\Z)' % re.escape(name), + text, + re.DOTALL, + ) + if not block_match: + raise SystemExit(f"could not find data.aws_iam_policy_document.{name} in {data_tf}") + block = block_match.group(1) + for actions_match in re.finditer(r"actions\s*=\s*\[(.*?)\]", block, re.DOTALL): + actions.update(re.findall(r'"([^"]+)"', actions_match.group(1))) + return actions + + +def compare(label: str, cfn: set, terraform: set) -> bool: + if cfn == terraform: + print(f"OK: {label} matches ({len(cfn)} actions)") + return True + only_cfn = cfn - terraform + only_tf = terraform - cfn + print(f"MISMATCH: {label}") + if only_cfn: + print(f" only in CloudFormation ({len(only_cfn)}): {sorted(only_cfn)}") + if only_tf: + print(f" only in Terraform ({len(only_tf)}): {sorted(only_tf)}") + return False + + +def main() -> int: + cfn_readonly = cfn_actions(["ReadOnlyAccessPolicy", "ReadOnlyAccessPolicy2"]) + cfn_deny = cfn_actions(["DenyActionsPolicy"]) + + ok = True + for tree in TERRAFORM_TREES: + data_tf = REPO_ROOT / tree / DATA_TF_RELATIVE + tf_readonly = terraform_actions(data_tf, READONLY_SOURCES) + tf_deny = terraform_actions(data_tf, [DENY_SOURCE]) + + ok = compare(f"Allow actions: CloudFormation vs {tree}", cfn_readonly, tf_readonly) and ok + ok = compare(f"Deny actions: CloudFormation vs {tree}", cfn_deny, tf_deny) and ok + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) From d4e7a9c0009a491e4476b372e10bb1a789a721ce Mon Sep 17 00:00:00 2001 From: Tim Thacker <123459940+tim-thacker-nullify@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:10:26 +1000 Subject: [PATCH 28/36] fix(helm): collector chart namespace, service account and default-value fixes (#63) * fix(helm): collector chart namespace, service account and default-value fixes - Remove the pre-install namespace hook and the bitnami/kubectl Job; the namespace now comes from --create-namespace, and a namespace helper falls back to the release namespace when serviceAccount.namespace is empty (default stays nullify for the IRSA trust). - Use the service-account-name helper in the CronJob and ServiceAccount. - Default clusterName, s3.bucket and kms.keyArn to "" and fail on empty or placeholder values, on a KMS alias instead of a key ARN, on a region that differs from the key's, and on a missing EKS role-arn or GKE role ARN. - Remove collector.dataCollection, which the binary never read. - Fix the labels helper joining custom labels onto the previous line, and apply nodeSelector/tolerations to the collector pod. - kubeVersion >=1.21, chart 0.4.0, and README/NOTES corrections. Co-Authored-By: Claude Opus 5 (1M context) * test(helm): expect the role-arn guard for the collector defaults case Co-Authored-By: Claude Opus 5 (1M context) * fix(helm): give the exact fix in the collector's KMS alias and region errors - An alias keyArn still fails, now saying to use the key ARN from the Nullify configure page and to contact Nullify if the page shows an alias. - The region mismatch error explains that --reuse-values from 0.2.0 carries us-east-1, and to upgrade with --reset-then-reuse-values or set the region. Co-Authored-By: Claude Opus 5 (1M context) * fix(helm): accept every AWS partition and allow opting out of the collector role-arn guard - The EKS role-arn guard accepts arn:aws, arn:aws-us-gov and arn:aws-cn. - serviceAccount.requireRoleArn (default true) lets EKS Pod Identity, instance-profile and kube2iam/kiam installs render without IRSA; an empty role-arn annotation is no longer rendered. Co-Authored-By: Claude Opus 5 (1M context) * docs(helm): document the alias migration, requireRoleArn and --reuse-values upgrades Co-Authored-By: Claude Opus 5 (1M context) * fix(helm): accept KMS alias ARNs in the collector and keep the role ARN commercial - collector.kms.keyArn accepts arn:aws:kms:::alias/ and key ARNs including multi-Region key/mrk-..., matching the connector CloudFormation AllowedPattern. The region default and check take the region from either form. - NOTES warns when an alias is used: IAM matches the key ARN, so the role needs the connector stack version that also grants key/*; the Terraform module grants only the given ARN. - The role-arn guard is back to arn:aws:iam:: only; Nullify's bucket and key are commercial-partition. serviceAccount.requireRoleArn=false stays the escape hatch. ci/eks-govcloud-role.yaml is removed. - CI: ci/eks-kms-alias.yaml and ci/eks-kms-mrk.yaml must render; a bare alias, a GovCloud alias ARN, an alias region mismatch and a GovCloud role ARN must fail. Co-Authored-By: Claude Opus 5 (1M context) * fix(helm): keep ci/ out of the collector chart package - .helmignore excludes ci/ from 0.4.0, so a later edit to a CI values file does not change a published version's contents and force a version bump. - Added with the 0.4.0 bump because 0.2.0 is already published; the file changes the package contents. Co-Authored-By: Claude Opus 5 (1M context) * docs(helm): state the KMS alias requirement instead of pointing at the current template - NOTES, README and values.yaml say what an alias in collector.kms.keyArn needs: the IAM role's KMS statement must cover the resolved key, for example arn:aws:kms:::key/*. - CloudFormation: connector releases that include PR #64 grant key/* in the account and region of NullifyKMSKeyArn; the template on main grants only the ARN given. Terraform: the module accepts an alias; check the module version's KMS statement covers key/*. A key ARN works with every version. - New README section "KMS alias ARNs" holds the detail; the table, 0.4.0 upgrade notes and troubleshooting link to it. Co-Authored-By: Claude Opus 5 (1M context) * fix(helm): refuse to unpublish live chart versions and document the merge order main's helm-release.yml rebuilds the Pages site from the collector chart alone, so merging this chain before the publish rework unpublishes collector 0.2.0 and any other version the live index serves. - the release job compares the live index against the single package it would deploy and fails, naming the required merge order, when it would drop one - README gains a maintainer section stating that order and why Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/helm-release.yml | 24 +++ IMPLEMENTATION.md | 15 +- README.md | 33 ++-- helm-charts/nullify-k8s-collector/.helmignore | 1 + helm-charts/nullify-k8s-collector/Chart.yaml | 5 +- helm-charts/nullify-k8s-collector/README.md | 141 +++++++++++------- .../ci/eks-kms-alias.yaml | 11 ++ .../nullify-k8s-collector/ci/eks-kms-mrk.yaml | 13 ++ .../nullify-k8s-collector/ci/eks-no-irsa.yaml | 10 ++ helm-charts/nullify-k8s-collector/ci/eks.yaml | 15 ++ helm-charts/nullify-k8s-collector/ci/gke.yaml | 16 ++ .../nullify-k8s-collector/templates/NOTES.txt | 65 +++++--- .../templates/_helpers.tpl | 51 ++++++- .../templates/clusterrolebinding.yaml | 4 +- .../templates/cronjob.yaml | 29 ++-- .../templates/namespace.yaml | 16 -- .../templates/pre-install-job.yaml | 34 ----- .../templates/serviceaccount.yaml | 15 +- .../nullify-k8s-collector/values-example.yaml | 36 ++--- helm-charts/nullify-k8s-collector/values.yaml | 57 ++++--- .../must-fail/defaults.yaml | 4 + .../must-fail/eks-missing-role-arn.yaml | 7 + .../must-fail/gke-missing-role-arn.yaml | 8 + .../must-fail/kms-bare-alias.yaml | 10 ++ .../must-fail/kms-govcloud-alias.yaml | 10 ++ .../must-fail/kms-placeholder.yaml | 10 ++ .../must-fail/placeholder-bucket.yaml | 10 ++ .../must-fail/placeholder-cluster-name.yaml | 10 ++ .../must-fail/region-mismatch-alias.yaml | 12 ++ .../must-fail/region-mismatch.yaml | 12 ++ .../must-fail/reuse-values-from-0.2.0.yaml | 16 ++ .../must-fail/role-arn-govcloud.yaml | 10 ++ 32 files changed, 492 insertions(+), 218 deletions(-) create mode 100644 helm-charts/nullify-k8s-collector/.helmignore create mode 100644 helm-charts/nullify-k8s-collector/ci/eks-kms-alias.yaml create mode 100644 helm-charts/nullify-k8s-collector/ci/eks-kms-mrk.yaml create mode 100644 helm-charts/nullify-k8s-collector/ci/eks-no-irsa.yaml create mode 100644 helm-charts/nullify-k8s-collector/ci/eks.yaml create mode 100644 helm-charts/nullify-k8s-collector/ci/gke.yaml delete mode 100644 helm-charts/nullify-k8s-collector/templates/namespace.yaml delete mode 100644 helm-charts/nullify-k8s-collector/templates/pre-install-job.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/defaults.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/eks-missing-role-arn.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/gke-missing-role-arn.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/kms-bare-alias.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/kms-govcloud-alias.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/kms-placeholder.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/placeholder-bucket.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/placeholder-cluster-name.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/region-mismatch-alias.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/region-mismatch.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/reuse-values-from-0.2.0.yaml create mode 100644 tests/helm/nullify-k8s-collector/must-fail/role-arn-govcloud.yaml diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 2d0659d..32167ae 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -43,6 +43,30 @@ jobs: echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Using Chart.yaml version: $VERSION" + # This job rebuilds the whole Pages site from this one chart, so deploying + # it drops every other version the live index serves. The publish rework + # (PR #65 -> #70 -> #75) replaces this workflow with one that keeps them; + # until that is on main, fail here rather than unpublish. + - name: Refuse to unpublish versions the live index serves + env: + VERSION: ${{ steps.version.outputs.version }} + INDEX_URL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/index.yaml + run: | + index="$(mktemp)" + status="$(curl -sSL -o "$index" -w '%{http_code}' "$INDEX_URL")" + case "$status" in + 200) ;; + 404) echo "no published index yet"; exit 0 ;; + *) echo "::error::fetching $INDEX_URL returned HTTP $status"; exit 1 ;; + esac + published="$(grep -oE '[A-Za-z0-9_.-]+-[0-9]+\.[0-9]+\.[0-9]+[A-Za-z0-9_.+-]*\.tgz' "$index" | sort -u || true)" + dropped="$(printf '%s\n' "$published" | grep -v '^$' | grep -vxF "nullify-k8s-collector-${VERSION}.tgz" || true)" + if [ -n "$dropped" ]; then + echo "::error::deploying would unpublish: $(echo $dropped). Merge the Helm publish rework (PR #65 -> #70 -> #75) to main first; its workflow republishes every version." + exit 1 + fi + echo "the live index serves only nullify-k8s-collector-${VERSION}.tgz or nothing" + - name: Add Helm repos (if chart has dependencies) run: | helm repo add bitnami https://charts.bitnami.com/bitnami || true diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 7448580..736ffc2 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -120,17 +120,18 @@ serviceAccount: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::YOUR-ACCOUNT:role/AWSIntegration-mycompany-NullifyReadOnlyRole -aws: - region: "us-east-1" - s3Bucket: "mycompany-nullify-k8s-collector" - collector: + clusterName: "my-eks-cluster" # exact EKS cluster name + s3: + bucket: "nullify-bucket-from-configure-page" + kms: + keyArn: "arn:aws:kms:::key/" # key ARN, not an alias + # aws.region defaults to the key's region, which is the Nullify bucket's region schedule: "0 0 * * *" # Daily at midnight - dataCollection: - excludeNamespaces: "kube-system,kube-public" - metadataOnly: true ``` +The IRSA role comes from the connector's EKS integration (CloudFormation `EnableEKSIntegration=true`, or Terraform `enable_kubernetes_integration`), which trusts `system:serviceaccount:nullify:nullify-k8s-collector-sa`. + ## Production Deployment Checklist ### Pre-Deployment diff --git a/README.md b/README.md index 9e6f9ba..ff4d71d 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,6 @@ helm repo update # 2. Create your production values file (get values from Nullify configure page) cat > values-production.yaml << EOF collector: - aws: - region: "us-west-2" # Extract from your KMS key ARN clusterName: "your-cluster-name" # Must match your actual EKS cluster name s3: bucket: "your-nullify-bucket" @@ -268,12 +266,10 @@ nullify-cloud-connector/ │ ├── values-example.yaml # Example production configuration │ ├── README.md # Chart-specific documentation │ └── templates/ # Kubernetes resource templates -│ ├── namespace.yaml # Namespace creation │ ├── serviceaccount.yaml # IRSA service account │ ├── clusterrole.yaml # Read-only cluster permissions │ ├── clusterrolebinding.yaml # RBAC binding -│ ├── cronjob.yaml # Main collector CronJob -│ └── pre-install-job.yaml # Pre-installation validation +│ └── cronjob.yaml # Main collector CronJob │ ├── aws-integration-setup/ # 🏗️ AWS INFRASTRUCTURE │ ├── 🏗️ cloudformation/ # CloudFormation Templates @@ -346,7 +342,7 @@ All required values are provided in the Nullify configure page. ```yaml collector: aws: - region: "us-west-2" # Extract from your KMS key ARN + region: "" # Nullify bucket's region; empty = the KMS key's region clusterName: "your-cluster-name" # Must match your actual EKS cluster name s3: bucket: "your-nullify-bucket" @@ -381,12 +377,7 @@ eks_cluster_arns = [ ```yaml collector: - # Data collection filters - dataCollection: - excludeNamespaces: "kube-system,kube-public" - includeResources: "pods,services,deployments" - metadataOnly: true # Only collect metadata - + # The collector has no namespace or resource filters. # Resource limits resources: limits: @@ -423,6 +414,24 @@ helm uninstall nullify-collector --namespace nullify kubectl create job --from=cronjob/nullify-k8s-collector manual-collection -n nullify ``` +## 🚚 **Publishing the Helm charts (maintainers)** + +`main`'s `helm-release.yml` rebuilds the GitHub Pages site from scratch out of +`helm-charts/nullify-k8s-collector` alone. It never fetches or merges the live +`index.yaml`, so a deploy **unpublishes every other chart version**, and +`helm install --version 0.2.0` stops resolving. + +**Required merge order:** the Helm publish rework (PR #65 → #70 → #75) has to +reach `main` before any branch that bumps a chart version, including the +collector chain (PR #68 → #72 → #77) and the readonly-access chart (PR #71 → +#76). Its workflow re-downloads every published version, verifies it against +its digest and republishes it, which is what makes a chart version permanent. + +Until then the release job refuses to deploy when the live index serves any +version it would drop, and names this order in the error. Once the rework is on +`main` that guard is gone with the workflow it lives in; on a merge conflict +between the two, take the rework's `helm-release.yml`. + ## 📚 **Documentation** | Document | Description | diff --git a/helm-charts/nullify-k8s-collector/.helmignore b/helm-charts/nullify-k8s-collector/.helmignore new file mode 100644 index 0000000..93b6cee --- /dev/null +++ b/helm-charts/nullify-k8s-collector/.helmignore @@ -0,0 +1 @@ +ci/ diff --git a/helm-charts/nullify-k8s-collector/Chart.yaml b/helm-charts/nullify-k8s-collector/Chart.yaml index 566a7ab..a783b9c 100644 --- a/helm-charts/nullify-k8s-collector/Chart.yaml +++ b/helm-charts/nullify-k8s-collector/Chart.yaml @@ -2,11 +2,12 @@ apiVersion: v2 name: nullify-k8s-collector description: A Helm chart for Nullify's Kubernetes information collector for cloud security scanning type: application -version: 0.3.0 +version: 0.4.0 +kubeVersion: ">=1.21.0-0" maintainers: - name: Nullify url: https://nullify.ai keywords: - security - kubernetes - - scanning \ No newline at end of file + - scanning diff --git a/helm-charts/nullify-k8s-collector/README.md b/helm-charts/nullify-k8s-collector/README.md index 5d85263..fbd4a2f 100644 --- a/helm-charts/nullify-k8s-collector/README.md +++ b/helm-charts/nullify-k8s-collector/README.md @@ -4,8 +4,14 @@ This Helm chart deploys a Kubernetes collector for the Nullify platform to gathe ## Prerequisites -- Kubernetes 1.16+ -- Helm 3.0+ +- Kubernetes 1.21+ (the chart uses `batch/v1` CronJob) +- Helm 3 +- **EKS:** the Nullify AWS integration deployed with its EKS integration enabled (see [AWS IAM Configuration](#aws-iam-configuration)) +- **GKE:** the cluster registered with Nullify (see [GKE](#gke-with-workload-identity-federation)) +- Egress from the collector pod over HTTPS to `public.ecr.aws`, and to STS and S3 in the Nullify bucket's region + +> **Image tag:** the default `collector.image.tag` is the floating `k8s-collector-latest`, pulled on every run. +> Pin a versioned tag once Nullify publishes one. ## Supported platforms @@ -19,81 +25,95 @@ The same chart runs on **EKS** and **GKE**. Select the platform via `cloudProvid In both cases the collector uploads cluster metadata to the same Nullify-managed S3 bucket. No long-lived AWS credential is stored in the customer cluster. -## Configuration +## Installation + +The chart does not create its namespace. Let Helm create it: + +```bash +helm repo add nullify https://nullify-platform.github.io/nullify-cloud-connector/ +helm repo update +helm upgrade --install nullify-k8s-collector nullify/nullify-k8s-collector \ + --namespace nullify --create-namespace \ + -f values-production.yaml +``` + +With Flux, set `spec.targetNamespace: nullify` and `spec.install.createNamespace: true` on the HelmRelease. + +All namespaced objects go to `serviceAccount.namespace` (default `nullify`), whatever `--namespace` says; set it to `""` to use the release namespace. +On EKS keep the defaults: the connector's IRSA trust only allows `system:serviceaccount:nullify:nullify-k8s-collector-sa`. -The following table lists the configurable parameters of the chart and their default values. +The chart refuses to render when a required value is missing or still a placeholder, and names the value in the error. + +## Configuration -> **Required values**: `collector.clusterName`, `collector.s3.bucket`, `collector.kms.keyArn`. -> Platform-specific required values are listed below. Get these from the Nullify configure page. +> **Required values**: `collector.clusterName`, `collector.s3.bucket`, `collector.kms.keyArn`, and +> `serviceAccount.annotations."eks.amazonaws.com/role-arn"` (EKS) or `collector.gke.awsRoleArn` (GKE). +> Get these from the Nullify configure page. | Parameter | Description | Default | |-----------|-------------|---------| | `cloudProvider` | Platform the collector runs on: `aws` (EKS) or `gcp` (GKE) | `aws` | | `serviceAccount.create` | If true, create a new service account | `true` | -| `serviceAccount.annotations` | Annotations for the service account. The chart renders only the annotation that matches `cloudProvider`. | See [values.yaml](./values.yaml) | +| `serviceAccount.requireRoleArn` | EKS: fail the render unless `eks.amazonaws.com/role-arn` is an `arn:aws:iam::` role ARN. Set `false` when the pod gets AWS credentials another way (see [Other Kubernetes Clusters](#other-kubernetes-clusters)). | `true` | +| `serviceAccount.annotations` | Annotations for the service account. The chart renders only the annotation that matches `cloudProvider`. | `eks.amazonaws.com/role-arn: ""` | | `serviceAccount.name` | Name of the service account | `nullify-k8s-collector-sa` | +| `serviceAccount.namespace` | Namespace of every namespaced object; `""` means the release namespace. Must already exist. | `nullify` | | `collector.image.repository` | Image repository | `public.ecr.aws/w4o2j2x4/integrations` | | `collector.image.tag` | Image tag | `k8s-collector-latest` | | `collector.image.pullPolicy` | Pull policy | `Always` | | `collector.schedule` | CronJob schedule | `0 0 * * *` (daily at midnight) | -| `collector.s3.bucket` | S3 bucket for storing data (from Nullify configure page) | `YOUR-NULLIFY-S3-BUCKET` | +| `collector.s3.bucket` | S3 bucket for storing data (from Nullify configure page) | `""` (required) | | `collector.s3.keyPrefix` | S3 key prefix | `k8s-collector` | -| `collector.aws.region` | AWS region | `us-east-1` | -| `collector.clusterName` | Cluster name (must match your actual cluster name) | `YOUR-CLUSTER-NAME` | -| `collector.kms.keyArn` | KMS key ARN for encryption (from Nullify configure page) | `""` | +| `collector.aws.region` | Region of the **Nullify S3 bucket**, not your cluster. Defaults to the region in `collector.kms.keyArn`; any other value is rejected, because S3 only accepts a KMS key from the bucket's region. | `""` | +| `collector.clusterName` | Cluster name (must match your actual cluster name) | `""` (required) | +| `collector.kms.keyArn` | KMS key ARN `arn:aws:kms:::key/` or alias ARN `arn:aws:kms:::alias/`, from the Nullify configure page. With an alias, the IAM role's KMS statement must cover the key the alias resolves to, for example `arn:aws:kms:::key/*`; see [KMS alias ARNs](#kms-alias-arns). | `""` (required) | | `collector.debug.enabled` | Enable debug logging for troubleshooting | `false` | | `collector.gke.awsRoleArn` | **GKE only.** Nullify-owned federated AWS IAM role ARN (provided after cluster registration). | `""` | | `collector.gke.audience` | **GKE only.** Token audience for the projected SA token. Do not change unless Nullify asks you to. | `sts.amazonaws.com` | | `collector.gke.webIdentityTokenPath` | **GKE only.** In-pod path of the projected SA token. | `/var/run/secrets/tokens/gcp-sa-token` | -| `labels` | Additional labels for the collector resources | `null` | +| `nodeSelector` / `tolerations` | Collector pod placement | `{}` / `[]` | +| `labels` | Additional labels for the collector resources | `{}` | + +The collector has no namespace or resource filters: it always lists every kind its ClusterRole allows, in every namespace. ## Security Context -The collector runs with: +The collector container runs with: -- Non-root user (UID 1001) +- `runAsNonRoot: true` - Read-only root filesystem - No privilege escalation - All capabilities dropped ## AWS IAM Configuration -> 📖 **Note**: The IAM role ARN required for IRSA configuration is provided in the Nullify configure page. Log in to your Nullify configure page, navigate to Configure > Integrations, and select AWS integration to obtain the required values. +> 📖 **Note**: The IAM role ARN, S3 bucket and KMS key ARN are provided in the Nullify configure page (Configure > Integrations > AWS). ### EKS with IRSA (IAM Roles for Service Accounts) -For EKS clusters, use IAM Roles for Service Accounts (IRSA): - -1. Create an IAM policy with S3 permissions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::your-bucket-name", - "arn:aws:s3:::your-bucket-name/*" - ] - } - ] -} -``` +Do not create the IRSA role by hand. The Nullify AWS integration role gains the collector's trust and permissions when you enable its EKS integration: + +- **CloudFormation** ([template](../../aws-integration-setup/cloudformation/README.md)): update the stack with `EnableEKSIntegration=true`, `EKSOidcProviderURL=oidc.eks..amazonaws.com/id/` (no `https://`), `NullifyS3Bucket` and `NullifyKMSKeyArn` (the key ARN). +- **Terraform** ([module](../../aws-integration-setup/terraform/README.md)): set `enable_kubernetes_integration = true`, `eks_cluster_arns`, `s3_bucket_name` and `kms_key_arn`. `kubernetes_namespace` and `service_account_name` must match this chart's `serviceAccount.namespace` and `serviceAccount.name`. Deploy the collector with either this chart or the `k8s-resources` module, not both. + +The cluster needs an IAM OIDC provider (`eksctl utils associate-iam-oidc-provider --cluster --approve`). + +The collector only uploads. It needs `s3:PutObject` on the Nullify bucket and `kms:GenerateDataKey` on the key, both granted by the integration. It never reads from the bucket, and the integration explicitly denies `s3:GetObject`. -2. Create an IAM role for the service account and attach the policy +### KMS alias ARNs -3. In your values.yaml file, specify the IAM role ARN: +IAM evaluates KMS permissions against the key an alias resolves to, not the alias. When `collector.kms.keyArn` is an alias ARN, the IAM role's KMS statement must cover that key, for example `kms:GenerateDataKey` on `arn:aws:kms:::key/*` in Nullify's KMS account and region. Otherwise uploads are denied. + +- **CloudFormation:** connector releases that include PR #64 grant `key/*` in the account and region of the `NullifyKMSKeyArn` parameter. A stack created from an earlier template grants only the ARN it was given; update it to a later release. +- **Terraform:** the module accepts an alias in `kms_key_arn`. Check that your module version's KMS statement also covers `key/*`; one that grants only `kms_key_arn` denies uploads. +- Either way, a key ARN (`key/`) works with every version. + +Then set the role ARN in your values: ```yaml serviceAccount: annotations: - eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/NullifyCollectorRole" + eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/AWSIntegration-yourcompany-NullifyReadOnlyRole" ``` ### GKE with Workload Identity Federation @@ -131,8 +151,6 @@ cloudProvider: gcp collector: clusterName: "my-gke-cluster" - aws: - region: "us-east-1" # AWS region of the Nullify S3 bucket s3: bucket: "your-nullify-bucket" kms: @@ -145,24 +163,36 @@ collector: Then install: ```bash -helm install nullify-k8s-collector ./nullify-k8s-collector -f values-gke.yaml +helm upgrade --install nullify-k8s-collector nullify/nullify-k8s-collector \ + --namespace nullify --create-namespace -f values-gke.yaml ``` ### Other Kubernetes Clusters -For clusters outside EKS and GKE, you'll need to provide AWS credentials through -other means: +For clusters outside EKS and GKE, or EKS without IRSA, you'll need to provide AWS +credentials through other means, and set `serviceAccount.requireRoleArn: false` +so the chart renders without the role-arn annotation: +- Using EKS Pod Identity for the collector's ServiceAccount - Using AWS environment variables in the pod - Using instance profiles for nodes running on EC2 - Using a solution like Kube2IAM or kiam -## Uninstallation +## Upgrading to 0.4.0 -To uninstall/delete the `k8s-collector` deployment: +- `namespace.create` and `namespace.requireNamespace` are removed, along with the pre-install namespace hook and the `bitnami/kubectl` Job. Existing namespaces are not deleted. New installs need `--create-namespace`, or an existing namespace. +- `collector.dataCollection.*` is removed. The collector never read `EXCLUDE_NAMESPACES`, `INCLUDE_RESOURCES` or `METADATA_ONLY`. +- `collector.clusterName`, `collector.s3.bucket` and `collector.kms.keyArn` now default to `""`, and rendering fails when they are empty or still the old placeholders. The EKS role-arn annotation is validated the same way, unless `serviceAccount.requireRoleArn: false`. +- `collector.kms.keyArn` must be an `arn:aws:kms:` key or alias ARN. With an alias, the IAM role's KMS statement must cover the resolved key (for example `arn:aws:kms:::key/*`), or uploads are denied; see [KMS alias ARNs](#kms-alias-arns). +- `collector.aws.region` now defaults to the KMS key's region instead of `us-east-1`. +- `helm upgrade --reuse-values` from 0.2.0 keeps 0.2.0's defaults, including `collector.aws.region: us-east-1`, so the region check fails unless the key is in `us-east-1`. Upgrade with `--reset-then-reuse-values` (Helm 3.14+) or `-f `, or set `collector.aws.region` to the key's region. +- `nodeSelector` and `tolerations` now apply to the collector pod; before, only the removed pre-install Job used them. + +## Uninstallation ```bash -helm delete k8s-collector +helm uninstall nullify-k8s-collector --namespace nullify +kubectl delete namespace nullify # optional; Helm does not delete the namespace ``` ## Troubleshooting @@ -174,22 +204,23 @@ If the CronJob is not creating jobs on schedule: 1. Check that the cron schedule is valid: ```bash - kubectl -n nullify get cronjob k8s-collector -o jsonpath='{.spec.schedule}' + kubectl -n nullify get cronjob nullify-k8s-collector -o jsonpath='{.spec.schedule}' ``` 2. Check the CronJob status: ```bash - kubectl -n nullify describe cronjob k8s-collector + kubectl -n nullify describe cronjob nullify-k8s-collector ``` ### Access Issues If the job is failing due to S3 access issues: -1. Verify that the IAM role or credentials have the necessary permissions -2. Check that the S3 bucket exists and is accessible -3. Examine the job logs for detailed error messages +1. `AccessDenied` on `kms:GenerateDataKey` with an alias in `collector.kms.keyArn`: the IAM role's KMS statement does not cover the key the alias resolves to. Grant `arn:aws:kms:::key/*` in Nullify's KMS account (see [KMS alias ARNs](#kms-alias-arns)), or use the key ARN. +2. `AccessDenied` on `sts:AssumeRoleWithWebIdentity`: the service account must be `nullify/nullify-k8s-collector-sa`, and the integration's EKS OIDC provider URL must match the cluster. +3. Wrong-region or redirect errors: `collector.aws.region` must be the Nullify bucket's region (leave it empty). +4. Examine the job logs for detailed error messages. ### Debug Mode @@ -204,7 +235,7 @@ To enable debug logging for troubleshooting: 2. **Upgrade the deployment**: ```bash - helm upgrade nullify-collector nullify/nullify-k8s-collector -f values.yaml + helm upgrade nullify-k8s-collector nullify/nullify-k8s-collector -n nullify -f values.yaml ``` 3. **Check debug logs**: diff --git a/helm-charts/nullify-k8s-collector/ci/eks-kms-alias.yaml b/helm-charts/nullify-k8s-collector/ci/eks-kms-alias.yaml new file mode 100644 index 0000000..1e6a295 --- /dev/null +++ b/helm-charts/nullify-k8s-collector/ci/eks-kms-alias.yaml @@ -0,0 +1,11 @@ +# EKS install with the KMS alias ARN the configure page shows today; the region comes from the alias ARN. +cloudProvider: aws +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:ap-southeast-2:123456789012:alias/example-customer-upload +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/helm-charts/nullify-k8s-collector/ci/eks-kms-mrk.yaml b/helm-charts/nullify-k8s-collector/ci/eks-kms-mrk.yaml new file mode 100644 index 0000000..725a828 --- /dev/null +++ b/helm-charts/nullify-k8s-collector/ci/eks-kms-mrk.yaml @@ -0,0 +1,13 @@ +# EKS install with a multi-Region KMS key ARN and an explicit matching region. +cloudProvider: aws +collector: + clusterName: example-eks-cluster + aws: + region: us-east-2 + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:us-east-2:123456789012:key/mrk-1234abcd12ab34cd56ef1234567890ab +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/helm-charts/nullify-k8s-collector/ci/eks-no-irsa.yaml b/helm-charts/nullify-k8s-collector/ci/eks-no-irsa.yaml new file mode 100644 index 0000000..c0f6a7b --- /dev/null +++ b/helm-charts/nullify-k8s-collector/ci/eks-no-irsa.yaml @@ -0,0 +1,10 @@ +# EKS without IRSA (Pod Identity, instance profile, kube2iam): no role-arn annotation. +cloudProvider: aws +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:us-east-2:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + requireRoleArn: false diff --git a/helm-charts/nullify-k8s-collector/ci/eks.yaml b/helm-charts/nullify-k8s-collector/ci/eks.yaml new file mode 100644 index 0000000..4586114 --- /dev/null +++ b/helm-charts/nullify-k8s-collector/ci/eks.yaml @@ -0,0 +1,15 @@ +# EKS install with every required value; labels exercise the labels helper. +cloudProvider: aws +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole +labels: + environment: ci +nodeSelector: + node-type: system diff --git a/helm-charts/nullify-k8s-collector/ci/gke.yaml b/helm-charts/nullify-k8s-collector/ci/gke.yaml new file mode 100644 index 0000000..344b4b9 --- /dev/null +++ b/helm-charts/nullify-k8s-collector/ci/gke.yaml @@ -0,0 +1,16 @@ +# GKE install in the release namespace, with an explicit matching region. +cloudProvider: gcp +collector: + clusterName: example-gke-cluster + aws: + region: us-east-1 + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:us-east-1:123456789012:key/00000000-0000-0000-0000-000000000000 + gke: + awsRoleArn: arn:aws:iam::123456789012:role/NullifyK8sCollectorRole +serviceAccount: + namespace: "" + annotations: + iam.gke.io/gcp-service-account: nullify-k8s-collector@example-project.iam.gserviceaccount.com diff --git a/helm-charts/nullify-k8s-collector/templates/NOTES.txt b/helm-charts/nullify-k8s-collector/templates/NOTES.txt index 68ff03e..df047ee 100644 --- a/helm-charts/nullify-k8s-collector/templates/NOTES.txt +++ b/helm-charts/nullify-k8s-collector/templates/NOTES.txt @@ -16,25 +16,48 @@ The data collected includes (but is not limited to): ## Configuration - Cloud Provider: {{ .Values.cloudProvider | default "aws" }} +- Cluster Name: {{ .Values.collector.clusterName }} - S3 Bucket: {{ .Values.collector.s3.bucket }} - S3 Key Prefix: {{ .Values.collector.s3.keyPrefix }} -- AWS Region: {{ .Values.collector.aws.region }} +- AWS Region: {{ .Values.collector.aws.region | default "region of collector.kms.keyArn" }} +- Namespace: {{ include "k8s-collector.namespace" . }} - Schedule: {{ .Values.collector.schedule }} +{{- if contains ":alias/" (toString .Values.collector.kms.keyArn) }} -{{- if and .Values.collector.dataCollection .Values.collector.dataCollection.excludeNamespaces }} -- Excluded Namespaces: {{ .Values.collector.dataCollection.excludeNamespaces }} +WARNING: collector.kms.keyArn is a KMS alias ARN. IAM matches the key the alias resolves +to, not the alias, so the IAM role's KMS statement must cover that key, for example +kms:GenerateDataKey on arn:aws:kms:::key/* in Nullify's KMS account +and region. Otherwise uploads are denied. +- CloudFormation: connector releases that include nullify-cloud-connector PR #64 grant + key/* in the account and region of the NullifyKMSKeyArn parameter. A stack created + from an earlier template grants only the ARN it was given. +- Terraform: the module accepts an alias in kms_key_arn. Check that your module version's + KMS statement also covers key/*; if it grants only kms_key_arn, uploads are denied. +Or use the key ARN, which every version grants. {{- end }} -{{- if and .Values.collector.dataCollection .Values.collector.dataCollection.metadataOnly }} -- Collection Mode: Metadata only (no resource content) -{{- else }} -- Collection Mode: Full resource data +{{- if ne (include "k8s-collector.namespace" .) .Release.Namespace }} + +NOTE: the collector's objects are in namespace "{{ include "k8s-collector.namespace" . }}" +(serviceAccount.namespace), not the release namespace "{{ .Release.Namespace }}". +{{- end }} +{{- with .Values.namespace }} +{{- if .create }} + +NOTE: namespace.create was removed in chart 0.4.0 and is ignored. Create the namespace with +`helm install --create-namespace`, or `spec.install.createNamespace: true` in a Flux HelmRelease. +{{- end }} +{{- end }} +{{- if .Values.collector.dataCollection }} + +NOTE: collector.dataCollection was removed in chart 0.4.0 and is ignored. The collector never +supported namespace or resource filters, so these values had no effect. {{- end }} {{- if eq (.Values.cloudProvider | default "aws") "gcp" }} ## Authentication (GKE Projected SA Token → AWS STS) -- Nullify AWS Role: {{ .Values.collector.gke.awsRoleArn | default "NOT SET — ask Nullify for the role ARN" }} +- Nullify AWS Role: {{ .Values.collector.gke.awsRoleArn | default .Values.collector.gke.nullifyAwsRoleArn }} - Token Audience: {{ .Values.collector.gke.audience }} The collector uses a projected Kubernetes ServiceAccount token (signed by the @@ -43,30 +66,26 @@ AWS credential is stored in the cluster. No GKE Workload Identity setup required {{- else }} ## Authentication (EKS IRSA) -{{- if (index .Values.serviceAccount.annotations "eks.amazonaws.com/role-arn") }} -- IAM Role: {{ index .Values.serviceAccount.annotations "eks.amazonaws.com/role-arn" }} -{{- else }} -NOTE: No IAM role was specified for the ServiceAccount. Make sure the collector -has necessary AWS credentials through other means (instance profile, environment variables, etc.) -{{- end }} +- IAM Role: {{ index (.Values.serviceAccount.annotations | default dict) "eks.amazonaws.com/role-arn" | default "none set; the pod needs AWS credentials from outside this chart (serviceAccount.requireRoleArn=false)" }} +- ServiceAccount: system:serviceaccount:{{ include "k8s-collector.namespace" . }}:{{ include "k8s-collector.serviceAccountName" . }} + (the connector CloudFormation trust only allows system:serviceaccount:nullify:nullify-k8s-collector-sa) {{- end }} ## Required AWS Permissions -The IAM role or credentials used by the collector need these permissions (already granted -when Nullify's IAM resources are installed): -- s3:PutObject -- s3:GetObject -- s3:ListBucket +The collector only uploads. Its role needs s3:PutObject on the Nullify bucket and +kms:GenerateDataKey on the KMS key; the connector's EKS integration (CloudFormation +EnableEKSIntegration, or Terraform enable_kubernetes_integration) grants both. +It never reads from the bucket. ## Useful Commands You can verify the CronJob is created with: - kubectl -n {{ .Values.serviceAccount.namespace }} get cronjob {{ include "k8s-collector.name" . }} + kubectl -n {{ include "k8s-collector.namespace" . }} get cronjob {{ include "k8s-collector.name" . }} To check when the job is scheduled to run next: - kubectl -n {{ .Values.serviceAccount.namespace }} get cronjob {{ include "k8s-collector.name" . }} -o wide + kubectl -n {{ include "k8s-collector.namespace" . }} get cronjob {{ include "k8s-collector.name" . }} -o wide To check logs from the most recent job run: - kubectl -n {{ .Values.serviceAccount.namespace }} logs job/$(kubectl -n {{ .Values.serviceAccount.namespace }} get jobs --sort-by=.metadata.creationTimestamp -l app.kubernetes.io/name={{ include "k8s-collector.name" . }} -o jsonpath="{.items[-1].metadata.name}") + kubectl -n {{ include "k8s-collector.namespace" . }} logs job/$(kubectl -n {{ include "k8s-collector.namespace" . }} get jobs --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1].metadata.name}") To run the job manually: - kubectl -n {{ .Values.serviceAccount.namespace }} create job --from=cronjob/{{ include "k8s-collector.name" . }} {{ include "k8s-collector.name" . }}-manual-$(date +%s) + kubectl -n {{ include "k8s-collector.namespace" . }} create job --from=cronjob/{{ include "k8s-collector.name" . }} {{ include "k8s-collector.name" . }}-manual-$(date +%s) diff --git a/helm-charts/nullify-k8s-collector/templates/_helpers.tpl b/helm-charts/nullify-k8s-collector/templates/_helpers.tpl index 8111d35..c2e883a 100644 --- a/helm-charts/nullify-k8s-collector/templates/_helpers.tpl +++ b/helm-charts/nullify-k8s-collector/templates/_helpers.tpl @@ -42,7 +42,7 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- with .Values.labels }} -{{- toYaml . }} +{{ toYaml . }} {{- end }} {{- end }} @@ -63,4 +63,51 @@ Create the name of the service account to use {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} -{{- end }} +{{- end }} + +{{/* +Namespace of every namespaced object: serviceAccount.namespace, or the release +namespace when that is empty. +*/}} +{{- define "k8s-collector.namespace" -}} +{{- default .Release.Namespace .Values.serviceAccount.namespace -}} +{{- end }} + +{{/* +AWS region of the Nullify bucket. S3 only accepts an SSE-KMS key from the +bucket's own region, so the region of the key or alias ARN is the default and +any other value is rejected. Call after "k8s-collector.validateCollector". +*/}} +{{- define "k8s-collector.awsRegion" -}} +{{- $keyRegion := index (splitList ":" .Values.collector.kms.keyArn) 3 -}} +{{- $region := .Values.collector.aws.region | default $keyRegion -}} +{{- if ne $region $keyRegion -}} +{{- fail (printf "collector.aws.region %q does not match the region of collector.kms.keyArn (%q). It must be the region of the Nullify S3 bucket, which is the key's region; leave it empty to use the key's region. An upgrade from chart 0.2.0 with --reuse-values carries that chart's default us-east-1: upgrade with --reset-then-reuse-values, or set collector.aws.region to the key's region." $region $keyRegion) -}} +{{- end -}} +{{- $region -}} +{{- end }} + +{{/* +Rejects missing or placeholder values the CronJob cannot run without. +*/}} +{{- define "k8s-collector.validateCollector" -}} +{{- $provider := .Values.cloudProvider | default "aws" -}} +{{- if not (has $provider (list "aws" "gcp")) -}} +{{- fail (printf "cloudProvider must be \"aws\" or \"gcp\", got %q" $provider) -}} +{{- end -}} +{{- $cluster := required "collector.clusterName is required: set it to the exact cluster name, which becomes the S3 key and is matched against your cloud inventory" .Values.collector.clusterName -}} +{{- if eq $cluster "YOUR-CLUSTER-NAME" -}} +{{- fail "collector.clusterName is still the placeholder YOUR-CLUSTER-NAME: set it to the exact cluster name" -}} +{{- end -}} +{{- $bucket := required "collector.s3.bucket is required: use the bucket shown on the Nullify configure page" .Values.collector.s3.bucket -}} +{{- if eq $bucket "YOUR-NULLIFY-S3-BUCKET" -}} +{{- fail "collector.s3.bucket is still the placeholder YOUR-NULLIFY-S3-BUCKET: use the bucket shown on the Nullify configure page" -}} +{{- end -}} +{{- $keyArn := required "collector.kms.keyArn is required: the collector refuses to upload without KMS encryption" .Values.collector.kms.keyArn -}} +{{- if not (regexMatch "^arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/(mrk-)?[a-f0-9-]+|alias/[A-Za-z0-9/_-]+)$" $keyArn) -}} +{{- fail (printf "collector.kms.keyArn must be a KMS key ARN (arn:aws:kms:::key/, including multi-Region mrk- keys) or alias ARN (arn:aws:kms:::alias/) from the Nullify configure page, got %q." $keyArn) -}} +{{- end -}} +{{- if eq $provider "gcp" -}} +{{- $_ := required "collector.gke.awsRoleArn is required when cloudProvider is gcp: Nullify provides it after you register the cluster's OIDC issuer" (.Values.collector.gke.awsRoleArn | default .Values.collector.gke.nullifyAwsRoleArn) -}} +{{- end -}} +{{- end }} diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrolebinding.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrolebinding.yaml index 7303e2a..d76086a 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrolebinding.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrolebinding.yaml @@ -12,5 +12,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "k8s-collector.serviceAccountName" . }} - namespace: {{ .Values.serviceAccount.namespace }} -{{- end }} \ No newline at end of file + namespace: {{ include "k8s-collector.namespace" . }} +{{- end }} diff --git a/helm-charts/nullify-k8s-collector/templates/cronjob.yaml b/helm-charts/nullify-k8s-collector/templates/cronjob.yaml index c776b2d..cb8ed3a 100644 --- a/helm-charts/nullify-k8s-collector/templates/cronjob.yaml +++ b/helm-charts/nullify-k8s-collector/templates/cronjob.yaml @@ -1,10 +1,11 @@ {{- if .Values.collector.enabled -}} +{{- include "k8s-collector.validateCollector" . -}} {{- $isGKE := eq (.Values.cloudProvider | default "aws") "gcp" -}} apiVersion: batch/v1 kind: CronJob metadata: name: {{ include "k8s-collector.name" . }} - namespace: {{ .Values.serviceAccount.namespace }} + namespace: {{ include "k8s-collector.namespace" . }} labels: {{- include "k8s-collector.labels" . | nindent 4 }} spec: @@ -19,7 +20,7 @@ spec: labels: {{- include "k8s-collector.selectorLabels" . | nindent 12 }} spec: - serviceAccountName: {{ .Values.serviceAccount.name }} + serviceAccountName: {{ include "k8s-collector.serviceAccountName" . }} containers: - name: collector image: "{{ .Values.collector.image.repository }}:{{ .Values.collector.image.tag }}" @@ -34,7 +35,7 @@ spec: - name: NULLIFY_S3_KEY_PREFIX value: "{{ .Values.collector.s3.keyPrefix }}" - name: AWS_REGION - value: "{{ .Values.collector.aws.region }}" + value: "{{ include "k8s-collector.awsRegion" . }}" - name: CLUSTER_NAME value: "{{ .Values.collector.clusterName }}" {{- if $isGKE }} @@ -46,22 +47,8 @@ spec: - name: AWS_WEB_IDENTITY_TOKEN_FILE value: "{{ .Values.collector.gke.webIdentityTokenPath }}" {{- end }} - {{- if and .Values.collector.dataCollection .Values.collector.dataCollection.excludeNamespaces }} - - name: EXCLUDE_NAMESPACES - value: "{{ .Values.collector.dataCollection.excludeNamespaces }}" - {{- end }} - {{- if and .Values.collector.dataCollection .Values.collector.dataCollection.includeResources }} - - name: INCLUDE_RESOURCES - value: "{{ .Values.collector.dataCollection.includeResources }}" - {{- end }} - {{- if and .Values.collector.dataCollection .Values.collector.dataCollection.metadataOnly }} - - name: METADATA_ONLY - value: "true" - {{- end }} - {{- if .Values.collector.kms.keyArn }} - name: NULLIFY_KMS_KEY_ARN value: "{{ .Values.collector.kms.keyArn }}" - {{- end }} {{- if .Values.collector.debug.enabled }} - name: ENABLE_DEBUG_LOG value: "true" @@ -74,6 +61,14 @@ spec: mountPath: {{ dir .Values.collector.gke.webIdentityTokenPath }} readOnly: true {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} {{- if $isGKE }} volumes: - name: gcp-sa-token diff --git a/helm-charts/nullify-k8s-collector/templates/namespace.yaml b/helm-charts/nullify-k8s-collector/templates/namespace.yaml deleted file mode 100644 index 5dee179..0000000 --- a/helm-charts/nullify-k8s-collector/templates/namespace.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.namespace.create -}} -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Values.serviceAccount.namespace }} - labels: - {{- include "k8s-collector.labels" . | nindent 4 }} - app.kubernetes.io/component: namespace - annotations: - # Add Helm hooks to ensure namespace is created at the beginning of the installation - "helm.sh/hook": "pre-install" - "helm.sh/hook-weight": "-10" - {{- with .Values.namespace.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/helm-charts/nullify-k8s-collector/templates/pre-install-job.yaml b/helm-charts/nullify-k8s-collector/templates/pre-install-job.yaml deleted file mode 100644 index f18b20a..0000000 --- a/helm-charts/nullify-k8s-collector/templates/pre-install-job.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.namespace.create -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "k8s-collector.fullname" . }}-namespace-creator - annotations: - "helm.sh/hook": "pre-install" - "helm.sh/hook-weight": "-20" - "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" -spec: - template: - metadata: - name: {{ include "k8s-collector.fullname" . }}-namespace-creator - spec: - serviceAccountName: default - restartPolicy: Never - containers: - - name: namespace-creator - image: bitnami/kubectl:latest - command: - - "/bin/sh" - - "-c" - - | - kubectl create namespace {{ .Values.serviceAccount.namespace }} || echo "Namespace already exists or cannot be created" - exit 0 - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/helm-charts/nullify-k8s-collector/templates/serviceaccount.yaml b/helm-charts/nullify-k8s-collector/templates/serviceaccount.yaml index 0e315b1..206d15c 100644 --- a/helm-charts/nullify-k8s-collector/templates/serviceaccount.yaml +++ b/helm-charts/nullify-k8s-collector/templates/serviceaccount.yaml @@ -1,9 +1,16 @@ {{- if .Values.serviceAccount.create -}} {{- $provider := .Values.cloudProvider | default "aws" -}} +{{- $values := .Values.serviceAccount.annotations | default dict -}} +{{- if and (eq $provider "aws") (ne (toString .Values.serviceAccount.requireRoleArn) "false") -}} +{{- $roleArn := index $values "eks.amazonaws.com/role-arn" | default "" | toString -}} +{{- if not (regexMatch "^arn:aws:iam::[0-9]{12}:role/.+" $roleArn) -}} +{{- fail (printf "serviceAccount.annotations.\"eks.amazonaws.com/role-arn\" must be the IAM role ARN from the Nullify configure page (arn:aws:iam:::role/AWSIntegration--NullifyReadOnlyRole), got %q. If the pod gets AWS credentials another way (EKS Pod Identity, a node instance profile, kube2iam or kiam), set serviceAccount.requireRoleArn=false." $roleArn) -}} +{{- end -}} +{{- end -}} {{- $annotations := dict -}} -{{- range $k, $v := .Values.serviceAccount.annotations -}} +{{- range $k, $v := $values -}} {{- if eq $k "eks.amazonaws.com/role-arn" -}} - {{- if eq $provider "aws" -}} + {{- if and (eq $provider "aws") (toString $v) -}} {{- $_ := set $annotations $k $v -}} {{- end -}} {{- else if eq $k "iam.gke.io/gcp-service-account" -}} @@ -17,8 +24,8 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.serviceAccount.name }} - namespace: {{ .Values.serviceAccount.namespace }} + name: {{ include "k8s-collector.serviceAccountName" . }} + namespace: {{ include "k8s-collector.namespace" . }} labels: {{- include "k8s-collector.labels" . | nindent 4 }} {{- with $annotations }} diff --git a/helm-charts/nullify-k8s-collector/values-example.yaml b/helm-charts/nullify-k8s-collector/values-example.yaml index 6b47e89..e74e399 100644 --- a/helm-charts/nullify-k8s-collector/values-example.yaml +++ b/helm-charts/nullify-k8s-collector/values-example.yaml @@ -2,48 +2,47 @@ # Copy this file to values-production.yaml and fill in your actual values # DO NOT commit values-production.yaml to version control # +# Install with: +# helm upgrade --install nullify-k8s-collector nullify/nullify-k8s-collector \ +# --namespace nullify --create-namespace -f values-production.yaml +# # The file below shows the EKS (IRSA) setup. See the bottom of this file # for the equivalent GKE (Workload Identity) setup. # # REQUIRED VALUES (get these from Nullify configure page): # - collector.clusterName: Your exact cluster name # - collector.s3.bucket: S3 bucket name -# - collector.kms.keyArn: KMS key ARN +# - collector.kms.keyArn: KMS key ARN (or alias ARN) # - EKS only: serviceAccount.annotations."eks.amazonaws.com/role-arn" # - GKE only: collector.gke.awsRoleArn cloudProvider: aws collector: - # AWS region (extract from your KMS key ARN) - aws: - region: "us-west-2" - + # Region of the Nullify S3 bucket, not your cluster's. Optional: defaults to + # the region in collector.kms.keyArn, and any other value is rejected. + # aws: + # region: "us-west-2" + # Must match your actual EKS cluster name exactly clusterName: "my-production-cluster" - + # S3 configuration (from Nullify configure page) s3: bucket: "your-nullify-bucket" keyPrefix: "k8s-collector" - + # KMS configuration (from Nullify configure page) kms: keyArn: "arn:aws:kms:us-west-2:123456789012:key/your-key-id" - + # Debug logging (optional) debug: enabled: false - + # Collection schedule (optional, default: daily at midnight) schedule: "0 2 * * *" - - # Data collection filters (optional) - dataCollection: - excludeNamespaces: "kube-system,kube-public,cert-manager" - # includeResources: "pods,services,deployments,configmaps" - # metadataOnly: true - + # Resource limits (optional) resources: limits: @@ -56,7 +55,7 @@ collector: # Service account with IRSA (from Nullify configure page) serviceAccount: annotations: - eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/NullifyReadOnlyRole" + eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/AWSIntegration-yourcompany-NullifyReadOnlyRole" # Node placement (optional) nodeSelector: @@ -84,8 +83,6 @@ labels: # cloudProvider: gcp # # collector: -# aws: -# region: "us-east-1" # AWS region of the Nullify S3 bucket # clusterName: "my-gke-cluster" # s3: # bucket: "your-nullify-bucket" @@ -95,4 +92,3 @@ labels: # gke: # # Provided by Nullify after you register your cluster's OIDC issuer URL. # awsRoleArn: "arn:aws:iam::123456789012:role/NullifyK8sCollectorRole" - diff --git a/helm-charts/nullify-k8s-collector/values.yaml b/helm-charts/nullify-k8s-collector/values.yaml index bbe2f70..1aaac4e 100644 --- a/helm-charts/nullify-k8s-collector/values.yaml +++ b/helm-charts/nullify-k8s-collector/values.yaml @@ -1,10 +1,10 @@ # Default values for nullify-k8s-collector # This is a YAML-formatted file. # -# REQUIRED VALUES (must be configured before deployment): +# REQUIRED VALUES (the chart refuses to render without them): # - collector.clusterName: Your exact cluster name # - collector.s3.bucket: S3 bucket from Nullify configure page -# - collector.kms.keyArn: KMS key ARN from Nullify configure page +# - collector.kms.keyArn: KMS key ARN (or alias ARN) from Nullify configure page # # Provider-specific required values: # - EKS: serviceAccount.annotations."eks.amazonaws.com/role-arn" @@ -15,6 +15,10 @@ # serviceAccount.annotations."iam.gke.io/gcp-service-account" # (Your GCP service account email, bound to the in-cluster # ServiceAccount via Workload Identity) +# +# The chart does not create its namespace. Install with +# helm upgrade --install nullify-k8s-collector nullify/nullify-k8s-collector \ +# --namespace nullify --create-namespace -f values.yaml # Which cluster platform the collector is running on. # Supported values: "aws" (EKS, default) | "gcp" (GKE). @@ -34,14 +38,17 @@ clusterRoleBinding: collector: enabled: true - # AWS region (extract from your KMS key ARN) + # Region of the Nullify S3 bucket (NOT your cluster's region). S3 only + # accepts a KMS key from the bucket's region, so leave this empty to use + # the region in collector.kms.keyArn; any other value is rejected. aws: - region: us-east-1 + region: "" # Must match your actual cluster name exactly - clusterName: "YOUR-CLUSTER-NAME" + clusterName: "" - # Container image + # Container image. Pin a versioned tag instead of k8s-collector-latest once + # Nullify publishes one. image: repository: public.ecr.aws/w4o2j2x4/integrations tag: k8s-collector-latest @@ -58,12 +65,17 @@ collector: # S3 configuration (from Nullify configure page) s3: - bucket: "YOUR-NULLIFY-S3-BUCKET" + bucket: "" keyPrefix: k8s-collector - # KMS configuration (from Nullify configure page) + # KMS key ARN from the Nullify configure page: + # arn:aws:kms:::key/, or an alias ARN + # arn:aws:kms:::alias/. With an alias, the IAM + # role's KMS statement must cover the resolved key, for example + # arn:aws:kms:::key/* (CloudFormation: connector releases + # that include PR #64). See "KMS alias ARNs" in the README. kms: - keyArn: "arn:aws:kms:REGION:ACCOUNT:key/KEY-ID" + keyArn: "" # Debug logging debug: @@ -81,12 +93,6 @@ collector: readOnlyRootFilesystem: true runAsNonRoot: true - # Data collection filters (optional) - dataCollection: - # excludeNamespaces: "kube-system,kube-public" - # includeResources: "pods,services,deployments" - # metadataOnly: true - # GKE-specific settings (only used when cloudProvider is "gcp"). gke: # Nullify-owned federated AWS IAM role the collector assumes via @@ -114,12 +120,6 @@ nameOverride: "" # Labels for all resources labels: {} -# Namespace configuration -namespace: - annotations: {} - create: true - requireNamespace: false - # Node selector nodeSelector: {} @@ -133,14 +133,23 @@ nodeSelector: {} # one that applies. Extra annotations are passed through unchanged. serviceAccount: annotations: - # Used when cloudProvider=aws (EKS). IAM role ARN from the Nullify - # configure page. - eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT:role/ROLE-NAME" + # Required when cloudProvider=aws (EKS). IAM role ARN from the Nullify + # configure page: arn:aws:iam:::role/AWSIntegration--NullifyReadOnlyRole + eks.amazonaws.com/role-arn: "" # Used when cloudProvider=gcp (GKE). Your GCP service account email, # e.g. nullify-k8s-collector@your-project.iam.gserviceaccount.com. # iam.gke.io/gcp-service-account: "YOUR-GCP-SA-EMAIL" create: true + # cloudProvider=aws: fail the render unless the role-arn annotation above is + # an arn:aws:iam:: role ARN. Set false only when + # the pod gets AWS credentials another way: EKS Pod Identity, a node + # instance profile, kube2iam or kiam. + requireRoleArn: true + # Keep the default name and namespace on EKS: the connector's CloudFormation + # IRSA trust only allows system:serviceaccount:nullify:nullify-k8s-collector-sa. name: nullify-k8s-collector-sa + # Namespace for every namespaced object. It must already exist (use + # --create-namespace). Empty means the release namespace. namespace: nullify # Pod tolerations diff --git a/tests/helm/nullify-k8s-collector/must-fail/defaults.yaml b/tests/helm/nullify-k8s-collector/must-fail/defaults.yaml new file mode 100644 index 0000000..542602b --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/defaults.yaml @@ -0,0 +1,4 @@ +# Helm renders templates in reverse path order, so the ServiceAccount's role-arn +# guard fires before the CronJob's clusterName guard. +# expect: must be the IAM role ARN from the Nullify configure page +{} diff --git a/tests/helm/nullify-k8s-collector/must-fail/eks-missing-role-arn.yaml b/tests/helm/nullify-k8s-collector/must-fail/eks-missing-role-arn.yaml new file mode 100644 index 0000000..79afa4d --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/eks-missing-role-arn.yaml @@ -0,0 +1,7 @@ +# expect: must be the IAM role ARN from the Nullify configure page +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:key/00000000-0000-0000-0000-000000000000 diff --git a/tests/helm/nullify-k8s-collector/must-fail/gke-missing-role-arn.yaml b/tests/helm/nullify-k8s-collector/must-fail/gke-missing-role-arn.yaml new file mode 100644 index 0000000..13c49aa --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/gke-missing-role-arn.yaml @@ -0,0 +1,8 @@ +# expect: collector.gke.awsRoleArn is required when cloudProvider is gcp +cloudProvider: gcp +collector: + clusterName: example-gke-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:us-east-1:123456789012:key/00000000-0000-0000-0000-000000000000 diff --git a/tests/helm/nullify-k8s-collector/must-fail/kms-bare-alias.yaml b/tests/helm/nullify-k8s-collector/must-fail/kms-bare-alias.yaml new file mode 100644 index 0000000..8545946 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/kms-bare-alias.yaml @@ -0,0 +1,10 @@ +# expect: collector.kms.keyArn must be a KMS key ARN +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: alias/example-customer-upload +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/kms-govcloud-alias.yaml b/tests/helm/nullify-k8s-collector/must-fail/kms-govcloud-alias.yaml new file mode 100644 index 0000000..1216f40 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/kms-govcloud-alias.yaml @@ -0,0 +1,10 @@ +# expect: collector.kms.keyArn must be a KMS key ARN +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws-us-gov:kms:us-gov-west-1:123456789012:alias/example-customer-upload +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/kms-placeholder.yaml b/tests/helm/nullify-k8s-collector/must-fail/kms-placeholder.yaml new file mode 100644 index 0000000..271c7b5 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/kms-placeholder.yaml @@ -0,0 +1,10 @@ +# expect: collector.kms.keyArn must be a KMS key ARN +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:REGION:ACCOUNT:key/KEY-ID +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/placeholder-bucket.yaml b/tests/helm/nullify-k8s-collector/must-fail/placeholder-bucket.yaml new file mode 100644 index 0000000..a707feb --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/placeholder-bucket.yaml @@ -0,0 +1,10 @@ +# expect: collector.s3.bucket is still the placeholder YOUR-NULLIFY-S3-BUCKET +collector: + clusterName: example-eks-cluster + s3: + bucket: YOUR-NULLIFY-S3-BUCKET + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/placeholder-cluster-name.yaml b/tests/helm/nullify-k8s-collector/must-fail/placeholder-cluster-name.yaml new file mode 100644 index 0000000..cfd2f9f --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/placeholder-cluster-name.yaml @@ -0,0 +1,10 @@ +# expect: collector.clusterName is still the placeholder YOUR-CLUSTER-NAME +collector: + clusterName: YOUR-CLUSTER-NAME + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/region-mismatch-alias.yaml b/tests/helm/nullify-k8s-collector/must-fail/region-mismatch-alias.yaml new file mode 100644 index 0000000..9f59271 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/region-mismatch-alias.yaml @@ -0,0 +1,12 @@ +# expect: does not match the region of collector.kms.keyArn +collector: + clusterName: example-eks-cluster + aws: + region: us-east-1 + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:alias/example-customer-upload +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/region-mismatch.yaml b/tests/helm/nullify-k8s-collector/must-fail/region-mismatch.yaml new file mode 100644 index 0000000..e507c79 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/region-mismatch.yaml @@ -0,0 +1,12 @@ +# expect: does not match the region of collector.kms.keyArn +collector: + clusterName: example-eks-cluster + aws: + region: us-east-1 + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:eu-central-1:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/reuse-values-from-0.2.0.yaml b/tests/helm/nullify-k8s-collector/must-fail/reuse-values-from-0.2.0.yaml new file mode 100644 index 0000000..53972d0 --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/reuse-values-from-0.2.0.yaml @@ -0,0 +1,16 @@ +# What `helm upgrade --reuse-values` hands 0.4.0 from a 0.2.0 install: 0.2.0's +# defaults (collector.aws.region us-east-1, namespace.create) plus the user's values. +# expect: upgrade with --reset-then-reuse-values +namespace: + create: true +collector: + aws: + region: us-east-1 + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:ap-southeast-2:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole diff --git a/tests/helm/nullify-k8s-collector/must-fail/role-arn-govcloud.yaml b/tests/helm/nullify-k8s-collector/must-fail/role-arn-govcloud.yaml new file mode 100644 index 0000000..47a553b --- /dev/null +++ b/tests/helm/nullify-k8s-collector/must-fail/role-arn-govcloud.yaml @@ -0,0 +1,10 @@ +# expect: must be the IAM role ARN from the Nullify configure page +collector: + clusterName: example-eks-cluster + s3: + bucket: example-nullify-bucket + kms: + keyArn: arn:aws:kms:us-east-2:123456789012:key/00000000-0000-0000-0000-000000000000 +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws-us-gov:iam::123456789012:role/AWSIntegration-example-NullifyReadOnlyRole From c38a83b359a36dc0f34bb47052393473ee507f82 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 16:36:32 +1000 Subject: [PATCH 29/36] fix(terraform): finish W58-2 in both collector-role trees #99 dropped the 11 never-collected kinds (cronjobs, HPA, PDB, ingressclasses, storageclasses, csidrivers, runtimeclasses, priorityclasses, flowschemas, certificatesigningrequests, CRDs) from the Helm ClusterRole only. The Terraform collector roles in both trees still granted them, so a k8s-resources install of #58 kept read access to kinds no collector reads, and the two collector roles differed by those 11 kinds. jobs is kept. Helm (0.3.0), TF v5 and TF v6 now grant identical kinds and verbs. Co-Authored-By: Claude Sonnet 5 --- .../modules/k8s-resources/main.tf | 51 +------------------ .../terraform/modules/k8s-resources/main.tf | 51 +------------------ 2 files changed, 2 insertions(+), 100 deletions(-) diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index ead382b..3ba84d6 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -66,19 +66,7 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { rule { api_groups = ["batch"] - resources = ["jobs", "cronjobs"] - verbs = ["list"] - } - - rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] - verbs = ["list"] - } - - rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] + resources = ["jobs"] verbs = ["list"] } @@ -86,7 +74,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { api_groups = ["networking.k8s.io"] resources = [ "ingresses", - "ingressclasses", "networkpolicies", ] verbs = ["list"] @@ -109,36 +96,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["list"] } - rule { - api_groups = ["storage.k8s.io"] - resources = ["storageclasses", "csidrivers"] - verbs = ["list"] - } - - rule { - api_groups = ["node.k8s.io"] - resources = ["runtimeclasses"] - verbs = ["list"] - } - - rule { - api_groups = ["scheduling.k8s.io"] - resources = ["priorityclasses"] - verbs = ["list"] - } - - rule { - api_groups = ["flowcontrol.apiserver.k8s.io"] - resources = ["flowschemas"] - verbs = ["list"] - } - - rule { - api_groups = ["certificates.k8s.io"] - resources = ["certificatesigningrequests"] - verbs = ["list"] - } - rule { api_groups = ["admissionregistration.k8s.io"] resources = [ @@ -150,12 +107,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["list"] } - rule { - api_groups = ["apiextensions.k8s.io"] - resources = ["customresourcedefinitions"] - verbs = ["list"] - } - rule { non_resource_urls = ["/version"] verbs = ["get"] diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index ead382b..3ba84d6 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -66,19 +66,7 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { rule { api_groups = ["batch"] - resources = ["jobs", "cronjobs"] - verbs = ["list"] - } - - rule { - api_groups = ["autoscaling"] - resources = ["horizontalpodautoscalers"] - verbs = ["list"] - } - - rule { - api_groups = ["policy"] - resources = ["poddisruptionbudgets"] + resources = ["jobs"] verbs = ["list"] } @@ -86,7 +74,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { api_groups = ["networking.k8s.io"] resources = [ "ingresses", - "ingressclasses", "networkpolicies", ] verbs = ["list"] @@ -109,36 +96,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["list"] } - rule { - api_groups = ["storage.k8s.io"] - resources = ["storageclasses", "csidrivers"] - verbs = ["list"] - } - - rule { - api_groups = ["node.k8s.io"] - resources = ["runtimeclasses"] - verbs = ["list"] - } - - rule { - api_groups = ["scheduling.k8s.io"] - resources = ["priorityclasses"] - verbs = ["list"] - } - - rule { - api_groups = ["flowcontrol.apiserver.k8s.io"] - resources = ["flowschemas"] - verbs = ["list"] - } - - rule { - api_groups = ["certificates.k8s.io"] - resources = ["certificatesigningrequests"] - verbs = ["list"] - } - rule { api_groups = ["admissionregistration.k8s.io"] resources = [ @@ -150,12 +107,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { verbs = ["list"] } - rule { - api_groups = ["apiextensions.k8s.io"] - resources = ["customresourcedefinitions"] - verbs = ["list"] - } - rule { non_resource_urls = ["/version"] verbs = ["get"] From ea9b3059f63bf7649c57a43b257fec6975b72bd2 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 17:02:23 +1000 Subject: [PATCH 30/36] docs(terraform): describe today's Secrets handling and list batch/jobs in managed-scan RBAC - Drop the server-side Table fetch claim and the per-cluster opt-out claim: neither exists in the deployed collector or managed scan. - Kinds-read row: identical from k8s-resources and chart 0.3.0; 0.2.0 grants extra kinds. - Add the batch/jobs row managed_scan.tf already grants. Co-Authored-By: Claude Opus 5 (1M context) --- aws-integration-setup/terraform/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 105f743..8a50627 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -86,7 +86,7 @@ terraform/ | Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | | AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | | Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | A list-only `nullify-readonly` ClusterRole and ClusterRoleBinding, applied by `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true` | -| Kinds read | Same kind list as the managed scan (both derive from the platform's collector) | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | +| Kinds read | Same kind list as the managed scan in `k8s-resources` and chart 0.3.0 and later (chart 0.2.0 grants extra kinds it never collects) | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | | Upgrades | You update the image | None | Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. @@ -103,6 +103,7 @@ The access entry carries the Kubernetes group `nullify-readonly`, bound to the C |---|---| | core | nodes, namespaces, pods, services, persistentvolumeclaims, persistentvolumes, configmaps, secrets, resourcequotas, limitranges, serviceaccounts | | apps | deployments, daemonsets, statefulsets, replicasets | +| batch | jobs | | networking.k8s.io | ingresses, networkpolicies | | discovery.k8s.io | endpointslices | | rbac.authorization.k8s.io | roles, rolebindings, clusterroles, clusterrolebindings | @@ -110,7 +111,7 @@ The access entry carries the Kubernetes group `nullify-readonly`, bound to the C Plus `get` on the non-resource URL `/version`. -Both modes list Secrets, but neither reads values. The managed scan requests Kubernetes' server-side [Table representation](https://kubernetes.io/docs/reference/using-api/api-concepts/#alternate-representations-of-resources) for Secrets, so only metadata (name, type, key count) crosses the API server boundary; values never leave the cluster. The in-cluster collector fetches full Secret objects but redacts every value before it uploads collected data. The `list` verb technically permits reading values in either mode -- Kubernetes has no metadata-only RBAC verb for Secrets -- but Nullify's collection code never exercises that. +Both modes list Secrets. The in-cluster collector redacts every Secret value inside the cluster before it uploads collected data. The managed scan receives Secret objects from the API server and redacts every value before anything is stored. The `list` verb permits reading values in either mode -- Kubernetes has no metadata-only RBAC verb for Secrets. ### `authorization = "admin_view_policy"` (opt-in) From e02477b5f0b142af229e3f815924cba40f28850d Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Tue, 15 Sep 2026 17:27:56 +1000 Subject: [PATCH 31/36] fix(terraform): match #58's networking rule formatting and state chart 0.2.0's kinds - k8s-resources main.tf (v5, v6): write the networking.k8s.io resources list the way #58 does, so a squash merge of #58 does not conflict on a formatting-only hunk. - README kinds-read row: identical from chart 0.4.0 (0.3.0 is never published); 0.2.0 grants a different set, not a superset. Co-Authored-By: Claude Opus 5 (1M context) --- .../terraform-v6/modules/k8s-resources/main.tf | 7 +++++-- aws-integration-setup/terraform/README.md | 2 +- .../terraform/modules/k8s-resources/main.tf | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index 264cf84..db5d27d 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -88,8 +88,11 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { rule { api_groups = ["networking.k8s.io"] - resources = ["ingresses", "networkpolicies"] - verbs = ["list"] + resources = [ + "ingresses", + "networkpolicies", + ] + verbs = ["list"] } rule { diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 8a50627..7278132 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -86,7 +86,7 @@ terraform/ | Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | | AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | | Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | A list-only `nullify-readonly` ClusterRole and ClusterRoleBinding, applied by `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true` | -| Kinds read | Same kind list as the managed scan in `k8s-resources` and chart 0.3.0 and later (chart 0.2.0 grants extra kinds it never collects) | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | +| Kinds read | Same kind list as the managed scan in `k8s-resources` and chart 0.4.0 and later. Chart 0.2.0 grants a different set: it adds endpoints, cronjobs and CRDs, and lacks configmaps, secrets, resourcequotas, limitranges, endpointslices and the admission kinds | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | | Upgrades | You update the image | None | Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index 264cf84..db5d27d 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -88,8 +88,11 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { rule { api_groups = ["networking.k8s.io"] - resources = ["ingresses", "networkpolicies"] - verbs = ["list"] + resources = [ + "ingresses", + "networkpolicies", + ] + verbs = ["list"] } rule { From c2178139223889ed34eeab95aec7a3c4fff0a452 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Thu, 17 Sep 2026 14:45:53 +1000 Subject: [PATCH 32/36] fix(aws): drop AdminView and tighten secret-shaped IAM RBAC is the only EKS authorization mode. Scope GetDocument, deny UserData and EventBridge target Input, pin KMS key/* to alias ARNs, and allow ECR image pull for scanning. Co-authored-by: Cursor --- .../nullify-cloudformation-template.json | 20 +++++--- aws-integration-setup/terraform-v6/README.md | 4 +- .../examples/managed-scan/main.tf | 2 +- .../examples/managed-scan/outputs.tf | 2 +- .../managed-scan/terraform.tfvars.example | 5 +- .../examples/managed-scan/variables.tf | 6 +-- .../modules/eks-managed-scan-access/main.tf | 26 +--------- .../eks-managed-scan-access/outputs.tf | 6 +-- .../tests/eks_managed_scan_access.tftest.hcl | 37 ++------------ .../eks-managed-scan-access/variables.tf | 10 ++-- .../modules/nullify-aws-integration/data.tf | 16 ++++-- .../modules/nullify-aws-integration/locals.tf | 26 +++++----- .../nullify-aws-integration/outputs.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 49 ++++++++++++++++--- .../nullify-aws-integration/variables.tf | 2 +- aws-integration-setup/terraform-v6/outputs.tf | 2 +- .../terraform-v6/terraform.tfvars.example | 2 +- .../terraform-v6/variables.tf | 9 +++- aws-integration-setup/terraform/README.md | 20 ++++---- .../terraform/examples/managed-scan/main.tf | 2 +- .../examples/managed-scan/outputs.tf | 2 +- .../managed-scan/terraform.tfvars.example | 5 +- .../examples/managed-scan/variables.tf | 6 +-- .../modules/eks-managed-scan-access/main.tf | 25 +--------- .../eks-managed-scan-access/outputs.tf | 6 +-- .../tests/eks_managed_scan_access.tftest.hcl | 37 ++------------ .../eks-managed-scan-access/variables.tf | 10 ++-- .../modules/nullify-aws-integration/data.tf | 16 ++++-- .../modules/nullify-aws-integration/locals.tf | 26 +++++----- .../nullify-aws-integration/outputs.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 49 ++++++++++++++++--- .../nullify-aws-integration/variables.tf | 2 +- aws-integration-setup/terraform/outputs.tf | 2 +- .../terraform/terraform.tfvars.example | 2 +- aws-integration-setup/terraform/variables.tf | 9 +++- helm-charts/nullify-k8s-collector/README.md | 13 +++-- 36 files changed, 221 insertions(+), 239 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 0cb371c..819f735 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -47,7 +47,7 @@ }, "NullifyKMSKeyArn": { "Type": "String", - "Description": "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed).", + "Description": "The ARN of the KMS key for key management operations (optional, provided by Nullify if needed). A key ARN is preferred over an alias ARN because IAM cannot resolve aliases in a policy Resource.", "Default": "", "AllowedPattern": "^(arn:aws:kms:[a-z0-9-]+:[0-9]{12}:(key/[a-f0-9-]+|alias/.+))?$", "ConstraintDescription": "Must be a valid KMS key ARN (key ID or alias) or empty string." @@ -125,9 +125,6 @@ "s3:GetObject*", "s3:DeleteObject*", "s3:RestoreObject", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:GetAuthorizationToken", "ecr:PutImage", "ecr:InitiateLayerUpload", "ecr:UploadLayerPart", @@ -162,6 +159,7 @@ "ec2:DeleteSnapshot", "ec2:CreateImage", "ec2:DeregisterImage", + "ec2:DescribeLaunchTemplateVersions", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", @@ -175,6 +173,7 @@ "dynamodb:Query", "dynamodb:Scan", "es:ESHttp*", + "events:ListTargetsByRule", "glue:GetConnection", "glue:GetConnections", "glue:GetPartition*", @@ -434,7 +433,6 @@ "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", - "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeLaunchTemplates", "ec2:DescribeManagedPrefixLists", "ec2:DescribeNatGateways", @@ -458,6 +456,9 @@ "ec2:GetSnapshotBlockPublicAccessState", "ec2:SearchTransitGateway*", "ecr:Describe*", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:GetAuthorizationToken", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", "ecs:Describe*", @@ -566,7 +567,6 @@ "sqs:GetQueueAttributes", "sqs:List*", "ssm:Describe*", - "ssm:GetDocument", "ssm:ListDocuments", "states:DescribeStateMachine", "states:List*", @@ -576,6 +576,14 @@ ], "Resource": "*", "Condition": {} + }, + { + "Effect": "Allow", + "Action": [ + "ssm:GetDocument" + ], + "Resource": "arn:*:ssm:*:*:document/SSM-SessionManagerRunShell", + "Condition": {} } ] } diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index 9ac3524..fb133a1 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -92,7 +92,7 @@ module "nullify_eks_access" { } ``` -The default `authorization = "rbac"` needs the list-only `nullify-readonly` ClusterRole on each cluster: apply it with `k8s-resources` (`enable_collector = false, enable_managed_scan_rbac = true`), or with any equivalent manifest of your own. `admin_view_policy` is a warned opt-in. The clusters must already exist when you plan; see `../terraform/README.md`. +`authorization = "rbac"` is the only supported mode. It needs the list-only `nullify-readonly` ClusterRole on each cluster: apply it with `k8s-resources` (`enable_collector = false, enable_managed_scan_rbac = true`), or with any equivalent manifest of your own. `AmazonEKSAdminViewPolicy` is not supported; if an existing cluster still has it associated with the Nullify role, disassociate it. The clusters must already exist when you plan; see `../terraform/README.md`. ## Required Variables @@ -126,7 +126,7 @@ A cluster cannot run the collector and the managed scan at once: the two paths r - `k8s-resources` now requires `cluster_name` while `enable_collector` is true, and sets it as the collector's `CLUSTER_NAME`. A deployment that never set it was uploading to `k8s-collector/default-name-data.json`; after this change it uploads to `k8s-collector/-data.json`. The old object is in Nullify's bucket, which your role cannot read or delete: ask Nullify to remove the stale `default-name` cluster, and register the new name on the configure page before the next run. - `k8s-resources` now requires `kms_key_arn` while `enable_collector` is true, with no opt-out. The shipped collector images refuse to upload without KMS encryption, so a deployment that left it empty was collecting successfully and then failing every upload. Set the value from the configure page; ask Nullify for a key if you have none. - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. -- The KMS policy grants `key/*` in the account and region of `kms_key_arn` whenever that account is not your own, for alias ARNs and key ARNs alike. An ARN in your own account grants only that key. +- The KMS policy grants `key/*` in the account and region of `kms_key_arn` only when that ARN is an alias. A concrete key ARN is preferred: it grants that ARN only. - The root no longer declares a `kubernetes` provider or `hashicorp/kubernetes` requirement: this tree's root never applied Kubernetes objects, so remove any provider configuration you added to satisfy it. ## Validation diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf index 86b8041..0113e40 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/main.tf @@ -64,7 +64,7 @@ module "eks_managed_scan_access" { # Argo CD applies the nullify-readonly ClusterRole instead. module "k8s_resources" { source = "../../modules/k8s-resources" - count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 + count = var.manage_rbac ? 1 : 0 enable_collector = false enable_managed_scan_rbac = true diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf index 91b6cdc..bc94e92 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/outputs.tf @@ -9,7 +9,7 @@ output "clusters_for_nullify" { } output "kubernetes_group_name" { - description = "Kubernetes group the RBAC binding names (null for admin_view_policy)" + description = "Kubernetes group the RBAC binding names" value = module.eks_managed_scan_access.kubernetes_group_name } diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example index 4fe07b7..94d13cc 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/terraform.tfvars.example @@ -16,9 +16,8 @@ eks_cluster_arn = "arn:aws:eks:eu-west-1:123456789012:cluster/prod" # REQUIRED: your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2) nullify_region = "eu-central-1" -# rbac (recommended): list-only ClusterRole bound to the access entry's group. -# admin_view_policy: AmazonEKSAdminViewPolicy, no Kubernetes objects, but it reads -# every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. +# rbac is the only supported authorization mode: list-only ClusterRole bound +# to the access entry's group. AmazonEKSAdminViewPolicy is not supported. # authorization = "rbac" # Set false if Helm, Flux or Argo CD applies the nullify-readonly ClusterRole instead. diff --git a/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf b/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf index f1b6096..4a4bb3c 100644 --- a/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf +++ b/aws-integration-setup/terraform-v6/examples/managed-scan/variables.tf @@ -35,19 +35,19 @@ variable "nullify_region" { variable "authorization" { type = string - description = "rbac (recommended) or admin_view_policy. admin_view_policy grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" + description = "rbac is the only supported mode. AmazonEKSAdminViewPolicy is not supported" default = "rbac" } variable "kubernetes_group_name" { type = string - description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole in rbac mode" + description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole" default = "nullify-readonly" } variable "manage_rbac" { type = bool - description = "In rbac mode, create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" + description = "Create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" default = true } diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf index bd32446..d4921c5 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/main.tf @@ -57,7 +57,7 @@ resource "aws_eks_access_entry" "nullify" { cluster_name = data.aws_eks_cluster.this[each.key].name principal_arn = var.principal_arn type = "STANDARD" - kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : [] + kubernetes_groups = [var.kubernetes_group_name] tags = var.tags lifecycle { @@ -70,23 +70,6 @@ resource "aws_eks_access_entry" "nullify" { } } -resource "aws_eks_access_policy_association" "admin_view" { - for_each = { for key, cluster in local.clusters : key => cluster if var.authorization == "admin_view_policy" } - - region = each.value.region - cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name - principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn - policy_arn = "arn:${each.value.partition}:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" - - access_scope { - type = "cluster" - } - - lifecycle { - replace_triggered_by = [aws_eks_access_entry.nullify[each.key].access_entry_arn] - } -} - locals { endpoints = { for key, cluster in data.aws_eks_cluster.this : key => { @@ -105,10 +88,3 @@ check "nullify_can_reach_cluster_endpoints" { error_message = "Nullify's ${var.nullify_region} egress IPs cannot reach the public endpoint of: ${join(", ", local.unreachable_clusters)}. This module does not change publicAccessCidrs; see the endpoint_allowlist output for the missing CIDRs and an update command. The check matches exact CIDRs, so a wider range that already covers Nullify's IPs also warns." } } - -check "admin_view_policy_is_broader_than_required" { - assert { - condition = var.authorization != "admin_view_policy" - error_message = "authorization = admin_view_policy associates AmazonEKSAdminViewPolicy: get, list and watch on every resource, including Secrets, custom resources and pods/log. On EKS 1.34 and earlier get pods/exec is enough to exec into pods, and these grants do not show in kubectl auth can-i --list. The default rbac mode grants list on only the kinds Nullify reads." - } -} diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf index 6e6d1bc..3431d1d 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/outputs.tf @@ -4,13 +4,13 @@ output "access_entry_arns" { } output "authorization" { - description = "How Kubernetes authorizes Nullify: rbac or admin_view_policy" + description = "How Kubernetes authorizes Nullify (rbac is the only supported mode)" value = var.authorization } output "kubernetes_group_name" { - description = "Group the RBAC binding must name (null for admin_view_policy)" - value = var.authorization == "rbac" ? var.kubernetes_group_name : null + description = "Group the RBAC binding must name" + value = var.kubernetes_group_name } output "nullify_egress_cidrs" { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 351d2ce..3598106 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -83,11 +83,6 @@ run "rbac_is_the_default" { error_message = "rbac mode must put the entry in the nullify-readonly group" } - assert { - condition = length(aws_eks_access_policy_association.admin_view) == 0 - error_message = "rbac mode must not associate any access policy" - } - assert { condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null error_message = "An endpoint open to 0.0.0.0/0 needs no update" @@ -149,34 +144,14 @@ run "private_endpoint_warns_without_a_command" { } } -run "admin_view_policy_is_cluster_scoped_and_warns" { +run "admin_view_policy_is_rejected" { command = plan variables { authorization = "admin_view_policy" } - expect_failures = [check.admin_view_policy_is_broader_than_required] - - assert { - condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].policy_arn == "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" - error_message = "admin_view_policy must associate AmazonEKSAdminViewPolicy" - } - - assert { - condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].access_scope[0].type == "cluster" - error_message = "The access policy must be cluster scoped" - } - - assert { - condition = output.kubernetes_group_name == null - error_message = "admin_view_policy binds no Kubernetes group" - } - - assert { - condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 - error_message = "admin_view_policy must plan kubernetes_groups as an empty set on the entry: the attribute is Optional+Computed, so null would keep whatever is in state. This pins the planned configuration under a mocked provider, not the applied result" - } + expect_failures = [var.authorization] } run "view_policy_is_not_offered" { @@ -209,18 +184,14 @@ run "cross_account_clusters_are_rejected" { expect_failures = [data.aws_eks_cluster.this] } -run "a_collector_cluster_is_rejected_even_under_admin_view_policy" { - # admin_view_policy creates no Kubernetes objects, so the k8s-resources - # ClusterRole's own precondition can never see this combination. The guard - # has to live here, on the access entry every authorization mode creates. +run "a_collector_cluster_is_rejected" { command = plan variables { - authorization = "admin_view_policy" collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] } - expect_failures = [data.aws_eks_cluster.this, check.admin_view_policy_is_broader_than_required] + expect_failures = [data.aws_eks_cluster.this] } run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { diff --git a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf index 8d156d4..a382171 100644 --- a/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/eks-managed-scan-access/variables.tf @@ -26,18 +26,18 @@ variable "cluster_arns" { variable "authorization" { type = string - description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + description = "How Kubernetes authorizes the access entry. rbac is the only supported mode: the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. AmazonEKSAdminViewPolicy is not supported: it grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" default = "rbac" validation { - condition = contains(["rbac", "admin_view_policy"], var.authorization) - error_message = "authorization must be rbac or admin_view_policy. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." + condition = var.authorization == "rbac" + error_message = "authorization must be rbac. AmazonEKSAdminViewPolicy is not supported: it grants get, list and watch on every resource, including Secrets and pods/log, and on EKS 1.34 and earlier allows exec into pods. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." } } variable "kubernetes_group_name" { type = string - description = "Kubernetes group on the access entry when authorization is rbac" + description = "Kubernetes group on the access entry" default = "nullify-readonly" validation { @@ -64,7 +64,7 @@ variable "tags" { variable "collector_cluster_arns" { type = list(string) - description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match -- this tree supports collector_cluster_arns spanning multiple regions in one instance, same as cluster_arns" + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match -- this tree supports collector_cluster_arns spanning multiple regions in one instance, same as cluster_arns" default = [] validation { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index f3654d5..3655afa 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -100,7 +100,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", - "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeLaunchTemplates", "ec2:DescribeManagedPrefixLists", "ec2:DescribeNatGateways", @@ -124,6 +123,9 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ec2:GetSnapshotBlockPublicAccessState", "ec2:SearchTransitGateway*", "ecr:Describe*", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:GetAuthorizationToken", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", "ecs:Describe*", @@ -220,7 +222,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "sqs:GetQueueAttributes", "sqs:List*", "ssm:Describe*", - "ssm:GetDocument", "ssm:ListDocuments", "states:DescribeStateMachine", "states:List*", @@ -230,6 +231,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { ] resources = ["*"] } + + statement { + effect = "Allow" + actions = ["ssm:GetDocument"] + resources = ["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"] + } } data "aws_iam_policy_document" "s3_access_policy" { @@ -288,9 +295,6 @@ data "aws_iam_policy_document" "deny_actions_policy" { "s3:GetObject*", "s3:DeleteObject*", "s3:RestoreObject", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:GetAuthorizationToken", "ecr:PutImage", "ecr:InitiateLayerUpload", "ecr:UploadLayerPart", @@ -325,6 +329,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "ec2:DeleteSnapshot", "ec2:CreateImage", "ec2:DeregisterImage", + "ec2:DescribeLaunchTemplateVersions", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", @@ -338,6 +343,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "dynamodb:Query", "dynamodb:Scan", "es:ESHttp*", + "events:ListTargetsByRule", "glue:GetConnection", "glue:GetConnections", "glue:GetPartition*", diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf index f88960e..c0e651e 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/locals.tf @@ -20,20 +20,18 @@ locals { enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. The key Nullify hands out lives in Nullify's account, so - # key/* is granted next to the ARN: IAM never resolves an alias ARN in a - # Resource, and the key policy on a key Nullify owns is the real gate on what - # this role can use. The wildcard is derived only when the ARN's account - # differs from this one, so a key pasted from the customer's own account - # grants that key alone and reaches nothing else in the account. For a - # Nullify-supplied key this grants what the CloudFormation template grants. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - kms_arn_is_full = length(local.kms_arn_parts) == 6 - kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" - kms_arn_is_cross_account = local.kms_arn_is_full && local.kms_arn_account != data.aws_caller_identity.current.account_id - kms_key_wildcard = local.kms_arn_is_cross_account ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] - kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] + # KMS configuration. IAM never resolves an alias ARN in a Resource, so + # key/* is granted next to an alias ARN in that account and region. A + # concrete key ARN grants that ARN only: it must not open every key in + # Nullify's account. Nullify's key policy is the real gate on a key + # Nullify owns. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_full = length(local.kms_arn_parts) == 6 + kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" + kms_arn_is_alias = local.kms_arn_is_full && strcontains(var.kms_key_arn, ":alias/") + kms_key_wildcard = local.kms_arn_is_alias ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf index db5825b..fa4561a 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/outputs.tf @@ -81,7 +81,7 @@ output "policy_arns" { } output "kms_policy_resources" { - description = "Resources the KMS policy grants: kms_key_arn, plus key/* in its account and region when that account is not this one (empty when kms_key_arn is not set)" + description = "Resources the KMS policy grants: a key ARN is granted as-is; an alias ARN also gets key/* in its account and region because IAM cannot resolve aliases. Empty when kms_key_arn is not set. A key ARN is preferred because it avoids the wildcard" value = local.kms_policy_resources } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 1a5c5fb..962df70 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -47,7 +47,7 @@ run "cross_account_alias_grants_the_alias_and_key_wildcard" { } } -run "cross_account_key_arn_grants_the_key_wildcard_too" { +run "a_key_arn_grants_only_that_key" { command = plan variables { @@ -55,12 +55,12 @@ run "cross_account_key_arn_grants_the_key_wildcard_too" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab", "arn:aws:kms:us-east-2:111122223333:key/*"]) - error_message = "A key ARN in Nullify's account must grant key/* too: this is what the CloudFormation template grants for the same input" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) + error_message = "A concrete key ARN must grant that ARN only: key/* would open every key in Nullify's account" } } -run "a_key_in_this_account_grants_only_that_key" { +run "an_alias_in_this_account_still_grants_key_wildcard" { command = plan variables { @@ -68,8 +68,8 @@ run "a_key_in_this_account_grants_only_that_key" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key"]) - error_message = "An ARN in the caller's own account must never derive key/*: that would reach every key in the account whose policy delegates to the root" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key", "arn:aws:kms:us-east-2:123456789012:key/*"]) + error_message = "An alias ARN must grant key/* because IAM does not resolve aliases" } } @@ -148,8 +148,7 @@ run "kubernetes_integration_needs_clusters" { # passing test only proves the module plans, not that a policy grants any # action. `statement` blocks are the data source's own configuration -- they # are plan-time known even when the resulting document is mocked -- so assert -# on those instead. Only the KMS, S3 access point and readonly policies are -# covered here: the trust and deny-list policies aren't in W58/N78's scope. +# on those instead. run "kms_policy_grants_the_expected_actions_and_resources" { command = plan @@ -208,3 +207,37 @@ run "readonly_policy_grants_a_read_only_action" { error_message = "The readonly policy statement must Allow, not Deny" } } + +run "permission_bar_scopes_secret_shaped_reads" { + command = plan + + assert { + condition = !contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ec2:DescribeLaunchTemplateVersions") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ec2:DescribeLaunchTemplates") + error_message = "DescribeLaunchTemplateVersions returns UserData in the clear and must not be allowed; DescribeLaunchTemplates stays" + } + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:GetDownloadUrlForLayer") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:BatchGetImage") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:GetAuthorizationToken") + error_message = "Image pull and ECR auth must be allowed so scanning can fetch customer images" + } + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part2.statement[0].actions, "lambda:GetFunction") && !contains(data.aws_iam_policy_document.readonly_policy_part2.statement[0].actions, "ssm:GetDocument") + error_message = "lambda:GetFunction stays on *; ssm:GetDocument must leave the wildcard allow" + } + + assert { + condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 + error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell" + } + + assert { + condition = contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ec2:DescribeLaunchTemplateVersions") && contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "events:ListTargetsByRule") + error_message = "UserData and EventBridge target Input must be denied" + } + + assert { + condition = !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:GetDownloadUrlForLayer") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:BatchGetImage") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:GetAuthorizationToken") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "lambda:GetFunction") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ssm:GetDocument") + error_message = "Image pull, lambda:GetFunction and scoped GetDocument must not be denied" + } +} diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf index 95c8808..fad6342 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/variables.tf @@ -97,7 +97,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and key/* in that same account and region, for key ARNs and alias ARNs alike: IAM does not resolve an alias in a policy Resource, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The kms_policy_resources output lists what was granted" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is preferred because it avoids the wildcard. IAM does not resolve an alias in a policy Resource, so an alias ARN also grants key/* in that same account and region. A concrete key ARN grants that ARN only and does not open every key in Nullify's account. Nullify's key policy is the real gate on a key Nullify owns. The kms_policy_resources output lists what was granted" default = "" validation { diff --git a/aws-integration-setup/terraform-v6/outputs.tf b/aws-integration-setup/terraform-v6/outputs.tf index c3f7bf0..18d20c9 100644 --- a/aws-integration-setup/terraform-v6/outputs.tf +++ b/aws-integration-setup/terraform-v6/outputs.tf @@ -58,7 +58,7 @@ output "managed_scan_clusters_for_nullify" { } output "managed_scan_kubernetes_group" { - description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled or for admin_view_policy)" + description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled)" value = one(module.eks_managed_scan_access[*].kubernetes_group_name) } diff --git a/aws-integration-setup/terraform-v6/terraform.tfvars.example b/aws-integration-setup/terraform-v6/terraform.tfvars.example index 0fd1d2f..f7a5627 100644 --- a/aws-integration-setup/terraform-v6/terraform.tfvars.example +++ b/aws-integration-setup/terraform-v6/terraform.tfvars.example @@ -33,7 +33,7 @@ enable_kubernetes_integration = false # ] # Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. -# Clusters can be in any region but must already exist when you plan. In rbac mode, bind the group +# Clusters can be in any region but must already exist when you plan. Bind the group # to a list-only nullify-readonly ClusterRole: the k8s-resources module applies one with # enable_managed_scan_rbac = true. # enable_managed_scan = true diff --git a/aws-integration-setup/terraform-v6/variables.tf b/aws-integration-setup/terraform-v6/variables.tf index 8b12167..5e1ef8a 100644 --- a/aws-integration-setup/terraform-v6/variables.tf +++ b/aws-integration-setup/terraform-v6/variables.tf @@ -116,13 +116,18 @@ variable "collector_cluster_arns" { variable "managed_scan_authorization" { type = string - description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + description = "rbac is the only supported mode: the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. AmazonEKSAdminViewPolicy is not supported" default = "rbac" + + validation { + condition = var.managed_scan_authorization == "rbac" + error_message = "managed_scan_authorization must be rbac. AmazonEKSAdminViewPolicy is not supported." + } } variable "managed_scan_kubernetes_group" { type = string - description = "Kubernetes group on the access entries in rbac mode" + description = "Kubernetes group on the access entries" default = "nullify-readonly" } diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 7278132..0ad2eed 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -86,7 +86,7 @@ terraform/ | Cluster endpoint | Any, including private-only | Public endpoint that admits Nullify's egress IPs | | AWS setup | `enable_kubernetes_integration = true`, S3 bucket or access point | `eks-managed-scan-access`: one access entry per cluster | | Kubernetes setup | `k8s-resources` (default) or the `nullify-k8s-collector` Helm chart | A list-only `nullify-readonly` ClusterRole and ClusterRoleBinding, applied by `k8s-resources` with `enable_collector = false, enable_managed_scan_rbac = true` | -| Kinds read | Same kind list as the managed scan in `k8s-resources` and chart 0.4.0 and later. Chart 0.2.0 grants a different set: it adds endpoints, cronjobs and CRDs, and lacks configmaps, secrets, resourcequotas, limitranges, endpointslices and the admission kinds | See [RBAC granted to Nullify](#rbac-granted-to-nullify-default-authorization--rbac) | +| Kinds read | Same kind list as the managed scan in `k8s-resources` and chart 0.4.0 and later. Chart 0.2.0 grants a different set: it adds endpoints, cronjobs and CRDs, and lacks configmaps, secrets, resourcequotas, limitranges, endpointslices and the admission kinds | See [RBAC granted to Nullify](#rbac-granted-to-nullify-authorization--rbac) | | Upgrades | You update the image | None | Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, the managed scan uses `sts:AssumeRole` with the external ID. @@ -95,7 +95,7 @@ Both modes can run on the same role: IRSA uses `sts:AssumeRoleWithWebIdentity`, Nullify lists Kubernetes resources from its own compute, using the read-only role this configuration creates. The scan only calls `list`, plus `get /version`. -### RBAC granted to Nullify (default, `authorization = "rbac"`) +### RBAC granted to Nullify (`authorization = "rbac"`) The access entry carries the Kubernetes group `nullify-readonly`, bound to the ClusterRole `nullify-readonly`: @@ -113,9 +113,7 @@ Plus `get` on the non-resource URL `/version`. Both modes list Secrets. The in-cluster collector redacts every Secret value inside the cluster before it uploads collected data. The managed scan receives Secret objects from the API server and redacts every value before anything is stored. The `list` verb permits reading values in either mode -- Kubernetes has no metadata-only RBAC verb for Secrets. -### `authorization = "admin_view_policy"` (opt-in) - -Associates `AmazonEKSAdminViewPolicy` at cluster scope, so no Kubernetes objects are needed. **It grants far more than the scan uses:** `get`, `list` and `watch` on every resource, including Secrets, custom resources and `pods/log`. On EKS 1.34 and earlier, `get pods/exec` is enough to exec into pods. Its grants do not show in `kubectl auth can-i --list`. The module prints a warning on every plan in this mode. `AmazonEKSViewPolicy` is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects. +`authorization` accepts only `rbac`. `AmazonEKSAdminViewPolicy` is not supported: it grants `get`, `list` and `watch` on every resource, including Secrets, custom resources and `pods/log`, and on EKS 1.34 and earlier `get pods/exec` is enough to exec into pods. If an existing cluster still has that policy associated with the Nullify role, disassociate it (`aws eks disassociate-access-policy --cluster-name --principal-arn --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy`). `AmazonEKSViewPolicy` is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects. ### Prerequisites @@ -123,7 +121,7 @@ Associates `AmazonEKSAdminViewPolicy` at cluster scope, so no Kubernetes objects - **Authentication mode.** Access entries need `API` or `API_AND_CONFIG_MAP`. Check with `aws eks describe-cluster --name --query cluster.accessConfig.authenticationMode`. Switch (one-way) with `aws eks update-cluster-config --name --access-config authenticationMode=API_AND_CONFIG_MAP`. The module fails at plan for `CONFIG_MAP` clusters. - **Same account.** Nullify assumes the role in the cluster's own account, so deploy `nullify-aws-integration` in every account that owns a scanned cluster. The module rejects clusters in another account. - **One region per module instance (AWS provider v5).** Instantiate `eks-managed-scan-access` once per region with `providers = { aws = aws. }`, as `examples/multi-cluster-complete` does. `../terraform-v6/` handles every region in one instance. -- **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`, plus `eks:AssociateAccessPolicy`, `eks:DisassociateAccessPolicy` and `eks:ListAssociatedAccessPolicies` for `admin_view_policy`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the same ClusterRole itself. +- **Terraform identity.** `eks:DescribeCluster`, `eks:CreateAccessEntry`, `eks:DescribeAccessEntry`, `eks:UpdateAccessEntry`, `eks:DeleteAccessEntry` and `eks:TagResource`. Creating the ClusterRole with `k8s-resources` needs rights to grant every permission in it, which in practice means cluster-admin. A GitOps controller (Flux, Argo CD) that already holds those rights can apply the same ClusterRole itself. - **Pass `principal_unique_id`.** EKS ties an access entry to the role's ID, so a recreated role with the same ARN is silently not authorized. With `principal_unique_id = module.nullify_aws_integration.role_unique_id`, the entries are replaced with the role. - **Existing access entry.** If this role is already mapped on a cluster, import it. The address depends on how you call this module: - Through the root module (`enable_managed_scan = true`, which wraps `eks_managed_scan_access` in `count`): `terraform import 'module.eks_managed_scan_access[0].aws_eks_access_entry.nullify[""]' :`. @@ -192,11 +190,11 @@ module "nullify_readonly_rbac" { } ``` -If any cluster in `cluster_arns` also runs the collector (a `k8s-resources` instance with `enable_collector = true`, an equivalent manifest, or the `nullify-k8s-collector` Helm chart), list its ARN in `collector_cluster_arns` too — the module fails at plan rather than double-registering it, in every `authorization` mode. +If any cluster in `cluster_arns` also runs the collector (a `k8s-resources` instance with `enable_collector = true`, an equivalent manifest, or the `nullify-k8s-collector` Helm chart), list its ARN in `collector_cluster_arns` too — the module fails at plan rather than double-registering it. Outputs: `access_entry_arns`, `authorization`, `kubernetes_group_name`, `nullify_egress_cidrs`, `clusters_for_nullify`, `endpoint_allowlist`. -The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (default `rbac`) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so in `rbac` mode apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true — that is only a proxy (granting the collector's IRSA trust is not the same as deploying it), so override it with the root's own `collector_cluster_arns` variable when trust is granted ahead of deployment, or when migrating a cluster from collector to managed scan: disable or remove the collector on that cluster first, then set the override so the guard stops blocking it before you also remove the cluster from `eks_cluster_arns` (which revokes its IRSA trust). +The root configuration exposes the same module through `enable_managed_scan`, `managed_scan_cluster_arns`, `nullify_region`, `managed_scan_authorization` (`rbac` only) and `managed_scan_kubernetes_group`. The root has no Kubernetes provider, so apply the RBAC separately. It also derives `collector_cluster_arns` for you from `eks_cluster_arns` whenever `enable_kubernetes_integration` is true — that is only a proxy (granting the collector's IRSA trust is not the same as deploying it), so override it with the root's own `collector_cluster_arns` variable when trust is granted ahead of deployment, or when migrating a cluster from collector to managed scan: disable or remove the collector on that cluster first, then set the override so the guard stops blocking it before you also remove the cluster from `eks_cluster_arns` (which revokes its IRSA trust). ## Multi-Cluster Support @@ -287,7 +285,7 @@ terraform init && terraform apply - a key ARN: `arn:aws:kms:::key/`, including multi-Region `key/mrk-...` keys - an alias ARN: `arn:aws:kms:::alias/` -The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and `arn::kms:::key/*` in that same account and region: IAM does not resolve an alias ARN in a policy's `Resource`, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The `kms_policy_resources` module output lists what was granted. +A key ARN is preferred because it avoids the wildcard. IAM does not resolve an alias ARN in a policy's `Resource`, so an alias ARN also grants `arn::kms:::key/*` in that same account and region. A concrete key ARN grants that ARN only and does not open every key in Nullify's account. Nullify's key policy is the real gate on a key Nullify owns. The `kms_policy_resources` module output lists what was granted. `k8s-resources` takes the same variable and writes no IAM policy: it forwards the value to the collector as `NULLIFY_KMS_KEY_ARN`, which S3 takes as `SSEKMSKeyId`. S3 resolves a bare key ID or a bare `alias/` against the calling account, and the key is in Nullify's, so only the two ARN forms above work there too; both modules validate for them. @@ -344,7 +342,7 @@ This is the same value as `collector.clusterName` in the `nullify-k8s-collector` `k8s-resources` variables include `enable_collector` (default `true`), `enable_managed_scan_rbac` (default `false`), `collector_image`, `cronjob_schedule` (default `0 0 * * *`), `kubernetes_namespace`, `service_account_name` and `enable_debug`. -A cluster cannot run the collector and the managed scan at once: the collector's upload registers it as on-prem keyed on `cluster_name`, the managed scan registers the same cluster as its EKS ARN, and nothing joins the two, so running both lists the cluster twice with its pods and containers duplicated. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination: pass it the cluster ARNs where `enable_collector` is `true` and the module fails at plan for any of them, in every `authorization` mode (`admin_view_policy` creates no Kubernetes objects, so a rule inside `k8s-resources` could never see it). To change mode, apply the new one and ask Nullify to remove the old cluster registration. +A cluster cannot run the collector and the managed scan at once: the collector's upload registers it as on-prem keyed on `cluster_name`, the managed scan registers the same cluster as its EKS ARN, and nothing joins the two, so running both lists the cluster twice with its pods and containers duplicated. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination: pass it the cluster ARNs where `enable_collector` is `true` and the module fails at plan for any of them. To change mode, apply the new one and ask Nullify to remove the old cluster registration. `collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. It is the same build as `k8s-collector-latest`, which the `nullify-k8s-collector` Helm chart deploys, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. @@ -391,7 +389,7 @@ module "nullify_k8s" { - `k8s-resources` collector resources moved to `count` instances. `moved` blocks keep existing state, so a plan should show no changes; check it before applying. - The root configuration no longer declares the Kubernetes provider. It never deployed Kubernetes resources. - A cluster outside the AWS provider's region now fails at plan time with an explanation instead of a not-found error. -- The KMS policy grants `key/*` in the account and region of `kms_key_arn` whenever that account is not your own, for alias ARNs and key ARNs alike — the same resources the CloudFormation template grants. An ARN in your own account grants only that key. +- The KMS policy grants `key/*` in the account and region of `kms_key_arn` only when that ARN is an alias. A concrete key ARN is preferred: it grants that ARN only. ## Security Considerations diff --git a/aws-integration-setup/terraform/examples/managed-scan/main.tf b/aws-integration-setup/terraform/examples/managed-scan/main.tf index 97b8144..2055626 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/main.tf +++ b/aws-integration-setup/terraform/examples/managed-scan/main.tf @@ -64,7 +64,7 @@ module "eks_managed_scan_access" { # Argo CD applies the nullify-readonly ClusterRole instead. module "k8s_resources" { source = "../../modules/k8s-resources" - count = var.authorization == "rbac" && var.manage_rbac ? 1 : 0 + count = var.manage_rbac ? 1 : 0 enable_collector = false enable_managed_scan_rbac = true diff --git a/aws-integration-setup/terraform/examples/managed-scan/outputs.tf b/aws-integration-setup/terraform/examples/managed-scan/outputs.tf index 91b6cdc..bc94e92 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/outputs.tf +++ b/aws-integration-setup/terraform/examples/managed-scan/outputs.tf @@ -9,7 +9,7 @@ output "clusters_for_nullify" { } output "kubernetes_group_name" { - description = "Kubernetes group the RBAC binding names (null for admin_view_policy)" + description = "Kubernetes group the RBAC binding names" value = module.eks_managed_scan_access.kubernetes_group_name } diff --git a/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example index 4fe07b7..94d13cc 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example +++ b/aws-integration-setup/terraform/examples/managed-scan/terraform.tfvars.example @@ -16,9 +16,8 @@ eks_cluster_arn = "arn:aws:eks:eu-west-1:123456789012:cluster/prod" # REQUIRED: your Nullify region from the configure page (ap-southeast-2, eu-central-1 or us-east-2) nullify_region = "eu-central-1" -# rbac (recommended): list-only ClusterRole bound to the access entry's group. -# admin_view_policy: AmazonEKSAdminViewPolicy, no Kubernetes objects, but it reads -# every resource including Secrets and pods/log, and allows exec on EKS <= 1.34. +# rbac is the only supported authorization mode: list-only ClusterRole bound +# to the access entry's group. AmazonEKSAdminViewPolicy is not supported. # authorization = "rbac" # Set false if Helm, Flux or Argo CD applies the nullify-readonly ClusterRole instead. diff --git a/aws-integration-setup/terraform/examples/managed-scan/variables.tf b/aws-integration-setup/terraform/examples/managed-scan/variables.tf index f1b6096..4a4bb3c 100644 --- a/aws-integration-setup/terraform/examples/managed-scan/variables.tf +++ b/aws-integration-setup/terraform/examples/managed-scan/variables.tf @@ -35,19 +35,19 @@ variable "nullify_region" { variable "authorization" { type = string - description = "rbac (recommended) or admin_view_policy. admin_view_policy grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" + description = "rbac is the only supported mode. AmazonEKSAdminViewPolicy is not supported" default = "rbac" } variable "kubernetes_group_name" { type = string - description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole in rbac mode" + description = "Kubernetes group on the access entry, bound to the nullify-readonly ClusterRole" default = "nullify-readonly" } variable "manage_rbac" { type = bool - description = "In rbac mode, create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" + description = "Create the nullify-readonly ClusterRole and binding with the Kubernetes provider. Set false when GitOps or Helm applies them" default = true } diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf index 00df32f..5a4699d 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/main.tf @@ -62,7 +62,7 @@ resource "aws_eks_access_entry" "nullify" { cluster_name = data.aws_eks_cluster.this[each.key].name principal_arn = var.principal_arn type = "STANDARD" - kubernetes_groups = var.authorization == "rbac" ? [var.kubernetes_group_name] : [] + kubernetes_groups = [var.kubernetes_group_name] tags = var.tags lifecycle { @@ -75,22 +75,6 @@ resource "aws_eks_access_entry" "nullify" { } } -resource "aws_eks_access_policy_association" "admin_view" { - for_each = { for key, cluster in local.clusters : key => cluster if var.authorization == "admin_view_policy" } - - cluster_name = aws_eks_access_entry.nullify[each.key].cluster_name - principal_arn = aws_eks_access_entry.nullify[each.key].principal_arn - policy_arn = "arn:${each.value.partition}:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" - - access_scope { - type = "cluster" - } - - lifecycle { - replace_triggered_by = [aws_eks_access_entry.nullify[each.key].access_entry_arn] - } -} - locals { endpoints = { for key, cluster in data.aws_eks_cluster.this : key => { @@ -109,10 +93,3 @@ check "nullify_can_reach_cluster_endpoints" { error_message = "Nullify's ${var.nullify_region} egress IPs cannot reach the public endpoint of: ${join(", ", local.unreachable_clusters)}. This module does not change publicAccessCidrs; see the endpoint_allowlist output for the missing CIDRs and an update command. The check matches exact CIDRs, so a wider range that already covers Nullify's IPs also warns." } } - -check "admin_view_policy_is_broader_than_required" { - assert { - condition = var.authorization != "admin_view_policy" - error_message = "authorization = admin_view_policy associates AmazonEKSAdminViewPolicy: get, list and watch on every resource, including Secrets, custom resources and pods/log. On EKS 1.34 and earlier get pods/exec is enough to exec into pods, and these grants do not show in kubectl auth can-i --list. The default rbac mode grants list on only the kinds Nullify reads." - } -} diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf index 6e6d1bc..3431d1d 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/outputs.tf @@ -4,13 +4,13 @@ output "access_entry_arns" { } output "authorization" { - description = "How Kubernetes authorizes Nullify: rbac or admin_view_policy" + description = "How Kubernetes authorizes Nullify (rbac is the only supported mode)" value = var.authorization } output "kubernetes_group_name" { - description = "Group the RBAC binding must name (null for admin_view_policy)" - value = var.authorization == "rbac" ? var.kubernetes_group_name : null + description = "Group the RBAC binding must name" + value = var.kubernetes_group_name } output "nullify_egress_cidrs" { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl index 2cc657a..e95e90f 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/tests/eks_managed_scan_access.tftest.hcl @@ -102,11 +102,6 @@ run "rbac_is_the_default" { error_message = "rbac mode must put the entry in the nullify-readonly group" } - assert { - condition = length(aws_eks_access_policy_association.admin_view) == 0 - error_message = "rbac mode must not associate any access policy" - } - assert { condition = output.endpoint_allowlist["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].update_command == null error_message = "An endpoint open to 0.0.0.0/0 needs no update" @@ -178,34 +173,14 @@ run "private_endpoint_warns_without_a_command" { } } -run "admin_view_policy_is_cluster_scoped_and_warns" { +run "admin_view_policy_is_rejected" { command = plan variables { authorization = "admin_view_policy" } - expect_failures = [check.admin_view_policy_is_broader_than_required] - - assert { - condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].policy_arn == "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" - error_message = "admin_view_policy must associate AmazonEKSAdminViewPolicy" - } - - assert { - condition = aws_eks_access_policy_association.admin_view["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].access_scope[0].type == "cluster" - error_message = "The access policy must be cluster scoped" - } - - assert { - condition = output.kubernetes_group_name == null - error_message = "admin_view_policy binds no Kubernetes group" - } - - assert { - condition = length(aws_eks_access_entry.nullify["arn:aws:eks:eu-west-1:123456789012:cluster/prod"].kubernetes_groups) == 0 - error_message = "admin_view_policy must plan kubernetes_groups as an empty set on the entry: the attribute is Optional+Computed, so null would keep whatever is in state. This pins the planned configuration under a mocked provider, not the applied result" - } + expect_failures = [var.authorization] } run "view_policy_is_not_offered" { @@ -238,18 +213,14 @@ run "cross_account_clusters_are_rejected" { expect_failures = [data.aws_eks_cluster.this] } -run "a_collector_cluster_is_rejected_even_under_admin_view_policy" { - # admin_view_policy creates no Kubernetes objects, so the k8s-resources - # ClusterRole's own precondition can never see this combination. The guard - # has to live here, on the access entry every authorization mode creates. +run "a_collector_cluster_is_rejected" { command = plan variables { - authorization = "admin_view_policy" collector_cluster_arns = ["arn:aws:eks:eu-west-1:123456789012:cluster/prod"] } - expect_failures = [data.aws_eks_cluster.this, check.admin_view_policy_is_broader_than_required] + expect_failures = [data.aws_eks_cluster.this] } run "a_cluster_not_in_collector_cluster_arns_is_unaffected" { diff --git a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf index cdb3c6b..a95b289 100644 --- a/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf +++ b/aws-integration-setup/terraform/modules/eks-managed-scan-access/variables.tf @@ -26,18 +26,18 @@ variable "cluster_arns" { variable "authorization" { type = string - description = "How Kubernetes authorizes the access entry. rbac (recommended): the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. admin_view_policy: associates AmazonEKSAdminViewPolicy at cluster scope and needs no Kubernetes objects. WARNING: AmazonEKSAdminViewPolicy grants get, list and watch on every resource, including Secrets, custom resources and pods/log; on EKS 1.34 and earlier get pods/exec is enough to exec into pods; and its grants do not show in kubectl auth can-i --list" + description = "How Kubernetes authorizes the access entry. rbac is the only supported mode: the entry carries kubernetes_group_name; bind it to a list-only nullify-readonly ClusterRole, which k8s-resources applies with enable_managed_scan_rbac = true. AmazonEKSAdminViewPolicy is not supported: it grants get, list and watch on every resource, including Secrets, custom resources and pods/log, and on EKS 1.34 and earlier allows exec into pods" default = "rbac" validation { - condition = contains(["rbac", "admin_view_policy"], var.authorization) - error_message = "authorization must be rbac or admin_view_policy. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." + condition = var.authorization == "rbac" + error_message = "authorization must be rbac. AmazonEKSAdminViewPolicy is not supported: it grants get, list and watch on every resource, including Secrets and pods/log, and on EKS 1.34 and earlier allows exec into pods. AmazonEKSViewPolicy is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects." } } variable "kubernetes_group_name" { type = string - description = "Kubernetes group on the access entry when authorization is rbac" + description = "Kubernetes group on the access entry" default = "nullify-readonly" validation { @@ -64,7 +64,7 @@ variable "tags" { variable "collector_cluster_arns" { type = list(string) - description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Checked for every authorization mode, including admin_view_policy, which creates no Kubernetes objects and so is invisible to any precondition inside k8s-resources. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match" + description = "The subset of cluster_arns where a k8s-resources instance with enable_collector = true (or an equivalent manifest, or the nullify-k8s-collector Helm chart) already registers the cluster as an on-prem collector target. The module refuses to also create an access entry for one of these: the collector keys the cluster on cluster_name, this module's access entry keys the same cluster on its EKS ARN, and nothing joins the two, so the cluster would be listed twice in inventory with its pods and containers duplicated. Compared as full ARNs, so a same-named cluster in a different account or region is never mistaken for a match" default = [] validation { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index 3ec66b4..eb9dd36 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -124,7 +124,6 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", - "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeLaunchTemplates", "ec2:DescribeManagedPrefixLists", "ec2:DescribeNatGateways", @@ -148,6 +147,9 @@ data "aws_iam_policy_document" "readonly_policy_part1" { "ec2:GetSnapshotBlockPublicAccessState", "ec2:SearchTransitGateway*", "ecr:Describe*", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:GetAuthorizationToken", "ecr:GetLifecyclePolicy", "ecr:GetRepositoryPolicy", "ecs:Describe*", @@ -244,7 +246,6 @@ data "aws_iam_policy_document" "readonly_policy_part2" { "sqs:GetQueueAttributes", "sqs:List*", "ssm:Describe*", - "ssm:GetDocument", "ssm:ListDocuments", "states:DescribeStateMachine", "states:List*", @@ -254,6 +255,12 @@ data "aws_iam_policy_document" "readonly_policy_part2" { ] resources = ["*"] } + + statement { + effect = "Allow" + actions = ["ssm:GetDocument"] + resources = ["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"] + } } data "aws_iam_policy_document" "s3_access_policy" { @@ -312,9 +319,6 @@ data "aws_iam_policy_document" "deny_actions_policy" { "s3:GetObject*", "s3:DeleteObject*", "s3:RestoreObject", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:GetAuthorizationToken", "ecr:PutImage", "ecr:InitiateLayerUpload", "ecr:UploadLayerPart", @@ -349,6 +353,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "ec2:DeleteSnapshot", "ec2:CreateImage", "ec2:DeregisterImage", + "ec2:DescribeLaunchTemplateVersions", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DeleteLogGroup", @@ -362,6 +367,7 @@ data "aws_iam_policy_document" "deny_actions_policy" { "dynamodb:Query", "dynamodb:Scan", "es:ESHttp*", + "events:ListTargetsByRule", "glue:GetConnection", "glue:GetConnections", "glue:GetPartition*", diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf index f88960e..c0e651e 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/locals.tf @@ -20,20 +20,18 @@ locals { enable_s3_access = var.s3_bucket_name != "" || var.nullify_s3_access_point_arn != "" s3_bucket_arn = var.s3_bucket_name != "" ? "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket_name}" : "" - # KMS configuration. The key Nullify hands out lives in Nullify's account, so - # key/* is granted next to the ARN: IAM never resolves an alias ARN in a - # Resource, and the key policy on a key Nullify owns is the real gate on what - # this role can use. The wildcard is derived only when the ARN's account - # differs from this one, so a key pasted from the customer's own account - # grants that key alone and reaches nothing else in the account. For a - # Nullify-supplied key this grants what the CloudFormation template grants. - enable_kms_access = var.kms_key_arn != "" - kms_arn_parts = split(":", var.kms_key_arn) - kms_arn_is_full = length(local.kms_arn_parts) == 6 - kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" - kms_arn_is_cross_account = local.kms_arn_is_full && local.kms_arn_account != data.aws_caller_identity.current.account_id - kms_key_wildcard = local.kms_arn_is_cross_account ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] - kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] + # KMS configuration. IAM never resolves an alias ARN in a Resource, so + # key/* is granted next to an alias ARN in that account and region. A + # concrete key ARN grants that ARN only: it must not open every key in + # Nullify's account. Nullify's key policy is the real gate on a key + # Nullify owns. + enable_kms_access = var.kms_key_arn != "" + kms_arn_parts = split(":", var.kms_key_arn) + kms_arn_is_full = length(local.kms_arn_parts) == 6 + kms_arn_account = local.kms_arn_is_full ? element(local.kms_arn_parts, 4) : "" + kms_arn_is_alias = local.kms_arn_is_full && strcontains(var.kms_key_arn, ":alias/") + kms_key_wildcard = local.kms_arn_is_alias ? ["arn:${element(local.kms_arn_parts, 1)}:kms:${element(local.kms_arn_parts, 3)}:${local.kms_arn_account}:key/*"] : [] + kms_policy_resources = local.enable_kms_access ? concat([var.kms_key_arn], local.kms_key_wildcard) : [] # Common tags common_tags = merge(var.tags, { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf index db5825b..fa4561a 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/outputs.tf @@ -81,7 +81,7 @@ output "policy_arns" { } output "kms_policy_resources" { - description = "Resources the KMS policy grants: kms_key_arn, plus key/* in its account and region when that account is not this one (empty when kms_key_arn is not set)" + description = "Resources the KMS policy grants: a key ARN is granted as-is; an alias ARN also gets key/* in its account and region because IAM cannot resolve aliases. Empty when kms_key_arn is not set. A key ARN is preferred because it avoids the wildcard" value = local.kms_policy_resources } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index d43efd3..cd48138 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -53,7 +53,7 @@ run "cross_account_alias_grants_the_alias_and_key_wildcard" { } } -run "cross_account_key_arn_grants_the_key_wildcard_too" { +run "a_key_arn_grants_only_that_key" { command = plan variables { @@ -61,12 +61,12 @@ run "cross_account_key_arn_grants_the_key_wildcard_too" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab", "arn:aws:kms:us-east-2:111122223333:key/*"]) - error_message = "A key ARN in Nullify's account must grant key/* too: this is what the CloudFormation template grants for the same input" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab"]) + error_message = "A concrete key ARN must grant that ARN only: key/* would open every key in Nullify's account" } } -run "a_key_in_this_account_grants_only_that_key" { +run "an_alias_in_this_account_still_grants_key_wildcard" { command = plan variables { @@ -74,8 +74,8 @@ run "a_key_in_this_account_grants_only_that_key" { } assert { - condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key"]) - error_message = "An ARN in the caller's own account must never derive key/*: that would reach every key in the account whose policy delegates to the root" + condition = tolist(output.kms_policy_resources) == tolist(["arn:aws:kms:us-east-2:123456789012:alias/my-own-key", "arn:aws:kms:us-east-2:123456789012:key/*"]) + error_message = "An alias ARN must grant key/* because IAM does not resolve aliases" } } @@ -168,8 +168,7 @@ run "issuer_urls_must_match_the_cluster_region" { # passing test only proves the module plans, not that a policy grants any # action. `statement` blocks are the data source's own configuration -- they # are plan-time known even when the resulting document is mocked -- so assert -# on those instead. Only the KMS, S3 access point and readonly policies are -# covered here: the trust and deny-list policies aren't in W58/N78's scope. +# on those instead. run "kms_policy_grants_the_expected_actions_and_resources" { command = plan @@ -228,3 +227,37 @@ run "readonly_policy_grants_a_read_only_action" { error_message = "The readonly policy statement must Allow, not Deny" } } + +run "permission_bar_scopes_secret_shaped_reads" { + command = plan + + assert { + condition = !contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ec2:DescribeLaunchTemplateVersions") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ec2:DescribeLaunchTemplates") + error_message = "DescribeLaunchTemplateVersions returns UserData in the clear and must not be allowed; DescribeLaunchTemplates stays" + } + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:GetDownloadUrlForLayer") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:BatchGetImage") && contains(data.aws_iam_policy_document.readonly_policy_part1.statement[0].actions, "ecr:GetAuthorizationToken") + error_message = "Image pull and ECR auth must be allowed so scanning can fetch customer images" + } + + assert { + condition = contains(data.aws_iam_policy_document.readonly_policy_part2.statement[0].actions, "lambda:GetFunction") && !contains(data.aws_iam_policy_document.readonly_policy_part2.statement[0].actions, "ssm:GetDocument") + error_message = "lambda:GetFunction stays on *; ssm:GetDocument must leave the wildcard allow" + } + + assert { + condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 + error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell" + } + + assert { + condition = contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ec2:DescribeLaunchTemplateVersions") && contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "events:ListTargetsByRule") + error_message = "UserData and EventBridge target Input must be denied" + } + + assert { + condition = !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:GetDownloadUrlForLayer") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:BatchGetImage") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ecr:GetAuthorizationToken") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "lambda:GetFunction") && !contains(data.aws_iam_policy_document.deny_actions_policy.statement[0].actions, "ssm:GetDocument") + error_message = "Image pull, lambda:GetFunction and scoped GetDocument must not be denied" + } +} diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf index 88f5b95..20811ea 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/variables.tf @@ -108,7 +108,7 @@ variable "tags" { variable "kms_key_arn" { type = string - description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. The key Nullify hands out is in Nullify's account, so the policy grants the ARN you pass and key/* in that same account and region, for key ARNs and alias ARNs alike: IAM does not resolve an alias in a policy Resource, and Nullify's key policy is the real gate on a key Nullify owns. The wildcard is derived only when the ARN's account differs from yours, so an ARN pasted from your own account grants that key alone. The kms_policy_resources output lists what was granted" + description = "The KMS ARN shown on the Nullify configure page (optional): a key ARN, including multi-Region key/mrk-... keys, or an alias ARN. A key ARN is preferred because it avoids the wildcard. IAM does not resolve an alias in a policy Resource, so an alias ARN also grants key/* in that same account and region. A concrete key ARN grants that ARN only and does not open every key in Nullify's account. Nullify's key policy is the real gate on a key Nullify owns. The kms_policy_resources output lists what was granted" default = "" validation { diff --git a/aws-integration-setup/terraform/outputs.tf b/aws-integration-setup/terraform/outputs.tf index c3f7bf0..18d20c9 100644 --- a/aws-integration-setup/terraform/outputs.tf +++ b/aws-integration-setup/terraform/outputs.tf @@ -58,7 +58,7 @@ output "managed_scan_clusters_for_nullify" { } output "managed_scan_kubernetes_group" { - description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled or for admin_view_policy)" + description = "Kubernetes group to bind to the nullify-readonly ClusterRole (null when disabled)" value = one(module.eks_managed_scan_access[*].kubernetes_group_name) } diff --git a/aws-integration-setup/terraform/terraform.tfvars.example b/aws-integration-setup/terraform/terraform.tfvars.example index 05970ae..c3a17c3 100644 --- a/aws-integration-setup/terraform/terraform.tfvars.example +++ b/aws-integration-setup/terraform/terraform.tfvars.example @@ -38,7 +38,7 @@ enable_kubernetes_integration = false # ] # Managed EKS scan: access entries so Nullify lists Kubernetes resources with no in-cluster agent. -# Clusters must be in aws_region and must already exist when you plan. In rbac mode, bind the group +# Clusters must be in aws_region and must already exist when you plan. Bind the group # to a list-only nullify-readonly ClusterRole: the k8s-resources module applies one with # enable_managed_scan_rbac = true. # enable_managed_scan = true diff --git a/aws-integration-setup/terraform/variables.tf b/aws-integration-setup/terraform/variables.tf index beac5ff..0575495 100644 --- a/aws-integration-setup/terraform/variables.tf +++ b/aws-integration-setup/terraform/variables.tf @@ -122,13 +122,18 @@ variable "collector_cluster_arns" { variable "managed_scan_authorization" { type = string - description = "rbac (recommended): the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. admin_view_policy: AmazonEKSAdminViewPolicy, which reads every resource including Secrets and pods/log, and allows exec into pods on EKS 1.34 and earlier" + description = "rbac is the only supported mode: the access entry carries managed_scan_kubernetes_group, which you bind to a list-only nullify-readonly ClusterRole. The k8s-resources module applies that ClusterRole with enable_managed_scan_rbac = true; the root has no Kubernetes provider, so apply it separately. AmazonEKSAdminViewPolicy is not supported" default = "rbac" + + validation { + condition = var.managed_scan_authorization == "rbac" + error_message = "managed_scan_authorization must be rbac. AmazonEKSAdminViewPolicy is not supported." + } } variable "managed_scan_kubernetes_group" { type = string - description = "Kubernetes group on the access entries in rbac mode" + description = "Kubernetes group on the access entries" default = "nullify-readonly" } diff --git a/helm-charts/nullify-k8s-collector/README.md b/helm-charts/nullify-k8s-collector/README.md index fbd4a2f..9e82087 100644 --- a/helm-charts/nullify-k8s-collector/README.md +++ b/helm-charts/nullify-k8s-collector/README.md @@ -66,7 +66,7 @@ The chart refuses to render when a required value is missing or still a placehol | `collector.s3.keyPrefix` | S3 key prefix | `k8s-collector` | | `collector.aws.region` | Region of the **Nullify S3 bucket**, not your cluster. Defaults to the region in `collector.kms.keyArn`; any other value is rejected, because S3 only accepts a KMS key from the bucket's region. | `""` | | `collector.clusterName` | Cluster name (must match your actual cluster name) | `""` (required) | -| `collector.kms.keyArn` | KMS key ARN `arn:aws:kms:::key/` or alias ARN `arn:aws:kms:::alias/`, from the Nullify configure page. With an alias, the IAM role's KMS statement must cover the key the alias resolves to, for example `arn:aws:kms:::key/*`; see [KMS alias ARNs](#kms-alias-arns). | `""` (required) | +| `collector.kms.keyArn` | KMS key ARN `arn:aws:kms:::key/` (preferred) or alias ARN `arn:aws:kms:::alias/`, from the Nullify configure page. A key ARN avoids a `key/*` wildcard. With an alias, the IAM role's KMS statement must also cover `arn:aws:kms:::key/*`; see [KMS alias ARNs](#kms-alias-arns). | `""` (required) | | `collector.debug.enabled` | Enable debug logging for troubleshooting | `false` | | `collector.gke.awsRoleArn` | **GKE only.** Nullify-owned federated AWS IAM role ARN (provided after cluster registration). | `""` | | `collector.gke.audience` | **GKE only.** Token audience for the projected SA token. Do not change unless Nullify asks you to. | `sts.amazonaws.com` | @@ -102,11 +102,10 @@ The collector only uploads. It needs `s3:PutObject` on the Nullify bucket and `k ### KMS alias ARNs -IAM evaluates KMS permissions against the key an alias resolves to, not the alias. When `collector.kms.keyArn` is an alias ARN, the IAM role's KMS statement must cover that key, for example `kms:GenerateDataKey` on `arn:aws:kms:::key/*` in Nullify's KMS account and region. Otherwise uploads are denied. +IAM evaluates KMS permissions against the key an alias resolves to, not the alias. A key ARN is preferred because the IAM role can grant that ARN alone. When `collector.kms.keyArn` is an alias ARN, the IAM role's KMS statement must also cover that key, for example `kms:GenerateDataKey` on `arn:aws:kms:::key/*` in Nullify's KMS account and region. Otherwise uploads are denied. -- **CloudFormation:** connector releases that include PR #64 grant `key/*` in the account and region of the `NullifyKMSKeyArn` parameter. A stack created from an earlier template grants only the ARN it was given; update it to a later release. -- **Terraform:** the module accepts an alias in `kms_key_arn`. Check that your module version's KMS statement also covers `key/*`; one that grants only `kms_key_arn` denies uploads. -- Either way, a key ARN (`key/`) works with every version. +- **Terraform:** an alias in `kms_key_arn` also grants `key/*` in that account and region. A concrete key ARN grants that ARN only. +- **CloudFormation:** the template grants the `NullifyKMSKeyArn` parameter as given. Prefer a key ARN; an alias ARN is not enough for `kms:GenerateDataKey` because IAM does not resolve aliases in a policy Resource. Then set the role ARN in your values: @@ -183,7 +182,7 @@ so the chart renders without the role-arn annotation: - `namespace.create` and `namespace.requireNamespace` are removed, along with the pre-install namespace hook and the `bitnami/kubectl` Job. Existing namespaces are not deleted. New installs need `--create-namespace`, or an existing namespace. - `collector.dataCollection.*` is removed. The collector never read `EXCLUDE_NAMESPACES`, `INCLUDE_RESOURCES` or `METADATA_ONLY`. - `collector.clusterName`, `collector.s3.bucket` and `collector.kms.keyArn` now default to `""`, and rendering fails when they are empty or still the old placeholders. The EKS role-arn annotation is validated the same way, unless `serviceAccount.requireRoleArn: false`. -- `collector.kms.keyArn` must be an `arn:aws:kms:` key or alias ARN. With an alias, the IAM role's KMS statement must cover the resolved key (for example `arn:aws:kms:::key/*`), or uploads are denied; see [KMS alias ARNs](#kms-alias-arns). +- `collector.kms.keyArn` must be an `arn:aws:kms:` key or alias ARN. A key ARN is preferred. With an alias, the IAM role's KMS statement must cover the resolved key (for example `arn:aws:kms:::key/*`), or uploads are denied; see [KMS alias ARNs](#kms-alias-arns). - `collector.aws.region` now defaults to the KMS key's region instead of `us-east-1`. - `helm upgrade --reuse-values` from 0.2.0 keeps 0.2.0's defaults, including `collector.aws.region: us-east-1`, so the region check fails unless the key is in `us-east-1`. Upgrade with `--reset-then-reuse-values` (Helm 3.14+) or `-f `, or set `collector.aws.region` to the key's region. - `nodeSelector` and `tolerations` now apply to the collector pod; before, only the removed pre-install Job used them. @@ -217,7 +216,7 @@ If the CronJob is not creating jobs on schedule: If the job is failing due to S3 access issues: -1. `AccessDenied` on `kms:GenerateDataKey` with an alias in `collector.kms.keyArn`: the IAM role's KMS statement does not cover the key the alias resolves to. Grant `arn:aws:kms:::key/*` in Nullify's KMS account (see [KMS alias ARNs](#kms-alias-arns)), or use the key ARN. +1. `AccessDenied` on `kms:GenerateDataKey` with an alias in `collector.kms.keyArn`: the IAM role's KMS statement does not cover the key the alias resolves to. Prefer the key ARN, or grant `arn:aws:kms:::key/*` in Nullify's KMS account (see [KMS alias ARNs](#kms-alias-arns)). 2. `AccessDenied` on `sts:AssumeRoleWithWebIdentity`: the service account must be `nullify/nullify-k8s-collector-sa`, and the integration's EKS OIDC provider URL must match the cluster. 3. Wrong-region or redirect errors: `collector.aws.region` must be the Nullify bucket's region (leave it empty). 4. Examine the job logs for detailed error messages. From aa0627a0ead433ef4d81154a9191910b15418161 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Thu, 17 Sep 2026 14:46:46 +1000 Subject: [PATCH 33/36] fix(aws): pin ssm:GetDocument to the Session Manager document partition Co-authored-by: Cursor --- .../cloudformation/nullify-cloudformation-template.json | 4 +++- .../terraform-v6/modules/nullify-aws-integration/data.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 4 ++-- .../terraform/modules/nullify-aws-integration/data.tf | 2 +- .../tests/nullify_aws_integration.tftest.hcl | 4 ++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 819f735..2ba8f52 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -582,7 +582,9 @@ "Action": [ "ssm:GetDocument" ], - "Resource": "arn:*:ssm:*:*:document/SSM-SessionManagerRunShell", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:ssm:*:*:document/SSM-SessionManagerRunShell" + }, "Condition": {} } ] diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 3655afa..173bc55 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -235,7 +235,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { statement { effect = "Allow" actions = ["ssm:GetDocument"] - resources = ["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"] + resources = ["arn:${data.aws_partition.current.partition}:ssm:*:*:document/SSM-SessionManagerRunShell"] } } diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index 962df70..4ce0279 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -227,8 +227,8 @@ run "permission_bar_scopes_secret_shaped_reads" { } assert { - condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 - error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell" + condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:aws:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 + error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell in this partition" } assert { diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index eb9dd36..b98e3f6 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -259,7 +259,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" { statement { effect = "Allow" actions = ["ssm:GetDocument"] - resources = ["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"] + resources = ["arn:${data.aws_partition.current.partition}:ssm:*:*:document/SSM-SessionManagerRunShell"] } } diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl index cd48138..198a6f5 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/tests/nullify_aws_integration.tftest.hcl @@ -247,8 +247,8 @@ run "permission_bar_scopes_secret_shaped_reads" { } assert { - condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:*:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 - error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell" + condition = length([for s in data.aws_iam_policy_document.readonly_policy_part2.statement : s if toset(s.actions) == toset(["ssm:GetDocument"]) && toset(s.resources) == toset(["arn:aws:ssm:*:*:document/SSM-SessionManagerRunShell"])]) == 1 + error_message = "ssm:GetDocument must be allowed only on SSM-SessionManagerRunShell in this partition" } assert { From 05202f3c619849b7a8d405523d8e012e6562e6c0 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Thu, 17 Sep 2026 15:14:41 +1000 Subject: [PATCH 34/36] fix(ci): compare partition-scoped IAM ARNs in the CFN/Terraform drift check Fn: :Sub and the Terraform partition interpolation are the same GetDocument grant; hashing the raw CFN dict crashed the check after that action was scoped. Co-authored-by: Cursor --- scripts/check_iam_policy_drift.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scripts/check_iam_policy_drift.py b/scripts/check_iam_policy_drift.py index 3f6c92b..166ce6e 100755 --- a/scripts/check_iam_policy_drift.py +++ b/scripts/check_iam_policy_drift.py @@ -40,6 +40,30 @@ # `resources` and `condition` are hashable, order-independent normal forms. Entry = tuple +# Partition-scoped ARNs are written with CFN `${AWS::Partition}` and Terraform +# `${data.aws_partition.current.partition}`. Collapse both to `${partition}` +# so a scoped grant like ssm:GetDocument on SSM-SessionManagerRunShell +# compares equal instead of leaving an unhashable Fn::Sub dict in the set. +_PARTITION_TOKEN = "${partition}" +_CFN_PARTITION = re.compile(r"\$\{AWS::Partition\}") +_TF_PARTITION = re.compile(r"\$\{data\.aws_partition\.current\.partition\}") + + +def _norm_cfn_value(value) -> str: + if isinstance(value, str): + return _CFN_PARTITION.sub(_PARTITION_TOKEN, value) + if isinstance(value, dict) and "Fn::Sub" in value: + template = value["Fn::Sub"] + if isinstance(template, list): + template = template[0] + if isinstance(template, str): + return _CFN_PARTITION.sub(_PARTITION_TOKEN, template) + raise SystemExit(f"unsupported CloudFormation IAM value: {value!r}") + + +def _norm_tf_value(value: str) -> str: + return _TF_PARTITION.sub(_PARTITION_TOKEN, value) + def _norm_resources(resources, negated: bool) -> tuple: key = "not_resources" if negated else "resources" @@ -73,6 +97,7 @@ def cfn_entries(resource_names: list) -> set: actions_negated = False if not isinstance(actions, list): actions = [actions] + actions = [_norm_cfn_value(action) for action in actions] if "NotResource" in statement: resources, resources_negated = statement["NotResource"], True @@ -81,6 +106,7 @@ def cfn_entries(resource_names: list) -> set: resources_negated = False if not isinstance(resources, list): resources = [resources] + resources = [_norm_cfn_value(resource) for resource in resources] conditions = [] for test, variable_values in (statement.get("Condition") or {}).items(): @@ -164,12 +190,14 @@ def terraform_entries(data_tf: Path, data_source_names: list) -> set: actions, actions_negated = not_actions, True else: actions, actions_negated = _string_list(block, "actions"), False + actions = [_norm_tf_value(action) for action in actions] not_resources = _string_list(block, "not_resources") if not_resources: resources, resources_negated = not_resources, True else: resources, resources_negated = _string_list(block, "resources"), False + resources = [_norm_tf_value(resource) for resource in resources] entries |= _flatten(effect, actions, actions_negated, resources, resources_negated, _conditions(block)) return entries From ac24227a5cca26aa5c203f544a3a38d8cb651c09 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Thu, 17 Sep 2026 17:12:09 +1000 Subject: [PATCH 35/36] fix(aws): deny SSM document bodies and pin the collector image DescribeDocument and DescribeAutomationExecutions return runbook content and automation InputParameters; deny-wins over ssm:Describe*. Helm now defaults to the same k8s-collector-3.46.0 tag as Terraform. --- .../cloudformation/nullify-cloudformation-template.json | 2 ++ aws-integration-setup/terraform-v6/README.md | 2 +- .../terraform-v6/modules/nullify-aws-integration/data.tf | 2 ++ aws-integration-setup/terraform/README.md | 2 +- .../terraform/modules/nullify-aws-integration/data.tf | 2 ++ helm-charts/nullify-k8s-collector/Chart.yaml | 2 +- helm-charts/nullify-k8s-collector/README.md | 8 ++++---- helm-charts/nullify-k8s-collector/values.yaml | 8 ++++---- 8 files changed, 17 insertions(+), 11 deletions(-) diff --git a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json index 2ba8f52..09f36b2 100644 --- a/aws-integration-setup/cloudformation/nullify-cloudformation-template.json +++ b/aws-integration-setup/cloudformation/nullify-cloudformation-template.json @@ -255,6 +255,8 @@ "iam:ListPoliciesGrantingServiceAccess", "kms:ListGrants", "ssm:DescribeSessions", + "ssm:DescribeDocument", + "ssm:DescribeAutomationExecutions", "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ], diff --git a/aws-integration-setup/terraform-v6/README.md b/aws-integration-setup/terraform-v6/README.md index fb133a1..3776542 100644 --- a/aws-integration-setup/terraform-v6/README.md +++ b/aws-integration-setup/terraform-v6/README.md @@ -119,7 +119,7 @@ module "nullify_eks_access" { A cluster cannot run the collector and the managed scan at once: the two paths register it under different identities, duplicating its inventory. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination -- `enable_collector` and `enable_managed_scan_rbac` may be set together on one `k8s-resources` instance to stage a cutover, since the ClusterRole it applies is inert until an access entry uses it. See "Kubernetes Resources" in `../terraform/README.md`. -`k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same build as the `k8s-collector-latest` tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. +`k8s-resources` defaults `collector_image` to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, the same tag the Helm chart deploys. Replace any explicit `nullify/k8s-collector:latest`: Nullify does not publish that Docker Hub image. ## Upgrade Notes diff --git a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf index 173bc55..efc29a5 100644 --- a/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform-v6/modules/nullify-aws-integration/data.tf @@ -425,6 +425,8 @@ data "aws_iam_policy_document" "deny_actions_policy" { "iam:ListPoliciesGrantingServiceAccess", "kms:ListGrants", "ssm:DescribeSessions", + "ssm:DescribeDocument", + "ssm:DescribeAutomationExecutions", "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ] diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 0ad2eed..6943e5a 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -344,7 +344,7 @@ This is the same value as `collector.clusterName` in the `nullify-k8s-collector` A cluster cannot run the collector and the managed scan at once: the collector's upload registers it as on-prem keyed on `cluster_name`, the managed scan registers the same cluster as its EKS ARN, and nothing joins the two, so running both lists the cluster twice with its pods and containers duplicated. This is enforced by `eks-managed-scan-access`'s `collector_cluster_arns` variable, not by a `k8s-resources` flag combination: pass it the cluster ARNs where `enable_collector` is `true` and the module fails at plan for any of them. To change mode, apply the new one and ask Nullify to remove the old cluster registration. -`collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. It is the same build as `k8s-collector-latest`, which the `nullify-k8s-collector` Helm chart deploys, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. +`collector_image` defaults to `public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0`, a pinned tag of Nullify's ECR Public image. The `nullify-k8s-collector` Helm chart defaults to the same tag, so both install paths run the same collector. Earlier versions defaulted to `nullify/k8s-collector:latest` on Docker Hub, which Nullify does not publish; if you set that value explicitly, replace it. ## Outputs diff --git a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf index b98e3f6..12ff65f 100644 --- a/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf +++ b/aws-integration-setup/terraform/modules/nullify-aws-integration/data.tf @@ -449,6 +449,8 @@ data "aws_iam_policy_document" "deny_actions_policy" { "iam:ListPoliciesGrantingServiceAccess", "kms:ListGrants", "ssm:DescribeSessions", + "ssm:DescribeDocument", + "ssm:DescribeAutomationExecutions", "states:ListExecutions", "workmailmessageflow:GetRawMessageContent" ] diff --git a/helm-charts/nullify-k8s-collector/Chart.yaml b/helm-charts/nullify-k8s-collector/Chart.yaml index a783b9c..0a7ffc5 100644 --- a/helm-charts/nullify-k8s-collector/Chart.yaml +++ b/helm-charts/nullify-k8s-collector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nullify-k8s-collector description: A Helm chart for Nullify's Kubernetes information collector for cloud security scanning type: application -version: 0.4.0 +version: 0.4.1 kubeVersion: ">=1.21.0-0" maintainers: - name: Nullify diff --git a/helm-charts/nullify-k8s-collector/README.md b/helm-charts/nullify-k8s-collector/README.md index 9e82087..e5aae27 100644 --- a/helm-charts/nullify-k8s-collector/README.md +++ b/helm-charts/nullify-k8s-collector/README.md @@ -10,8 +10,8 @@ This Helm chart deploys a Kubernetes collector for the Nullify platform to gathe - **GKE:** the cluster registered with Nullify (see [GKE](#gke-with-workload-identity-federation)) - Egress from the collector pod over HTTPS to `public.ecr.aws`, and to STS and S3 in the Nullify bucket's region -> **Image tag:** the default `collector.image.tag` is the floating `k8s-collector-latest`, pulled on every run. -> Pin a versioned tag once Nullify publishes one. +> **Image tag:** the default `collector.image.tag` is `k8s-collector-3.46.0`, +> the same pin as the Terraform `k8s-resources` module. ## Supported platforms @@ -59,8 +59,8 @@ The chart refuses to render when a required value is missing or still a placehol | `serviceAccount.name` | Name of the service account | `nullify-k8s-collector-sa` | | `serviceAccount.namespace` | Namespace of every namespaced object; `""` means the release namespace. Must already exist. | `nullify` | | `collector.image.repository` | Image repository | `public.ecr.aws/w4o2j2x4/integrations` | -| `collector.image.tag` | Image tag | `k8s-collector-latest` | -| `collector.image.pullPolicy` | Pull policy | `Always` | +| `collector.image.tag` | Image tag | `k8s-collector-3.46.0` | +| `collector.image.pullPolicy` | Pull policy | `IfNotPresent` | | `collector.schedule` | CronJob schedule | `0 0 * * *` (daily at midnight) | | `collector.s3.bucket` | S3 bucket for storing data (from Nullify configure page) | `""` (required) | | `collector.s3.keyPrefix` | S3 key prefix | `k8s-collector` | diff --git a/helm-charts/nullify-k8s-collector/values.yaml b/helm-charts/nullify-k8s-collector/values.yaml index 1aaac4e..556a5b0 100644 --- a/helm-charts/nullify-k8s-collector/values.yaml +++ b/helm-charts/nullify-k8s-collector/values.yaml @@ -47,12 +47,12 @@ collector: # Must match your actual cluster name exactly clusterName: "" - # Container image. Pin a versioned tag instead of k8s-collector-latest once - # Nullify publishes one. + # Same versioned tag as the Terraform k8s-resources default. Do not float + # on k8s-collector-latest: a moved tag is what every Helm install runs. image: repository: public.ecr.aws/w4o2j2x4/integrations - tag: k8s-collector-latest - pullPolicy: Always + tag: k8s-collector-3.46.0 + pullPolicy: IfNotPresent # Resource limits resources: From 480a0854d9701b4c7537b3488a2d2eada7bf78f6 Mon Sep 17 00:00:00 2001 From: Tim Thacker Date: Thu, 17 Sep 2026 17:20:30 +1000 Subject: [PATCH 36/36] fix(k8s): drop secrets list so Secret values cannot be read Kubernetes list on secrets returns .data; omit the grant from collector and managed-scan ClusterRoles. Co-authored-by: Cursor --- .../terraform-v6/modules/k8s-resources/main.tf | 6 +++--- .../modules/k8s-resources/managed_scan.tf | 7 ++++--- .../k8s-resources/tests/k8s_resources.tftest.hcl | 10 +++++----- aws-integration-setup/terraform/README.md | 6 +++--- .../terraform/modules/k8s-resources/main.tf | 6 +++--- .../terraform/modules/k8s-resources/managed_scan.tf | 7 ++++--- .../k8s-resources/tests/k8s_resources.tftest.hcl | 10 +++++----- .../nullify-k8s-collector/templates/clusterrole.yaml | 12 +++--------- 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf index db5d27d..43dec04 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf @@ -49,8 +49,9 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } # Kinds the collector lists, kept identical to the managed-scan ClusterRole - # (managed_scan.tf) and to collect.go. "jobs" is granted ahead of the - # monorepo's fix/k8s-deployment-matching-review stack (#12807 / #12830). + # (managed_scan.tf). Secrets are omitted: list returns values. "jobs" is + # granted ahead of the monorepo's fix/k8s-deployment-matching-review stack + # (#12807 / #12830). rule { api_groups = [""] resources = [ @@ -60,7 +61,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "nodes", "serviceaccounts", "configmaps", - "secrets", "resourcequotas", "limitranges", "persistentvolumes", diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf index 1af9649..a047f5a 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/managed_scan.tf @@ -1,10 +1,11 @@ locals { - # The kinds Nullify's managed EKS scan lists. Any other manifest or chart that - # grants this role must list exactly these kinds; the test suite pins the set. + # The kinds Nullify's managed EKS scan lists. Secrets are omitted: list + # returns values. Any other manifest or chart that grants this role must + # list exactly these kinds; the test suite pins the set. managed_scan_rules = [ { api_groups = [""] - resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"] + resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"] }, { api_groups = ["apps"] diff --git a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl index ea9a6d5..39ebcda 100644 --- a/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform-v6/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -140,15 +140,15 @@ run "managed_scan_rbac_only" { } assert { - condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 27 - error_message = "The managed-scan ClusterRole must list exactly the 27 kinds the scanner reads" + condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 26 + error_message = "The managed-scan ClusterRole must list exactly the 26 config kinds; secrets are omitted because list returns values" } # Pins the set, not just the size: swapping one kind for another keeps every # count assertion green. Anything else that binds this group -- a chart, a # manifest -- has to grant exactly these kinds. assert { - condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"]) + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"]) error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists" } @@ -193,8 +193,8 @@ run "managed_scan_rbac_only" { } assert { - condition = contains(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources, "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") - error_message = "The role must list secrets and grant no subresource" + condition = !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") + error_message = "The role must not list secrets (list returns values) and grant no subresource" } assert { diff --git a/aws-integration-setup/terraform/README.md b/aws-integration-setup/terraform/README.md index 6943e5a..f4c1709 100644 --- a/aws-integration-setup/terraform/README.md +++ b/aws-integration-setup/terraform/README.md @@ -101,7 +101,7 @@ The access entry carries the Kubernetes group `nullify-readonly`, bound to the C | API group | Resources (verb `list`) | |---|---| -| core | nodes, namespaces, pods, services, persistentvolumeclaims, persistentvolumes, configmaps, secrets, resourcequotas, limitranges, serviceaccounts | +| core | nodes, namespaces, pods, services, persistentvolumeclaims, persistentvolumes, configmaps, resourcequotas, limitranges, serviceaccounts | | apps | deployments, daemonsets, statefulsets, replicasets | | batch | jobs | | networking.k8s.io | ingresses, networkpolicies | @@ -111,9 +111,9 @@ The access entry carries the Kubernetes group `nullify-readonly`, bound to the C Plus `get` on the non-resource URL `/version`. -Both modes list Secrets. The in-cluster collector redacts every Secret value inside the cluster before it uploads collected data. The managed scan receives Secret objects from the API server and redacts every value before anything is stored. The `list` verb permits reading values in either mode -- Kubernetes has no metadata-only RBAC verb for Secrets. +Neither mode lists Secrets. Kubernetes has no metadata-only RBAC verb for Secrets: `list secrets` returns `.data` values, so the ClusterRole omits the kind. -`authorization` accepts only `rbac`. `AmazonEKSAdminViewPolicy` is not supported: it grants `get`, `list` and `watch` on every resource, including Secrets, custom resources and `pods/log`, and on EKS 1.34 and earlier `get pods/exec` is enough to exec into pods. If an existing cluster still has that policy associated with the Nullify role, disassociate it (`aws eks disassociate-access-policy --cluster-name --principal-arn --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy`). `AmazonEKSViewPolicy` is not offered: it cannot list nodes, persistent volumes, Secrets, RBAC or admission objects. +`authorization` accepts only `rbac`. `AmazonEKSAdminViewPolicy` is not supported: it grants `get`, `list` and `watch` on every resource, including Secrets, custom resources and `pods/log`, and on EKS 1.34 and earlier `get pods/exec` is enough to exec into pods. If an existing cluster still has that policy associated with the Nullify role, disassociate it (`aws eks disassociate-access-policy --cluster-name --principal-arn --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy`). `AmazonEKSViewPolicy` is not offered: it cannot list nodes, persistent volumes, RBAC or admission objects. ### Prerequisites diff --git a/aws-integration-setup/terraform/modules/k8s-resources/main.tf b/aws-integration-setup/terraform/modules/k8s-resources/main.tf index db5d27d..43dec04 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/main.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/main.tf @@ -49,8 +49,9 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { } # Kinds the collector lists, kept identical to the managed-scan ClusterRole - # (managed_scan.tf) and to collect.go. "jobs" is granted ahead of the - # monorepo's fix/k8s-deployment-matching-review stack (#12807 / #12830). + # (managed_scan.tf). Secrets are omitted: list returns values. "jobs" is + # granted ahead of the monorepo's fix/k8s-deployment-matching-review stack + # (#12807 / #12830). rule { api_groups = [""] resources = [ @@ -60,7 +61,6 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" { "nodes", "serviceaccounts", "configmaps", - "secrets", "resourcequotas", "limitranges", "persistentvolumes", diff --git a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf index 1af9649..a047f5a 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf +++ b/aws-integration-setup/terraform/modules/k8s-resources/managed_scan.tf @@ -1,10 +1,11 @@ locals { - # The kinds Nullify's managed EKS scan lists. Any other manifest or chart that - # grants this role must list exactly these kinds; the test suite pins the set. + # The kinds Nullify's managed EKS scan lists. Secrets are omitted: list + # returns values. Any other manifest or chart that grants this role must + # list exactly these kinds; the test suite pins the set. managed_scan_rules = [ { api_groups = [""] - resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"] + resources = ["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"] }, { api_groups = ["apps"] diff --git a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl index ea9a6d5..39ebcda 100644 --- a/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl +++ b/aws-integration-setup/terraform/modules/k8s-resources/tests/k8s_resources.tftest.hcl @@ -140,15 +140,15 @@ run "managed_scan_rbac_only" { } assert { - condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 27 - error_message = "The managed-scan ClusterRole must list exactly the 27 kinds the scanner reads" + condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 26 + error_message = "The managed-scan ClusterRole must list exactly the 26 config kinds; secrets are omitted because list returns values" } # Pins the set, not just the size: swapping one kind for another keeps every # count assertion green. Anything else that binds this group -- a chart, a # manifest -- has to grant exactly these kinds. assert { - condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "secrets", "resourcequotas", "limitranges", "serviceaccounts"]) + condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"]) error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists" } @@ -193,8 +193,8 @@ run "managed_scan_rbac_only" { } assert { - condition = contains(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources, "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") - error_message = "The role must list secrets and grant no subresource" + condition = !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "secrets") && !contains(flatten([for rule in kubernetes_cluster_role_v1.nullify_readonly[0].rule : rule.resources == null ? [] : rule.resources]), "pods/exec") + error_message = "The role must not list secrets (list returns values) and grant no subresource" } assert { diff --git a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml index 08a4226..b2bb1e9 100644 --- a/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml +++ b/helm-charts/nullify-k8s-collector/templates/clusterrole.yaml @@ -6,16 +6,10 @@ metadata: labels: {{- include "k8s-collector.labels" . | nindent 4 }} rules: - # Kinds the collector actually lists (context/internal/cloudintegrations/ - # k8sintegration/collect/collect.go in the platform monorepo), plus "jobs": - # the fix/k8s-deployment-matching-review stack (monorepo #12807, which - # #12830 sits on) starts collecting Jobs owned by a CronJob to resolve - # scheduled-run pod ownership, so that grant is added ahead of the - # collector code landing rather than dropped and re-added. No other kind - # here is collected today; each one removed a grant this role held with no - # reader. + # Kinds the collector actually lists, plus "jobs" (monorepo #12807 / #12830). + # Secrets are omitted: list returns values. - apiGroups: [""] - resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "secrets", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] + resources: ["pods", "services", "namespaces", "nodes", "serviceaccounts", "configmaps", "resourcequotas", "limitranges", "persistentvolumes", "persistentvolumeclaims"] verbs: ["list"] - apiGroups: ["apps"] resources: ["deployments", "replicasets", "statefulsets", "daemonsets"]