Send slack notification if aws-nuke fails

.tool-versions

@@ -1,3 +1,4 @@
 aws-nuke 3.56.2
 # Todo: Bring this up to date once other things are working
 terraform 1.0.5
+uv 0.9.13

aws-nuke.yaml

@@ -31,31 +31,51 @@ resource-types:
 presets:
   nuke:
     filters:
-      IAMRole:
-        - "codebuild-sandbox-nuke"
-        - "cloudwatch-sandbox-nuke"
       CloudWatchEventsRule:
         - "Rule: sandbox-nuke"
       CloudWatchEventsTarget:
         - "Rule: sandbox-nuke Target ID: trigger_build"
       CloudWatchLogsLogGroup:
         - "/aws/codebuild/sandbox-nuke"
-      CodeBuildProject:
-        - "sandbox-nuke"
+        - "/aws/lambda/notify-slack-sandbox-nuke"
       CodeBuildBuild:
         - type: contains
           value: "sandbox-nuke"
+      CodeBuildProject:
+        - "sandbox-nuke"
       CodeStarConnection:
         - "sandbox-nuke"
+      CodeStarNotificationRule:
+        - property: "Name"
+          value: "sandbox-nuke"
       DynamoDBTable:
         - "sandbox-terraform-locks"
       DynamoDBTableItem:
         - "sandbox-terraform-locks -> madetech-sandbox-terraform-state/sandbox.tfstate-md5"
         - "sandbox-terraform-locks -> madetech-sandbox-terraform-state/bootstrap.tfstate-md5"
+      IAMRole:
+        - "cloudwatch-sandbox-nuke"
+        - "codebuild-sandbox-nuke"
+        - "notify-slack-sandbox-nuke"
+        - "sns-logging-sandbox-nuke"
       IAMRolePolicy:
         - "cloudwatch-sandbox-nuke -> cloudwatch-sandbox-nuke"
+        - "notify-slack-sandbox-nuke -> notify-slack-sandbox-nuke"
+        - "notify-slack-sandbox-nuke -> notify-slack-sandbox-nuke-logs"
+        - "sns-logging-sandbox-nuke -> sns-logging-sandbox-nuke"
+      LambdaFunction:
+        - "notify-slack-sandbox-nuke"
       S3Bucket:
         - "s3://madetech-sandbox-terraform-state"
+      SNSSubscription:
+        - type: glob
+          value: "Owner:*:completions-sandbox-nuke:*"
+      SNSTopic:
+        - type: glob
+          value: "TopicARN:*:completions-sandbox-nuke"
+      SSMParameter:
+        - "/sandbox-nuke/slack-webhook"
+
   msp-support-interview-non-ephemeral-resources:
     filters:
       ECRRepository:

terraform/modules/nuke_pipeline/.gitignore

@@ -0,0 +1 @@
+tfbuild

terraform/modules/nuke_pipeline/notifications.tf

@@ -0,0 +1,115 @@
+resource "aws_codestarnotifications_notification_rule" "aws_nuke" {
+  detail_type    = "BASIC"
+  event_type_ids = ["codebuild-project-build-state-failed", "codebuild-project-build-state-succeeded"]
+  name           = local.name
+  resource       = aws_codebuild_project.aws_nuke.arn
+  target {
+    address = aws_sns_topic.completions.arn
+  }
+}
+
+resource "aws_sns_topic" "completions" {
+  name                             = "completions-${local.name}"
+  lambda_failure_feedback_role_arn = aws_iam_role.sns_logging.arn
+}
+
+resource "aws_sns_topic_policy" "completions" {
+  arn    = aws_sns_topic.completions.arn
+  policy = data.aws_iam_policy_document.completions_topic.json
+}
+
+data "aws_iam_policy_document" "completions_topic" {
+  statement {
+    actions = ["sns:Publish"]
+    principals {
+      type        = "Service"
+      identifiers = ["codestar-notifications.amazonaws.com"]
+    }
+    resources = [aws_sns_topic.completions.arn]
+  }
+}
+
+resource "aws_iam_role" "sns_logging" {
+  name               = "sns-logging-${local.name}"
+  assume_role_policy = data.aws_iam_policy_document.sns_logging_assume_role.json
+}
+
+data "aws_iam_policy_document" "sns_logging_assume_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["sns.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role_policy" "sns_logging" {
+  name   = "sns-logging-${local.name}"
+  role   = aws_iam_role.sns_logging.name
+  policy = data.aws_iam_policy_document.sns_logging.json
+}
+
+data "aws_iam_policy_document" "sns_logging" {
+  statement {
+    actions   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
+    resources = ["*"]
+  }
+}
+
+resource "aws_sns_topic_subscription" "completions" {
+  topic_arn = aws_sns_topic.completions.arn
+  protocol  = "lambda"
+  endpoint  = module.notify_slack_lambda.lambda_function_arn
+}
+
+module "notify_slack_lambda" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  function_name = "notify-slack-${local.name}"
+  runtime       = "python3.13"
+  architectures = ["arm64"]
+  source_path = [{
+    path = "${path.module}/notify_slack_lambda"
+    patterns = [
+      "src/*.py",
+      "pyproject.toml",
+      "uv.lock",
+    ]
+    commands = [
+      ":zip src .",
+      "rm -rf ../tfbuild/lambda/vendor",
+      "mkdir -p ../tfbuild/lambda/vendor",
+      "uv sync",
+      "uv export --no-default-groups > ../tfbuild/lambda/requirements.txt",
+      "pip3 install --target=../tfbuild/lambda/vendor -r ../tfbuild/lambda/requirements.txt",
+      ":zip ../tfbuild/lambda/vendor .",
+    ]
+  }]
+  handler = "main.lambda_handler"
+  environment_variables = {
+    SLACK_WEBHOOK_SSM_PARAMETER_NAME = data.aws_ssm_parameter.slack_webhook.name
+  }
+  publish            = true
+  attach_policy_json = true
+  policy_json        = data.aws_iam_policy_document.notify_slack_lambda.json
+  artifacts_dir      = "${path.module}/tfbuild/lambda/artifacts"
+  allowed_triggers = {
+    sns = {
+      service    = "sns"
+      source_arn = aws_sns_topic.completions.arn
+    }
+  }
+}
+
+data "aws_iam_policy_document" "notify_slack_lambda" {
+  statement {
+    actions   = ["ssm:GetParameter"]
+    resources = [data.aws_ssm_parameter.slack_webhook.arn]
+  }
+}
+
+data "aws_ssm_parameter" "slack_webhook" {
+  name            = "/${local.name}/slack-webhook"
+  with_decryption = false
+}

terraform/modules/nuke_pipeline/notify_slack_lambda/.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.so
+.pytest_cache
+.venv

terraform/modules/nuke_pipeline/notify_slack_lambda/.python-version

@@ -0,0 +1 @@
+3.13

terraform/modules/nuke_pipeline/notify_slack_lambda/README.md

@@ -0,0 +1,13 @@
+# Slack Webhook Lambda
+
+This Lambda is triggered whenever the `sandbox-nuke` CodeBuild build run fails. If this happens, the Lambda sends a notification to the `#cop-cloud` Slack channel with a URL to the failed build run.
+
+## Testing with Pytest
+
+```sh
+# Install all dependencies:
+uv sync
+
+# Run the tests:
+uv run pytest
+```

terraform/modules/nuke_pipeline/notify_slack_lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "notify-slack-lambda"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "requests>=2.32.5",
+]
+
+[dependency-groups]
+# Packages that preinstalled when running in AWS Lambda, but need to be installed when working locally:
+lambda-builtin = [
+    "boto3>=1.40.66",
+]
+
+# Packages for running the tests and working on the code:
+dev = [
+    "pytest>=8.4.2",
+    "pytest-mock>=3.15.1",
+    "ruff>=0.14.3",
+]

terraform/modules/nuke_pipeline/notify_slack_lambda/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = .

terraform/modules/nuke_pipeline/notify_slack_lambda/src/functions.py

@@ -0,0 +1,47 @@
+import json
+import requests
+from urllib.parse import quote
+
+
+def get_codebuild_event(sns_event: dict) -> dict:
+    """
+    Function for extracting the CodeBuild success/failure event
+    out of an SNS event.
+    """
+    return json.loads(sns_event["Records"][0]["Sns"]["Message"])
+
+
+def generate_build_url(data: dict) -> str:
+    """
+    Function for generating a URL for the CodeBuild Project run
+    Parameters:
+        data: Dict containing information about the build run
+    Returns:
+        Str containing the URL
+    """
+    aws_account = data["account"]
+    aws_region = data["region"]
+    project_name = data["detail"]["project-name"]
+    codebuild_run = data["detail"]["build-id"].split("build/")[1]
+    codebuild_run_encoded = quote(codebuild_run, safe="")
+    codebuild_run_url = f"https://{aws_region}.console.aws.amazon.com/codesuite/codebuild/{aws_account}/projects/{project_name}/build/{codebuild_run_encoded}/?region={aws_region}"
+    return codebuild_run_url
+
+
+def post_to_slack(url: str, payload: dict) -> dict:
+    """
+    Function for posting payload to Slack channel
+    Parameters:
+        url: Str containing the Webhook URL
+        payload: Dict containing the payload
+    Returns:
+        Dict containing status message
+    """
+    response = requests.post(url, json=payload)
+    response.raise_for_status()  # raises HTTPError for 4xx/5xx
+    return {
+        "success": response.ok,
+        "status_code": response.status_code,
+        "message": "Message successfully delivered to Slack",
+        "response_body": response.text,
+    }

terraform/modules/nuke_pipeline/notify_slack_lambda/src/main.py

@@ -0,0 +1,62 @@
+import os
+import logging
+import boto3
+
+try:
+    # When running under pytest:
+    from . import functions
+except ImportError:
+    # When running in Lambda:
+    import functions
+
+# Initialize the logger
+logger = logging.getLogger()
+logger.setLevel("INFO")
+
+# Set SSM client
+ssm = boto3.client("ssm")
+
+
+def lambda_handler(event, context) -> dict:
+    """
+    Main Lambda handler function
+    Parameters:
+        event: Dict containing the Lambda function event data
+        context: Lambda runtime context
+    Returns:
+        Dict containing status message
+    """
+    try:
+        # Access environment variables
+        slack_webhook_ssm_parameter_name = os.environ.get(
+            "SLACK_WEBHOOK_SSM_PARAMETER_NAME"
+        )
+        if not slack_webhook_ssm_parameter_name:
+            raise ValueError(
+                "Missing required environment variable SLACK_WEBHOOK_SSM_PARAMETER_NAME"
+            )
+
+        # Get SSM parameter value (raw exception will propagate if missing)
+        response = ssm.get_parameter(
+            Name=slack_webhook_ssm_parameter_name, WithDecryption=True
+        )
+        slack_webhook = response["Parameter"]["Value"]
+        if not slack_webhook:
+            raise ValueError(
+                f"SSM parameter '{slack_webhook_ssm_parameter_name}' is empty"
+            )
+
+        codebuild_event = functions.get_codebuild_event(event)
+
+        if codebuild_event["detail"]["build-status"] == "SUCCEEDED":
+            message = "aws-nuke succeeded."
+        else:
+            codebuild_run_url = functions.generate_build_url(codebuild_event)
+            message = f"aws-nuke has failed. See CodeBuild output here: {codebuild_run_url}"
+
+        functions.post_to_slack(slack_webhook, {"text": message})
+
+        return {"statusCode": 200, "message": "Message successfully delivered to Slack"}
+    except Exception as e:
+        logger.error(f"Error sending message: {str(e)}")
+        raise

terraform/modules/nuke_pipeline/notify_slack_lambda/tests/conftest.py

@@ -0,0 +1,5 @@
+import os
+
+# Tests shouldn't require working AWS credentials, but the boto3 library
+# requires a region to be defined even if no requests are going to be made.
+os.environ.setdefault("AWS_DEFAULT_REGION", "dummy")