feat: Switch internal service deploy command to deploy image digest under the hood (DBTP-3150)

dbt_platform_helper/COMMANDS.md

@@ -27,6 +27,7 @@
     - [platform-helper secrets](#platform-helper-secrets)
         - [platform-helper secrets create](#platform-helper-secrets-create)
         - [platform-helper secrets copy](#platform-helper-secrets-copy)
+        - [platform-helper secrets list](#platform-helper-secrets-list)
     - [platform-helper notify](#platform-helper-notify)
         - [platform-helper notify environment-progress](#platform-helper-notify-environment-progress)
         - [platform-helper notify post-message](#platform-helper-notify-post-message)
@@ -35,12 +36,12 @@
         - [platform-helper database dump](#platform-helper-database-dump)
         - [platform-helper database load](#platform-helper-database-load)
         - [platform-helper database copy](#platform-helper-database-copy)
+    - [platform-helper version](#platform-helper-version)
+        - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
     - [platform-helper service](#platform-helper-service)
         - [platform-helper service exec](#platform-helper-service-exec)
-        - [platform-helper service ls](#platform-helper-service-ls)
     - [platform-helper job](#platform-helper-job)
         - [platform-helper job run](#platform-helper-job-run)
-        - [platform-helper job ls](#platform-helper-job-ls)
 
 # platform-helper
 
@@ -73,6 +74,7 @@ platform-helper <command> [--version]
 - [`pipeline` ↪](#platform-helper-pipeline)
 - [`secrets` ↪](#platform-helper-secrets)
 - [`service` ↪](#platform-helper-service)
+- [`version` ↪](#platform-helper-version)
 
 # platform-helper application
 
@@ -611,7 +613,7 @@ This will load it's own platform-config.<workspace>.yml file
 ## Usage
 
 ```
-platform-helper secrets (create|copy) 
+platform-helper secrets (create|copy|list) 
 ```
 
 ## Options
@@ -623,6 +625,7 @@ platform-helper secrets (create|copy)
 
 - [`copy` ↪](#platform-helper-secrets-copy)
 - [`create` ↪](#platform-helper-secrets-create)
+- [`list` ↪](#platform-helper-secrets-list)
 
 # platform-helper secrets create
 
@@ -671,6 +674,28 @@ platform-helper secrets copy --app <application> --source <source> --target <tar
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
+# platform-helper secrets list
+
+[↩ Parent](#platform-helper-secrets)
+
+    [DELETED] List secret names and values for an environment.
+
+## Usage
+
+```
+platform-helper secrets list <application> <environment> 
+```
+
+## Arguments
+
+- `app <text>`
+- `env <text>`
+
+## Options
+
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
 # platform-helper notify
 
 [↩ Parent](#platform-helper)
@@ -918,6 +943,47 @@ platform-helper database copy --from <from_env> --to <to_env> --database <databa
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
+# platform-helper version
+
+[↩ Parent](#platform-helper)
+
+    Contains subcommands for getting version information about the current
+    project.
+
+## Usage
+
+```
+platform-helper version get-platform-helper-for-project 
+```
+
+## Options
+
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
+## Commands
+
+- [`get-platform-helper-for-project` ↪](#platform-helper-version-get-platform-helper-for-project)
+
+# platform-helper version get-platform-helper-for-project
+
+[↩ Parent](#platform-helper-version)
+
+    Print the version of platform-tools required by the current project
+
+## Usage
+
+```
+platform-helper version get-platform-helper-for-project [--pipeline <pipeline>] 
+```
+
+## Options
+
+- `--pipeline <text>`
+  - Take into account platform-tools version overrides in the specified pipeline
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
 # platform-helper service
 
 [↩ Parent](#platform-helper)
@@ -927,7 +993,7 @@ platform-helper database copy --from <from_env> --to <to_env> --database <databa
 ## Usage
 
 ```
-platform-helper service (exec|ls) 
+platform-helper service exec 
 ```
 
 ## Options
@@ -938,7 +1004,6 @@ platform-helper service (exec|ls)
 ## Commands
 
 - [`exec` ↪](#platform-helper-service-exec)
-- [`ls` ↪](#platform-helper-service-ls)
 
 # platform-helper service exec
 
@@ -975,29 +1040,6 @@ platform-helper service exec --app <application> --env <environment> --name <nam
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
-# platform-helper service ls
-
-[↩ Parent](#platform-helper-service)
-
-    Lists deployed services for the applicaiton and environment.
-
-## Usage
-
-```
-platform-helper service ls --app <application> --env <environment> 
-```
-
-## Options
-
-- `--app
--a <text>`
-  - Application name
-- `--env
--e <text>`
-  - Environment name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
 # platform-helper job
 
 [↩ Parent](#platform-helper)
@@ -1007,7 +1049,7 @@ platform-helper service ls --app <application> --env <environment>
 ## Usage
 
 ```
-platform-helper job (run|ls) 
+platform-helper job run 
 ```
 
 ## Options
@@ -1017,7 +1059,6 @@ platform-helper job (run|ls)
 
 ## Commands
 
-- [`ls` ↪](#platform-helper-job-ls)
 - [`run` ↪](#platform-helper-job-run)
 
 # platform-helper job run
@@ -1048,26 +1089,3 @@ platform-helper job run --app <application> --env <environment> --name <name> [-
   - Wait for the execution to finish and report it's final status
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
-
-# platform-helper job ls
-
-[↩ Parent](#platform-helper-job)
-
-    Lists deployed scheduled jobs.
-
-## Usage
-
-```
-platform-helper job ls --app <application> --env <environment> 
-```
-
-## Options
-
-- `--app
--a <text>`
-  - Application name
-- `--env
--e <text>`
-  - Environment name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.

dbt_platform_helper/domain/service.py

@@ -29,6 +29,7 @@
 from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.config_validator import ConfigValidator
+from dbt_platform_helper.providers.ecr import ECRProvider
 from dbt_platform_helper.providers.ecs import ECS
 from dbt_platform_helper.providers.ecs import NoClusterException
 from dbt_platform_helper.providers.environment_variable import (
@@ -91,6 +92,7 @@ def __init__(
         load_application=load_application,
         installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
         ecs_provider: ECS = None,
+        ecr_provider: ECRProvider = ECRProvider(),
         s3_provider: S3Provider = None,
         logs_provider: LogsProvider = None,
         autoscaling_provider: AutoscalingProvider = None,
@@ -108,6 +110,7 @@ def __init__(
         self.load_application = load_application
         self.installed_version_provider = installed_version_provider
         self.ecs_provider = ecs_provider
+        self.ecr_provider = ecr_provider
         self.s3_provider = s3_provider
         self.logs_provider = logs_provider
         self.autoscaling_provider = autoscaling_provider
@@ -581,6 +584,14 @@ def deploy(
 
         task_definition = json.loads(s3_response)
 
+        # Resolve image digest from image uri, image_uri is pulled from the task definition.
+        image_digest = ""
+        for container in task_definition["containerDefinitions"]:
+            if container["name"] == service:
+                image_digest = self.ecr_provider.get_image_digest_for_uri(
+                    f"{container['image']}:{image_tag}"
+                )
+
         self.io.info(
             f"Deploying image tag '{image_tag}' to service '{ecs_service_name}' in environment '{environment}'.\n"
         )
@@ -590,6 +601,7 @@ def deploy(
             environment=environment,
             service=service,
             image_tag=image_tag,
+            image_digest=image_digest,
             task_definition=task_definition,
         )
 

dbt_platform_helper/providers/ecr.py

@@ -1,3 +1,4 @@
+import re
 from collections import defaultdict
 
 import botocore
@@ -73,6 +74,28 @@ def get_commit_tag_for_reference(self, application_name: str, codebase: str, ima
                 self.click_io.warn(NO_ASSOCIATED_COMMIT_TAG_WARNING.format(image_ref=image_ref))
                 return image_ref
 
+    def get_image_digest_for_uri(self, image_uri: str) -> str:
+        pattern = (
+            r"^(?P<account>\d+)\.dkr\.ecr\.(?P<region>[^.]+)\.amazonaws\.com/"
+            r"(?P<repository>.+):(?P<tag>[^:]+)$"
+        )
+
+        match = re.match(pattern, image_uri)
+
+        repository = match.group("repository")
+        tag = match.group("tag")
+
+        response = self._get_client().batch_get_image(
+            repositoryName=repository,
+            imageIds=[{"imageTag": tag}],
+        )
+
+        failures = response.get("failures", [])
+        if failures:
+            raise AWSException(f"Error for repo '{repository}' and image tag '{tag}': {failures}")
+
+        return response["images"][0]["imageId"]["imageDigest"]
+
     def _get_ecr_images(self, repository, image_ref, next_page_token):
         params = {"repositoryName": repository, "filter": {"tagStatus": "TAGGED"}}
         if next_page_token:

dbt_platform_helper/providers/ecs.py

@@ -240,6 +240,7 @@ def register_task_definition(
         environment: str,
         service: str,
         task_definition: dict,
+        image_digest: str,
         image_tag: Optional[str] = None,
     ) -> str:
         """Register a new task definition revision using provided model and
@@ -249,7 +250,7 @@ def register_task_definition(
             if container["name"] == service:
 
                 # Append tag to the image URI
-                container["image"] = f"{container['image']}:{image_tag}"
+                container["image"] = f"{container['image']}@{image_digest}"
 
                 # Add DataDog Docker labels https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/?tab=ecs#partial-configuration
                 container["dockerLabels"] = {

tests/platform_helper/domain/test_service.py

@@ -207,6 +207,7 @@ def test_migrate_copilot_manifests_sets_depends_on_for_remaining_sidecars(copilo
 class ServiceManagerMocks:
     def __init__(self, app_name="myapp", env_name="dev", account_id="111122223333"):
         self.ecs_provider = Mock()
+        self.ecr_provider = Mock()
         self.s3_provider = Mock()
         self.logs_provider = Mock()
         self.autoscaling_provider = Mock()
@@ -222,6 +223,7 @@ def __init__(self, app_name="myapp", env_name="dev", account_id="111122223333"):
     def params(self):
         return dict(
             ecs_provider=self.ecs_provider,
+            ecr_provider=self.ecr_provider,
             load_application=self.load_application,
             s3_provider=self.s3_provider,
             logs_provider=self.logs_provider,
@@ -257,7 +259,10 @@ def test_service_deploy_success():
     mocks = ServiceManagerMocks()
     service_manager = ServiceManager(**mocks.params())
 
-    mocks.s3_provider.get_object.return_value = json.dumps({"fakeTaskDefinition": "FAKE"})
+    mocks.ecr_provider.get_image_digest_for_uri.return_value = "sha256:123456"
+    mocks.s3_provider.get_object.return_value = json.dumps(
+        {"containerDefinitions": [{"name": "web", "image": "some-image"}]}
+    )
 
     mocks.ecs_provider.register_task_definition.return_value = (
         "arn:aws:ecs:eu-west-2:111122223333:task-definition/myapp-dev-web-task-def:999"
@@ -298,7 +303,9 @@ def test_service_deploy_success():
     register_task_def_kwargs = mocks.ecs_provider.register_task_definition.call_args.kwargs
     assert register_task_def_kwargs["service"] == "web"
     assert register_task_def_kwargs["image_tag"] == "tag-123"
-    assert register_task_def_kwargs["task_definition"] == {"fakeTaskDefinition": "FAKE"}
+    assert register_task_def_kwargs["task_definition"] == {
+        "containerDefinitions": [{"name": "web", "image": "some-image"}]
+    }
 
     describe_autoscaling_target_kwargs = (
         mocks.autoscaling_provider.describe_autoscaling_target.call_args.kwargs
@@ -338,7 +345,9 @@ def test_service_deploy_exits_early_when_desired_count_zero():
     mocks = ServiceManagerMocks()
     service_manager = ServiceManager(**mocks.params())
 
-    mocks.s3_provider.get_object.return_value = json.dumps({"fakeTaskDefinition": "FAKE"})
+    mocks.s3_provider.get_object.return_value = json.dumps(
+        {"containerDefinitions": [{"name": "web", "image": "some-image"}]}
+    )
     mocks.ecs_provider.register_task_definition.return_value = (
         "arn:aws:ecs:eu-west-2:111122223333:task-definition/myapp-dev-web-task-def:999"
     )
@@ -437,7 +446,9 @@ def test_service_deploy_failed(time_sleep):
     mocks = ServiceManagerMocks()
     service_manager = ServiceManager(**mocks.params())
 
-    mocks.s3_provider.get_object.return_value = json.dumps({"fakeTaskDefinition": "FAKE"})
+    mocks.s3_provider.get_object.return_value = json.dumps(
+        {"containerDefinitions": [{"name": "web", "image": "some-image"}]}
+    )
     mocks.ecs_provider.register_task_definition.return_value = (
         "arn:aws:ecs:eu-west-2:111122223333:task-definition/myapp-dev-web-task-def:999"
     )

tests/platform_helper/providers/test_ecr.py

@@ -288,3 +288,18 @@ def test_get_commit_tag_for_reference_recasts_exceptions_as_more_specific_except
     expected_error = expected_message
 
     assert actual_error == expected_error
+
+
+def test_get_digest_success():
+
+    mocks = ECRProviderMocks()
+    mocks.client_mock.batch_get_image.return_value = {
+        "images": [{"imageId": {"imageDigest": "sha256:123456"}}]
+    }
+
+    ecr_provider = ECRProvider(**mocks.params())
+    digest = ecr_provider.get_image_digest_for_uri(
+        image_uri="563763463626.dkr.ecr.eu-west-2.amazonaws.com/some-app/web:latest"
+    )
+
+    assert digest == "sha256:123456"