diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 0000000..3215b75
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,78 @@
+# storoku:ignore
+
+name: Deploy
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches: [main]
+ workflow_run:
+ workflows: [Releaser]
+ types: [completed]
+ branches: [main]
+ workflow_dispatch:
+ inputs:
+ environment:
+ type: choice
+ description: Environment
+ options:
+ - warm-staging
+ - forge-production
+ - forge-test
+
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
+jobs:
+ # apply staging on pushes to main, plan otherwise
+ warm-staging:
+ uses: ./.github/workflows/terraform.yml
+ with:
+ env: warm-staging
+ workspace: warm-staging
+ network: warm
+ did: did:web:staging.up.warm.storacha.network
+ apply: ${{ github.event_name != 'pull_request' }}
+ secrets:
+ aws-account-id: ${{ secrets.WARM_STAGING_AWS_ACCOUNT_ID }}
+ aws-region: ${{ secrets.WARM_STAGING_AWS_REGION }}
+ region: ${{ secrets.WARM_STAGING_AWS_REGION }}
+ private-key: ${{ secrets.WARM_STAGING_PRIVATE_KEY }}
+ cloudflare-zone-id: ${{ secrets.WARM_STAGING_CLOUDFLARE_ZONE_ID }}
+ cloudflare-api-token: ${{ secrets.WARM_STAGING_CLOUDFLARE_API_TOKEN }}
+
+ # apply prod and test on successful release, plan otherwise
+ forge-production:
+ uses: ./.github/workflows/terraform.yml
+ with:
+ env: forge-production
+ workspace: forge-prod
+ network: forge
+ did: did:web:up.forge.storacha.network
+ apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-production') }}
+ secrets:
+ aws-account-id: ${{ secrets.FORGE_PROD_AWS_ACCOUNT_ID }}
+ aws-region: ${{ secrets.FORGE_PROD_AWS_REGION }}
+ region: ${{ secrets.FORGE_PROD_AWS_REGION }}
+ private-key: ${{ secrets.FORGE_PROD_PRIVATE_KEY }}
+ cloudflare-zone-id: ${{ secrets.FORGE_PROD_CLOUDFLARE_ZONE_ID }}
+ cloudflare-api-token: ${{ secrets.FORGE_PROD_CLOUDFLARE_API_TOKEN }}
+
+ forge-test:
+ uses: ./.github/workflows/terraform.yml
+ with:
+ env: forge-test
+ workspace: forge-test
+ network: test
+ did: did:web:up.test.storacha.network
+ apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-test') }}
+ secrets:
+ aws-account-id: ${{ secrets.FORGE_TEST_AWS_ACCOUNT_ID }}
+ aws-region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+ region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+ private-key: ${{ secrets.FORGE_TEST_PRIVATE_KEY }}
+ cloudflare-zone-id: ${{ secrets.FORGE_TEST_CLOUDFLARE_ZONE_ID }}
+ cloudflare-api-token: ${{ secrets.FORGE_TEST_CLOUDFLARE_API_TOKEN }}
diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml
new file mode 100644
index 0000000..3dede96
--- /dev/null
+++ b/.github/workflows/terraform.yml
@@ -0,0 +1,108 @@
+name: Terraform
+
+on:
+ workflow_call:
+ inputs:
+ env:
+ required: true
+ type: string
+ workspace:
+ required: true
+ type: string
+ network:
+ required: false
+ default: "hot"
+ type: string
+ did:
+ required: true
+ type: string
+ apply:
+ required: true
+ type: boolean
+ secrets:
+ aws-account-id:
+ required: true
+ aws-region:
+ required: true
+ private-key:
+ required: true
+ region:
+ required: true
+ cloudflare-zone-id:
+ required: true
+ cloudflare-api-token:
+ required: true
+
+concurrency:
+ group: ${{ github.workflow }}-${{ inputs.workspace }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+env:
+ AWS_ACCOUNT_ID: ${{ secrets.aws-account-id }}
+ AWS_REGION: ${{ secrets.aws-region }}
+ ENV: ${{ inputs.env }}
+ TF_WORKSPACE: ${{ inputs.workspace }}
+ TF_VAR_network: ${{ inputs.network }}
+ TF_VAR_private_key: ${{ secrets.private-key }}
+ TF_VAR_did: ${{ inputs.did }}
+ TF_VAR_app: sprue
+ TF_VAR_domain_base:
+ TF_VAR_allowed_account_id: ${{ secrets.aws-account-id }}
+ TF_VAR_region: ${{ secrets.region }}
+ TF_VAR_cloudflare_zone_id: ${{ secrets.cloudflare-zone-id }}
+ CLOUDFLARE_API_TOKEN: ${{ secrets.cloudflare-api-token }}
+ DEPLOY_ENV: ci
+
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
+jobs:
+ terraform:
+ runs-on: ubuntu-24.04-arm
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-region: ${{ env.AWS_REGION }}
+ role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/terraform-ci
+
+ - uses: opentofu/setup-opentofu@v1
+
+ - name: Tofu Init
+ run: |
+ touch .tfworkspace
+ make init
+ working-directory: deploy
+
+ # just plan if !inputs.apply
+ - name: Terraform Plan
+ if: ${{ !inputs.apply }}
+ run: |
+ make plan
+ working-directory: deploy
+
+ # build and push docker image and apply if inputs.apply
+ - name: Set up Docker Buildx
+ if: ${{ inputs.apply }}
+ uses: docker/setup-buildx-action@v3
+
+ - name: Build + Push Docker ECR
+ if: ${{ inputs.apply }}
+ run: |
+ make docker-push
+ working-directory: deploy
+
+ - name: Terraform Apply
+ if: ${{ inputs.apply }}
+ run: |
+ make apply
+ working-directory: deploy
+
+ - name: Wait For Deployment
+ if: ${{ inputs.apply }}
+ run: |
+ make wait-deploy
+ working-directory: deploy
diff --git a/.storoku.json b/.storoku.json
new file mode 100644
index 0000000..f52dfdb
--- /dev/null
+++ b/.storoku.json
@@ -0,0 +1,22 @@
+{
+ "app": "sprue",
+ "privateKeyEnvVar": "SPRUE_IDENTITY_PRIVATE_KEY",
+ "didEnvVar": "SPRUE_IDENTITY_SERVICE_DID",
+ "port": 0,
+ "js": null,
+ "domainBase": "",
+ "cloudflare": true,
+ "createDB": false,
+ "caches": null,
+ "topics": null,
+ "queues": null,
+ "buckets": null,
+ "secrets": null,
+ "tables": null,
+ "networks": [
+ "forge",
+ "warm",
+ "test"
+ ],
+ "writeToContainer": false
+}
\ No newline at end of file
diff --git a/deploy/.env.production.local.tpl b/deploy/.env.production.local.tpl
new file mode 100644
index 0000000..ffce172
--- /dev/null
+++ b/deploy/.env.production.local.tpl
@@ -0,0 +1,39 @@
+<%
+if [ "$TF_WORKSPACE" == "forge-prod" ]; then
+ DEPLOYMENT_PREFIX="forge-prod-upload-api"
+
+ AGENT_MESSAGE_BUCKET="forge-prod-upload-api-workflow-store-0"
+ DELEGATION_BUCKET="forge-prod-upload-api-delegation-0"
+ UPLOAD_SHARDS_BUCKET="forge-prod-upload-api-upload-shards-0"
+elif [ "$TF_WORKSPACE" == "forge-test" ]; then
+ DEPLOYMENT_PREFIX="forge-test-w3infra"
+
+ AGENT_MESSAGE_BUCKET="workflow-store-forge-test-0"
+ DELEGATION_BUCKET="delegation-forge-test-0"
+ UPLOAD_SHARDS_BUCKET="upload-shards-forge-test-0"
+else
+ DEPLOYMENT_PREFIX="staging-warm-upload-api"
+
+ AGENT_MESSAGE_BUCKET="staging-warm-upload-api-workflow-store-0"
+ DELEGATION_BUCKET="staging-warm-upload-api-delegation-0"
+ UPLOAD_SHARDS_BUCKET="staging-warm-upload-api-upload-shards-0"
+fi
+%>
+
+SPRUE_DYNAMODB_AGENT_INDEX_TABLE=<%= $DEPLOYMENT_PREFIX %>-agent-index
+SPRUE_DYNAMODB_BLOB_REGISTRY_TABLE=<%= $DEPLOYMENT_PREFIX %>-blob-registry
+SPRUE_DYNAMODB_CONSUMER_TABLE=<%= $DEPLOYMENT_PREFIX %>-consumer
+SPRUE_DYNAMODB_CUSTOMER_TABLE=<%= $DEPLOYMENT_PREFIX %>-customer
+SPRUE_DYNAMODB_DELEGATION_TABLE=<%= $DEPLOYMENT_PREFIX %>-delegation
+SPRUE_DYNAMODB_SPACE_METRICS_TABLE=<%= $DEPLOYMENT_PREFIX %>-space-metrics
+SPRUE_DYNAMODB_ADMIN_METRICS_TABLE=<%= $DEPLOYMENT_PREFIX %>-admin-metrics
+SPRUE_DYNAMODB_REPLICA_TABLE=<%= $DEPLOYMENT_PREFIX %>-replica
+SPRUE_DYNAMODB_REVOCATION_TABLE=<%= $DEPLOYMENT_PREFIX %>-revocation
+SPRUE_DYNAMODB_STORAGE_PROVIDER_TABLE=<%= $DEPLOYMENT_PREFIX %>-storage-provider
+SPRUE_DYNAMODB_SUBSCRIPTION_TABLE=<%= $DEPLOYMENT_PREFIX %>-subscription
+SPRUE_DYNAMODB_SPACE_DIFF_TABLE=<%= $DEPLOYMENT_PREFIX %>-space-diff
+SPRUE_DYNAMODB_UPLOAD_TABLE=<%= $DEPLOYMENT_PREFIX %>-upload
+
+SPRUE_S3_AGENT_MESSAGE_BUCKET=<%= AGENT_MESSAGE_BUCKET %>
+SPRUE_S3_DELEGATION_BUCKET=<%= DELEGATION_BUCKET %>
+SPRUE_S3_UPLOAD_SHARDS_BUCKET=<%= UPLOAD_SHARDS_BUCKET %>
diff --git a/deploy/.env.terraform.tpl b/deploy/.env.terraform.tpl
new file mode 100644
index 0000000..dcfb6b7
--- /dev/null
+++ b/deploy/.env.terraform.tpl
@@ -0,0 +1,9 @@
+# copy to .env.terraform and set missing vars
+TF_WORKSPACE= # your name here
+TF_VAR_app=sprue
+TF_VAR_did= # did for your env
+TF_VAR_private_key= # private_key or your env -- do not commit to repo!
+TF_VAR_allowed_account_id=505595374361
+TF_VAR_region=us-east-2
+TF_VAR_cloudflare_zone_id=37783d6f032b78cd97ce37ab6fd42848
+CLOUDFLARE_API_TOKEN= # enter a cloudflare api token
\ No newline at end of file
diff --git a/deploy/.gitignore b/deploy/.gitignore
new file mode 100644
index 0000000..5907d3d
--- /dev/null
+++ b/deploy/.gitignore
@@ -0,0 +1,8 @@
+# Deployment
+.env.production.local
+.env.terraform
+.terraform
+.tfworkspace
+app/code_deploy.sh
+shared/builds
+app/builds
diff --git a/deploy/Makefile b/deploy/Makefile
new file mode 100644
index 0000000..2ef442a
--- /dev/null
+++ b/deploy/Makefile
@@ -0,0 +1,151 @@
+ifneq (,$(wildcard ./.env.terraform))
+ include .env.terraform
+ export
+else
+ ifneq ($(DEPLOY_ENV), ci)
+ $(error You haven't setup your .env file. Please refer to the readme)
+ endif
+endif
+
+ECR_URI=$(TF_VAR_allowed_account_id).dkr.ecr.us-west-2.amazonaws.com
+REPLICATED_ECR_URI=$(TF_VAR_allowed_account_id).dkr.ecr.$(TF_VAR_region).amazonaws.com
+IMAGE_TAG_BASE=$(ECR_URI)/$(TF_VAR_app)-ecr:$(TF_WORKSPACE)
+REPLICATED_IMAGE_TAG_BASE=$(REPLICATED_ECR_URI)/$(TF_VAR_app)-ecr:$(TF_WORKSPACE)
+
+# This is a hack to eval the image tag only when it's a dependency, cause we don't want to set it until docker builds
+get_image_tag = IMAGE_TAG = $$(IMAGE_TAG_BASE)-$$(shell docker inspect --format $$(format-filter) $$(IMAGE_TAG_BASE)-latest | sed -e 's/sha256://g')
+get_replicated_image_tag = REPLICATED_IMAGE_TAG = $$(REPLICATED_IMAGE_TAG_BASE)-$$(shell docker inspect --format $$(format-filter) $$(IMAGE_TAG_BASE)-latest | sed -e 's/sha256://g')
+
+.PHONY: eval_image_tag
+
+eval_image_tag:
+ $(eval $(get_image_tag))
+ $(eval $(get_replicated_image_tag))
+
+# GET the absolute location for .env.production.local, again, only after it exists
+
+
+get_env_file = ENV_FILE = $$(abspath .env.production.local)
+
+.PHONY: eval_env_file
+eval_env_file: .env.production.local
+ $(eval $(get_env_file))
+
+.PHONY: docker-login
+docker-login:
+ aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $(ECR_URI)
+
+.env.terraform:
+
+.env.production.local: .env.terraform .env.production.local.tpl
+ ./esh -s /bin/bash -o .env.production.local .env.production.local.tpl
+
+docker-build: docker-login .env.production.local
+ docker buildx build --platform linux/arm64 -t $(IMAGE_TAG_BASE)-latest --load ..
+
+format-filter ="{{ .Id }}"
+
+docker-tag: docker-build eval_image_tag
+ docker tag $(IMAGE_TAG_BASE)-latest $(IMAGE_TAG)
+
+docker-push: docker-tag eval_image_tag
+ docker push $(IMAGE_TAG)
+
+.PHONY: clean-terraform
+
+clean-terraform: eval_image_tag
+ tofu -chdir=app destroy -var="image_tag=$(REPLICATED_IMAGE_TAG)"
+
+.PHONY: clean-shared
+
+clean-shared:
+ TF_WORKSPACE=default tofu -chdir=shared destroy
+
+.PHONY: clean
+
+clean: clean-terraform clean-shared
+
+app/.terraform:
+ TF_WORKSPACE=default tofu -chdir=app init
+
+shared/.terraform:
+ TF_WORKSPACE=default tofu -chdir=shared init
+
+.tfworkspace:
+ tofu -chdir=app workspace new $(TF_WORKSPACE)
+ touch .tfworkspace
+
+.PHONY: init
+
+init: app/.terraform shared/.terraform .tfworkspace
+
+.PHONY: upgrade-shared
+
+upgrade-shared:
+ TF_WORKSPACE=default tofu -chdir=shared init --upgrade
+
+.PHONY: upgrade-app
+
+upgrade-app: upgrade-shared
+ tofu -chdir=app init -upgrade
+
+.PHONY: upgrade
+upgrade: upgrade-shared upgrade-app
+
+.PHONY: validate-shared
+
+validate-shared: shared/.terraform
+ TF_WORKSPACE=default tofu -chdir=shared validate
+
+.PHONY: validate-app
+
+validate-app: app/.terraform .tfworkspace
+ tofu -chdir=app validate
+
+.PHONY: validate
+
+validate: validate-shared validate-app
+
+.PHONY: plan-shared
+
+plan-shared: shared/.terraform
+ TF_WORKSPACE=default tofu -chdir=shared plan
+
+.PHONY: plan-app
+
+plan-app: app/.terraform .tfworkspace eval_image_tag eval_env_file
+ tofu -chdir=app plan -var="image_tag=$(REPLICATED_IMAGE_TAG)" -var='env_files=["$(ENV_FILE)"]'
+
+.PHONY: plan
+
+plan: plan-shared plan-app
+
+ifeq ($(DEPLOY_ENV), ci)
+APPLY_ARGS=-input=false --auto-approve
+else
+APPLY_ARGS=""
+endif
+
+.PHONY: apply-shared
+apply-shared: shared/.terraform
+ TF_WORKSPACE=default tofu -chdir=shared apply $(APPLY_ARGS)
+
+.PHONY: apply-app
+apply-app: app/.terraform .tfworkspace docker-push eval_image_tag eval_env_file
+ tofu -chdir=app apply -var="image_tag=$(REPLICATED_IMAGE_TAG)" -var='env_files=["$(ENV_FILE)"]' $(APPLY_ARGS)
+
+.PHONY: apply
+apply: apply-shared apply-app
+
+.PHONY: console-shared
+console-shared: shared/.terraform
+ TF_WORKSPACE=default tofu -chdir=shared console
+
+.PHONY: console
+console: app/.terraform .tfworkspace eval_image_tag eval_env_file
+ tofu -chdir=app console -var="image_tag=$(REPLICATED_IMAGE_TAG)" -var='env_files=["$(ENV_FILE)"]'
+
+.PHONY: wait-deploy
+wait-deploy:
+ aws deploy wait deployment-successful --region $(TF_VAR_region) --deployment-id $(shell aws deploy list-deployments --region $(TF_VAR_region) --deployment-group-name $(TF_WORKSPACE)-$(TF_VAR_app)-code-deploy-deployment-group --application-name $(TF_WORKSPACE)-$(TF_VAR_app)-code-deploy-app --query "deployments[0]" --output text)
+
diff --git a/deploy/app/main.tf b/deploy/app/main.tf
new file mode 100644
index 0000000..a19757f
--- /dev/null
+++ b/deploy/app/main.tf
@@ -0,0 +1,70 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 6.0.0"
+ }
+ archive = {
+ source = "hashicorp/archive"
+ }
+ }
+ backend "s3" {
+ bucket = "storacha-terraform-state"
+ key = "storacha/${var.app}/terraform.tfstate"
+ region = "us-west-2"
+ encrypt = true
+ }
+}
+
+provider "aws" {
+ allowed_account_ids = [var.allowed_account_id]
+ region = var.region
+ default_tags {
+ tags = {
+ "Environment" = terraform.workspace
+ "ManagedBy" = "OpenTofu"
+ Owner = "storacha"
+ Team = "Storacha Engineering"
+ Organization = "Storacha"
+ Project = "${var.app}"
+ }
+ }
+}
+
+
+
+module "app" {
+ source = "github.com/storacha/storoku//app?ref=v0.6.2"
+ private_key = var.private_key
+ private_key_env_var = "SPRUE_IDENTITY_PRIVATE_KEY"
+ principal_mapping = var.principal_mapping
+ did = var.did
+ did_env_var = "SPRUE_IDENTITY_SERVICE_DID"
+ app = var.app
+ appState = var.app
+ write_to_container = false
+ environment = terraform.workspace
+ network = var.network
+ # if there are any env vars you want available only to your container
+ # in the vpc as opposed to set in the dockerfile, enter them here
+ # NOTE: do not put sensitive data in env-vars. use secrets
+ deployment_env_vars = []
+ image_tag = var.image_tag
+ create_db = false
+ # enter secret values your app will use here -- these will be available
+ # as env vars in the container at runtime
+ secrets = {
+ }
+ # enter external secrets (provisioned out-of-band) here
+ external_secrets = []
+ # enter any sqs queues you want to create here
+ queues = []
+ caches = []
+ topics = []
+ tables = [
+ ]
+ buckets = [
+ ]
+ env_files = var.env_files
+ domain_base = var.domain_base
+}
diff --git a/deploy/app/upload_service.tf b/deploy/app/upload_service.tf
new file mode 100644
index 0000000..053cbb8
--- /dev/null
+++ b/deploy/app/upload_service.tf
@@ -0,0 +1,173 @@
+locals {
+ deployment_prefix = (
+ terraform.workspace == "forge-prod" ? "forge-prod-upload-api" :
+ terraform.workspace == "forge-test" ? "forge-test-w3infra" :
+ "staging-warm-upload-api"
+ )
+
+ agent_index_table_name = "${local.deployment_prefix}-agent-index"
+ blob_registry_table_name = "${local.deployment_prefix}-blob-registry"
+ consumer_table_name = "${local.deployment_prefix}-consumer"
+ customer_table_name = "${local.deployment_prefix}-customer"
+ delegation_table_name = "${local.deployment_prefix}-delegation"
+ space_metrics_table_name = "${local.deployment_prefix}-space-metrics"
+ admin_metrics_table_name = "${local.deployment_prefix}-admin-metrics"
+ replica_table_name = "${local.deployment_prefix}-replica"
+ revocation_table_name = "${local.deployment_prefix}-revocation"
+ storage_provider_table_name = "${local.deployment_prefix}-storage-provider"
+ subscription_table_name = "${local.deployment_prefix}-subscription"
+ space_diff_table_name = "${local.deployment_prefix}-space-diff"
+ upload_table_name = "${local.deployment_prefix}-upload"
+
+ agent_message_bucket_name = (
+ terraform.workspace == "forge-prod" ? "forge-prod-upload-api-workflow-store-0" :
+ terraform.workspace == "forge-test" ? "workflow-store-forge-test-0" :
+ "staging-warm-upload-api-workflow-store-0"
+ )
+ delegation_bucket_name = (
+ terraform.workspace == "forge-prod" ? "forge-prod-upload-api-delegation-0" :
+ terraform.workspace == "forge-test" ? "delegation-forge-test-0" :
+ "staging-warm-upload-api-delegation-0"
+ )
+ upload_shards_bucket_name = (
+ terraform.workspace == "forge-prod" ? "forge-prod-upload-api-upload-shards-0" :
+ terraform.workspace == "forge-test" ? "upload-shards-forge-test-0" :
+ "staging-warm-upload-api-upload-shards-0"
+ )
+}
+
+# Upload service DynamoDB tables
+data "aws_dynamodb_table" "agent_index_table" {
+ name = local.agent_index_table_name
+}
+
+data "aws_dynamodb_table" "blob_registry_table" {
+ name = local.blob_registry_table_name
+}
+
+data "aws_dynamodb_table" "consumer_table" {
+ name = local.consumer_table_name
+}
+
+data "aws_dynamodb_table" "customer_table" {
+ name = local.customer_table_name
+}
+
+data "aws_dynamodb_table" "delegation_table" {
+ name = local.delegation_table_name
+}
+
+data "aws_dynamodb_table" "space_metrics_table" {
+ name = local.space_metrics_table_name
+}
+
+data "aws_dynamodb_table" "admin_metrics_table" {
+ name = local.admin_metrics_table_name
+}
+
+data "aws_dynamodb_table" "replica_table" {
+ name = local.replica_table_name
+}
+
+data "aws_dynamodb_table" "revocation_table" {
+ name = local.revocation_table_name
+}
+
+data "aws_dynamodb_table" "storage_provider_table" {
+ name = local.storage_provider_table_name
+}
+
+data "aws_dynamodb_table" "subscription_table" {
+ name = local.subscription_table_name
+}
+
+data "aws_dynamodb_table" "space_diff_table" {
+ name = local.space_diff_table_name
+}
+
+data "aws_dynamodb_table" "upload_table" {
+ name = local.upload_table_name
+}
+
+# Upload service S3 buckets
+data "aws_s3_bucket" "agent_message_bucket" {
+ bucket = local.agent_message_bucket_name
+}
+
+data "aws_s3_bucket" "delegation_bucket" {
+ bucket = local.delegation_bucket_name
+}
+
+data "aws_s3_bucket" "upload_shards_bucket" {
+ bucket = local.upload_shards_bucket_name
+}
+
+# Policies
+data "aws_iam_policy_document" "task_upload_service_dynamodb_query_document" {
+ statement {
+ actions = [
+ "dynamodb:Query",
+ ]
+ resources = [
+ data.aws_dynamodb_table.agent_index_table.arn,
+ data.aws_dynamodb_table.blob_registry_table.arn,
+ data.aws_dynamodb_table.consumer_table.arn,
+ data.aws_dynamodb_table.customer_table.arn,
+ data.aws_dynamodb_table.delegation_table.arn,
+ data.aws_dynamodb_table.space_metrics_table.arn,
+ data.aws_dynamodb_table.admin_metrics_table.arn,
+ data.aws_dynamodb_table.replica_table.arn,
+ data.aws_dynamodb_table.revocation_table.arn,
+ data.aws_dynamodb_table.storage_provider_table.arn,
+ data.aws_dynamodb_table.subscription_table.arn,
+ data.aws_dynamodb_table.space_diff_table.arn,
+ data.aws_dynamodb_table.upload_table.arn,
+ ]
+ }
+}
+
+resource "aws_iam_policy" "task_upload_service_dynamodb_query" {
+ name = "${terraform.workspace}-${var.app}-task-upload-service-dynamodb-query"
+ description = "This policy will be used by the ECS task to query data from upload-service DynamoDB tables"
+ policy = data.aws_iam_policy_document.task_upload_service_dynamodb_query_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "task_upload_service_dynamodb_query" {
+ role = module.app.deployment.task_role.name
+ policy_arn = aws_iam_policy.task_upload_service_dynamodb_query.arn
+}
+
+data "aws_iam_policy_document" "task_upload_service_s3_get_document" {
+ statement {
+ actions = [
+ "s3:GetObject",
+ ]
+ resources = [
+ "${data.aws_s3_bucket.agent_message_bucket.arn}/*",
+ "${data.aws_s3_bucket.delegation_bucket.arn}/*",
+ "${data.aws_s3_bucket.upload_shards_bucket.arn}/*",
+ ]
+ }
+ statement {
+ actions = [
+ "s3:ListBucket",
+ "s3:GetBucketLocation",
+ ]
+ resources = [
+ data.aws_s3_bucket.agent_message_bucket.arn,
+ data.aws_s3_bucket.delegation_bucket.arn,
+ data.aws_s3_bucket.upload_shards_bucket.arn,
+ ]
+ }
+}
+
+resource "aws_iam_policy" "task_upload_service_s3_get" {
+ name = "${terraform.workspace}-${var.app}-task-upload-service-s3-get"
+ description = "This policy will be used by the ECS task to get objects from upload-service S3 buckets"
+ policy = data.aws_iam_policy_document.task_upload_service_s3_get_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "task_upload_service_s3_get" {
+ role = module.app.deployment.task_role.name
+ policy_arn = aws_iam_policy.task_upload_service_s3_get.arn
+}
diff --git a/deploy/app/variables.tf b/deploy/app/variables.tf
new file mode 100644
index 0000000..562d308
--- /dev/null
+++ b/deploy/app/variables.tf
@@ -0,0 +1,53 @@
+variable "app" {
+ description = "The name of the application"
+ type = string
+}
+
+variable "allowed_account_id" {
+ description = "account id used for AWS"
+ type = string
+}
+
+variable "region" {
+ description = "aws region for all services"
+ type = string
+}
+
+variable "private_key" {
+ description = "private_key for the peer for this deployment"
+ type = string
+}
+
+variable "did" {
+ description = "DID for this deployment (did:web:... for example)"
+ type = string
+}
+
+variable "image_tag" {
+ description = "ECR image tag to deploy with"
+ type = string
+}
+
+variable "principal_mapping" {
+ type = string
+ description = "JSON encoded mapping of did:web to did:key"
+ default = ""
+}
+
+variable "env_files" {
+ description = "list of environment variable files to upload"
+ type = list(string)
+ default = []
+}
+
+variable "domain_base" {
+ type = string
+ default = ""
+}
+
+variable "network" {
+ description = "The network to use (defaults to the default 'hot' network)"
+ type = string
+ default = "hot"
+}
+
diff --git a/deploy/esh b/deploy/esh
new file mode 100755
index 0000000..8a8f377
--- /dev/null
+++ b/deploy/esh
@@ -0,0 +1,366 @@
+#!/bin/sh
+# vim: set ts=4:
+#---help---
+# USAGE:
+# esh [options] [--] [...]
+# esh <-h | -V>
+#
+# Process and evaluate an ESH template.
+#
+# ARGUMENTS:
+# Path of the template file or "-" to read from STDIN.
+# Variable(s) specified as = to pass into the
+# template (the have higher priority than environment
+# variables).
+#
+# OPTIONS:
+# -d Don't evaluate template, just dump a shell script.
+# -o Output file or "-" for STDOUT. Defaults to "-".
+# -s Command name or path of the shell to use for template
+# evaluation. It must not contain spaces.
+# Defaults to "/bin/sh".
+# -h Show this help message and exit.
+# -V Print version and exit.
+#
+# ENVIRONMENT:
+# ESH_AWK Command name of path of the awk program to use.
+# It must not contain spaces. Defaults to "awk".
+# ESH_MAX_DEPTH Maximum include depth. Defaults to 3.
+# ESH_SHELL Same as -s.
+#
+# EXIT STATUS:
+# 0 Clean exit, no error has encountered.
+# 1 Generic error.
+# 10 Invalid usage.
+# 11 ESH syntax error.
+# 12 Include error: file not found.
+# 13 Include error: exceeded max include depth (ESH_MAX_DEPTH).
+#
+# Please report bugs at .
+#---help---
+set -eu
+
+readonly PROGNAME='esh'
+readonly VERSION='0.3.2'
+readonly SCRIPTPATH="$0"
+
+AWK_CONVERTER=$(cat <<'AWK'
+function fail(code, msg) {
+ state = "ERROR"
+ # FIXME: /dev/stderr is not portable
+ printf("%s: %s\n", line_info(), msg) > "/dev/stderr"
+ exit code
+}
+function line_info() {
+ return FILENAME ? (filenames[depth] ":" linenos[depth]) : "(init)" # (init) if inside BEGIN
+}
+# IMPORTANT: This is the only function that should print a newline.
+function puts(str) {
+ print(line_info()) > MAP_FILE
+ print(str)
+}
+function fputs(str) {
+ printf("%s", str)
+}
+function trim(str) {
+ gsub(/^[ \t\r\n]+|[ \t\r\n]+$/, "", str)
+ return str
+}
+function read(len, _str) {
+ if (len == "") {
+ _str = buff
+ buff = ""
+ } else if (len > 0) {
+ _str = substr(buff, 1, len)
+ buff = substr(buff, len + 1, length(buff))
+ }
+ return _str
+}
+function skip(len) {
+ buff = substr(buff, len + 1, length(buff))
+}
+function flush(len, _str) {
+ _str = read(len)
+
+ if (state == "TEXT") {
+ gsub("'", "'\\''", _str)
+ }
+ if (state != "COMMENT") {
+ fputs(_str)
+ }
+}
+function file_exists(filename, _junk) {
+ if ((getline _junk < filename) >= 0) {
+ close(filename)
+ return 1
+ }
+ return 0
+}
+function dirname(path) {
+ return sub(/\/[^\/]+\/*$/, "/", path) ? path : ""
+}
+function include(filename) {
+ if (index(filename, "/") != 1) { # if does not start with "/"
+ filename = dirname(filenames[depth]) filename
+ }
+ if (!file_exists(filename)) {
+ fail(12, "cannot include " filename ": not a file or not readable")
+ }
+ if (depth > MAX_DEPTH) {
+ fail(13, "cannot include " filename ": exceeded maximum depth of " MAX_DEPTH)
+ }
+ buffs[depth] = buff
+ states[depth] = state
+ filenames[depth + 1] = filename
+ depth++
+
+ init()
+ while ((getline buff < filename) > 0) {
+ if (print_nl && state != "COMMENT") {
+ puts("")
+ }
+ process_line()
+ }
+ end_text()
+ close(filename)
+
+ depth--
+ buff = buffs[depth]
+ state = states[depth]
+}
+function init() {
+ buff = ""
+ linenos[depth] = 0
+ print_nl = 0
+ start_text()
+}
+function start_text() {
+ puts("")
+ fputs("printf '%s' '")
+ state = "TEXT"
+}
+function end_text() {
+ if (state != "TEXT") { return }
+ puts("' #< " line_info())
+ state = "UNDEF"
+}
+function process_line() {
+ print_nl = 1
+ linenos[depth]++
+
+ while (buff != "") {
+ print_nl = 1
+
+ if (state == "TEXT" && match(buff, /<%/)) {
+ flush(RSTART - 1) # print buff before "<%"
+ skip(2) # skip "<%"
+
+ flag = substr(buff, 1, 1)
+ if (flag != "%") {
+ end_text()
+ }
+ if (flag == "%") { # <%%
+ skip(1)
+ fputs("<%")
+ } else if (flag == "=") { # <%=
+ skip(1)
+ fputs("__print ")
+ state = "TAG"
+ } else if (flag == "+") { # <%+
+ if (!match(buff, /[^%]%>/)) {
+ fail(11, "syntax error: <%+ must be closed on the same line")
+ }
+ filename = trim(substr(buff, 2, match(buff, /.-?%>/) - 1))
+ skip(RSTART)
+ include(filename)
+ state = "TAG"
+ } else if (flag == "#") { # <%#
+ state = "COMMENT"
+ } else {
+ state = "TAG"
+ }
+ } else if (state != "TEXT" && match(buff, /%>/)) {
+ flag = RSTART > 1 ? substr(buff, RSTART - 1, 1) : ""
+
+ if (flag == "%") { # %%>
+ flush(RSTART - 2)
+ skip(1)
+ flush(2)
+ } else if (flag == "-") { # -%>
+ flush(RSTART - 2)
+ skip(3)
+ print_nl = 0
+ } else { # %>
+ flush(RSTART - 1)
+ skip(2)
+ }
+ if (flag != "%") {
+ start_text()
+ }
+ } else {
+ flush()
+ }
+ }
+}
+BEGIN {
+ FS = ""
+ depth = 0
+
+ puts("#!" (SHELL ~ /\// ? SHELL : "/usr/bin/env " SHELL))
+ puts("set -eu")
+ puts("if ( set -o pipefail 2>/dev/null ); then set -o pipefail; fi")
+ puts("__print() { printf '%s' \"$*\"; }")
+
+ split(VARS, _lines, /\n/)
+ for (_i in _lines) {
+ puts(_lines[_i])
+ }
+ init()
+}
+{
+ if (NR == 1) {
+ filenames[0] = FILENAME # this var is not defined in BEGIN so we must do it here
+ }
+ buff = $0
+ process_line()
+
+ if (print_nl && state != "COMMENT") {
+ puts("")
+ }
+}
+END {
+ end_text()
+}
+AWK
+)
+AWK_ERR_FILTER=$(cat <<'AWK'
+function line_info(lno, _line, _i) {
+ while ((getline _line < MAPFILE) > 0 && _i++ < lno) { }
+ close(MAPFILE)
+ return _line
+}
+{
+ if (match($0, "^" SRCFILE ":( line)? ?[0-9]+:") && match(substr($0, 1, RLENGTH), /[0-9]+:$/)) {
+ lno = substr($0, RSTART, RLENGTH - 1) + 0
+ msg = substr($0, RSTART + RLENGTH + 1) # v-- some shells duplicate filename
+ msg = index(msg, SRCFILE ":") == 1 ? substr(msg, length(SRCFILE) + 3) : msg
+ print(line_info(lno) ": " msg)
+ } else if ($0 != "") {
+ print($0)
+ }
+}
+AWK
+)
+readonly AWK_CONVERTER AWK_ERR_FILTER
+
+print_help() {
+ sed -En '/^#---help---/,/^#---help---/p' "$SCRIPTPATH" | sed -E 's/^# ?//; 1d;$d;'
+}
+
+filter_shell_stderr() {
+ $ESH_AWK \
+ -v SRCFILE="$1" \
+ -v MAPFILE="$2" \
+ -- "$AWK_ERR_FILTER"
+}
+
+evaluate() {
+ local srcfile="$1"
+ local mapfile="$2"
+
+ # This FD redirection magic is for swapping stdout/stderr back and forth.
+ exec 3>&1
+ { set +e; $ESH_SHELL "$srcfile" 2>&1 1>&3; echo $? >>"$mapfile"; } \
+ | filter_shell_stderr "$srcfile" "$mapfile" >&2
+ exec 3>&-
+
+ return $(tail -n 1 "$mapfile")
+}
+
+convert() {
+ local input="$1"
+ local vars="$2"
+ local map_file="${3:-"/dev/null"}"
+
+ $ESH_AWK \
+ -v MAX_DEPTH="$ESH_MAX_DEPTH" \
+ -v SHELL="$ESH_SHELL" \
+ -v MAP_FILE="$map_file" \
+ -v VARS="$vars" \
+ -- "$AWK_CONVERTER" "$input"
+}
+
+process() {
+ local input="$1"
+ local vars="$2"
+ local evaluate="${3:-yes}"
+ local ret=0 tmpfile mapfile
+
+ if [ "$evaluate" = yes ]; then
+ tmpfile=$(mktemp)
+ mapfile=$(mktemp)
+
+ convert "$input" "$vars" "$mapfile" > "$tmpfile" || ret=$?
+ test $ret -ne 0 || evaluate "$tmpfile" "$mapfile" || ret=$?
+
+ rm -f "$tmpfile" "$mapfile"
+ else
+ convert "$input" "$vars" || ret=$?
+ fi
+ return $ret
+}
+
+: ${ESH_AWK:="awk"}
+: ${ESH_MAX_DEPTH:=3}
+: ${ESH_SHELL:="/bin/sh"}
+EVALUATE='yes'
+OUTPUT=''
+
+while getopts ':dho:s:V' OPT; do
+ case "$OPT" in
+ d) EVALUATE=no;;
+ h) print_help; exit 0;;
+ o) OUTPUT="$OPTARG";;
+ s) ESH_SHELL="$OPTARG";;
+ V) echo "$PROGNAME $VERSION"; exit 0;;
+ '?') echo "$PROGNAME: unknown option: -$OPTARG" >&2; exit 10;;
+ esac
+done
+shift $(( OPTIND - 1 ))
+
+if [ $# -eq 0 ]; then
+ printf "$PROGNAME: %s\n\n" 'missing argument ' >&2
+ print_help >&2
+ exit 10
+fi
+
+INPUT="$1"; shift
+if [ "$INPUT" != '-' ] && ! [ -f "$INPUT" -a -r "$INPUT" ]; then
+ echo "$PROGNAME: can't read $INPUT: not a file or not readable" >&2; exit 10
+fi
+
+# Validate arguments.
+for arg in "$@"; do
+ case "$arg" in
+ *=*) ;;
+ *) echo "$PROGNAME: illegal argument: $arg" >&2; exit 10;;
+ esac
+done
+
+# Format variables into shell variable assignments.
+vars=''; for item in "$@"; do
+ vars="$vars\n${item%%=*}='$(
+ printf %s "${item#*=}" | $ESH_AWK "{ gsub(/'/, \"'\\\\\\\''\"); print }"
+ )'"
+done
+
+export ESH="$0"
+
+if [ "${OUTPUT#-}" ]; then
+ tmpfile="$(mktemp)"
+ trap 'rm -f -- "$tmpfile"' EXIT HUP INT TERM
+ process "$INPUT" "$vars" "$EVALUATE" > "$tmpfile"
+ cat "$tmpfile" > "$OUTPUT"
+else
+ process "$INPUT" "$vars" "$EVALUATE"
+fi
diff --git a/deploy/shared/main.tf b/deploy/shared/main.tf
new file mode 100644
index 0000000..edad29e
--- /dev/null
+++ b/deploy/shared/main.tf
@@ -0,0 +1,65 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 5.86.0"
+ }
+ cloudflare = {
+ source = "cloudflare/cloudflare"
+ version = "~> 5"
+ }
+ }
+ backend "s3" {
+ bucket = "storacha-terraform-state"
+ key = "storacha/${var.app}/shared.tfstate"
+ region = "us-west-2"
+ encrypt = true
+ }
+}
+
+provider "aws" {
+ allowed_account_ids = [var.allowed_account_id]
+ region = "us-west-2"
+ default_tags {
+ tags = {
+ Environment = "shared"
+ ManagedBy = "OpenTofu"
+ Owner = "storacha"
+ Team = "Storacha Engineering"
+ Organization = "Storacha"
+ Project = "${var.app}"
+ }
+ }
+}
+
+provider "aws" {
+ alias = "dev"
+ allowed_account_ids = [var.allowed_account_id]
+ region = "us-east-2"
+ default_tags {
+ tags = {
+ Environment = "dev"
+ ManagedBy = "OpenTofu"
+ Owner = "storacha"
+ Team = "Storacha Engineering"
+ Organization = "Storacha"
+ Project = "${var.app}"
+ }
+ }
+}
+
+module "shared" {
+ source = "github.com/storacha/storoku//shared?ref=v0.6.2"
+ providers = {
+ aws = aws
+ aws.dev = aws.dev
+ }
+ create_db = false
+ caches = []
+ networks = ["forge","warm","test",]
+ app = var.app
+ create_shared_dev_resources = var.create_shared_dev_resources
+ zone_id = var.cloudflare_zone_id
+ domain_base = var.domain_base
+ setup_cloudflare = true
+}
diff --git a/deploy/shared/outputs.tf b/deploy/shared/outputs.tf
new file mode 100644
index 0000000..8d0688d
--- /dev/null
+++ b/deploy/shared/outputs.tf
@@ -0,0 +1,19 @@
+output "route53_zones" {
+ value = module.shared.route53_zones
+}
+
+output "dev_vpc" {
+ value = module.shared.dev_vpc
+}
+
+output "dev_caches" {
+ value = module.shared.dev_caches
+}
+
+output "dev_databases" {
+ value = module.shared.dev_databases
+}
+
+output "dev_kms" {
+ value = module.shared.dev_kms
+}
\ No newline at end of file
diff --git a/deploy/shared/variables.tf b/deploy/shared/variables.tf
new file mode 100644
index 0000000..747dc8b
--- /dev/null
+++ b/deploy/shared/variables.tf
@@ -0,0 +1,25 @@
+variable "app" {
+ description = "The name of the application"
+ type = string
+}
+
+variable "allowed_account_id" {
+ description = "account id used for AWS"
+ type = string
+}
+
+variable "domain_base" {
+ type = string
+ default = ""
+}
+
+variable "create_shared_dev_resources" {
+ description = "create shared resources (vpc, caches, db, kms) for dev environments"
+ type = bool
+ default = false
+}
+
+
+variable "cloudflare_zone_id" {
+ type = string
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 71b45ed..14708ad 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -103,7 +103,7 @@ func SetDefaults(v *viper.Viper) {
v.SetDefault("log.level", "info")
}
-// BindEnvVars sets up environment variable binding with UPLOAD_ prefix.
+// BindEnvVars sets up environment variable binding with SPRUE_ prefix.
func BindEnvVars(v *viper.Viper) {
v.SetEnvPrefix("SPRUE")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
diff --git a/pkg/dynamo/store.go b/pkg/dynamo/store.go
index a74d9e8..13bb5e7 100644
--- a/pkg/dynamo/store.go
+++ b/pkg/dynamo/store.go
@@ -52,8 +52,10 @@ var _ state.StateStore = (*Store)(nil)
// New creates a new DynamoDB-backed state store.
func New(ctx context.Context, cfg Config, logger *zap.Logger) (*Store, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
+ opts := []func(*awsconfig.LoadOptions) error{}
+
+ if cfg.Region != "" {
+ opts = append(opts, awsconfig.WithRegion(cfg.Region))
}
if cfg.Endpoint != "" {