From 24cdfb2054a682f1830bca63af0c0ae229f45f7d Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:33:45 -0700 Subject: [PATCH 01/54] add bucket access resource and comments --- scripts/import-tf-init-env.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/import-tf-init-env.sh b/scripts/import-tf-init-env.sh index 2b2bdfb2..8f5730c7 100644 --- a/scripts/import-tf-init-env.sh +++ b/scripts/import-tf-init-env.sh @@ -26,9 +26,19 @@ cd infrastructure/terraform/aws/environments/init-$ENV terraform init +# workflow data storage terraform import module.project_storage_$ENV.aws_dynamodb_table.github_workflows github-workflows-$ID_ENV + +# tf state storage +terraform import module.project_storage_$ENV.aws_s3_bucket.tfstate mxfactorial-tfstate-$ID_ENV + +# artifact storage terraform import module.project_storage_$ENV.aws_s3_bucket.artifacts mxfactorial-artifacts-$ID_ENV + +# docker image storage +terraform import module.project_storage_$ENV.aws_ecr_repository.go_migrate go-migrate-$ID_ENV + +# client hosting terraform import module.project_storage_$ENV.aws_s3_bucket.client_origin mxfactorial-client-$ID_ENV -terraform import module.project_storage_$ENV.aws_s3_bucket.tfstate mxfactorial-tfstate-$ID_ENV -terraform import module.project_storage_$ENV.aws_s3_bucket_public_access_block.client_origin mxfactorial-client-$ID_ENV -terraform import module.project_storage_$ENV.aws_ecr_repository.go_migrate go-migrate-$ID_ENV \ No newline at end of file +terraform import module.project_storage_$ENV.aws_s3_bucket_website_configuration.client_origin mxfactorial-client-$ID_ENV +terraform import module.project_storage_$ENV.aws_s3_bucket_public_access_block.client_origin mxfactorial-client-$ID_ENV \ No newline at end of file From 7a3b422ac6e801dc14bb901515becd143af094c3 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:34:22 -0700 Subject: [PATCH 02/54] set terraform variable default --- infrastructure/terraform/aws/environments/init-dev/main.tf | 2 +- infrastructure/terraform/aws/environments/init-prod/main.tf | 1 - .../terraform/aws/modules/project-storage/v001/variables.tf | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/aws/environments/init-dev/main.tf b/infrastructure/terraform/aws/environments/init-dev/main.tf index 24c05ffc..c8417129 100644 --- a/infrastructure/terraform/aws/environments/init-dev/main.tf +++ b/infrastructure/terraform/aws/environments/init-dev/main.tf @@ -19,7 +19,7 @@ provider "aws" { // creates 3 buckets and 1 dynamodb table module "project_storage_dev" { source = "../../modules/project-storage/v001" - force_destroy_tfstate = true + force_destroy_storage = true env = local.ENV env_id = local.ENV_ID artifacts_bucket_name_prefix = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default diff --git a/infrastructure/terraform/aws/environments/init-prod/main.tf b/infrastructure/terraform/aws/environments/init-prod/main.tf index afb298b2..cb949001 100644 --- a/infrastructure/terraform/aws/environments/init-prod/main.tf +++ b/infrastructure/terraform/aws/environments/init-prod/main.tf @@ -18,7 +18,6 @@ provider "aws" { module "project_storage_prod" { source = "../../modules/project-storage/v001" - force_destroy_tfstate = false env = local.ENV env_id = local.ENV_ID artifacts_bucket_name_prefix = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf b/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf index 99ba3767..fae6caf4 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf @@ -5,4 +5,7 @@ variable "client_origin_bucket_name_prefix" {} variable "tfstate_bucket_name_prefix" {} variable "ddb_table_name_prefix" {} variable "ddb_table_hash_key" {} -variable "force_destroy_tfstate" { type = bool } +variable "force_destroy_storage" { + type = bool + default = false +} From 9d00a09912dfb20150fd1422404bebd2dd708418 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:35:17 -0700 Subject: [PATCH 03/54] test for missing project.yaml values in tf locals --- .../modules/project-storage/v001/dynamodb.tf | 7 ------- .../aws/modules/project-storage/v001/locals.tf | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/dynamodb.tf b/infrastructure/terraform/aws/modules/project-storage/v001/dynamodb.tf index e27dd897..129d7044 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/dynamodb.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/dynamodb.tf @@ -1,10 +1,3 @@ -locals { - PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) - STORAGE_ENV_VAR = local.PROJECT_CONF.infrastructure.terraform.aws.modules.project-storage.env_var.set - DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default - DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default -} - // avoids dependency on github workflow job outputs when sharing values resource "aws_dynamodb_table" "github_workflows" { name = "${local.DDB_TABLE_NAME_PREFIX}-${local.ID_ENV}" diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf index f4ada81f..4ed8c896 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf @@ -1,3 +1,19 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF = "project.yaml" + CONF_FILE = yamldecode(file("../../../../../${local.PROJECT_CONF}")) + STORAGE_ENV_VAR = local.CONF_FILE.infrastructure.terraform.aws.modules.project-storage.env_var.set + DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default + DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + GO_MIGRATE = "go-migrate" +} + +// fails if services are not found in project.yaml +resource "terraform_data" "locals_test" { + lifecycle { + precondition { + condition = lookup(local.CONF_FILE.migrations, local.GO_MIGRATE, null) != null + error_message = "${local.GO_MIGRATE} not found in ${local.PROJECT_CONF}" + } + } } From 7b681a4997aabbc1e390ca28daac35f13f7455d1 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:36:49 -0700 Subject: [PATCH 04/54] remove bucket config deprecation warning --- .../aws/modules/project-storage/v001/s3.tf | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/s3.tf b/infrastructure/terraform/aws/modules/project-storage/v001/s3.tf index c5b2f92c..68cc0338 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/s3.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/s3.tf @@ -1,28 +1,32 @@ resource "aws_s3_bucket" "artifacts" { bucket = "${var.artifacts_bucket_name_prefix}-${local.ID_ENV}" - force_destroy = true + force_destroy = var.force_destroy_storage } resource "aws_s3_bucket" "tfstate" { bucket = "${var.tfstate_bucket_name_prefix}-${local.ID_ENV}" - force_destroy = var.force_destroy_tfstate + force_destroy = var.force_destroy_storage } resource "aws_s3_bucket" "client_origin" { - bucket = "${var.client_origin_bucket_name_prefix}-${local.ID_ENV}" + bucket = "${var.client_origin_bucket_name_prefix}-${local.ID_ENV}" + force_destroy = var.force_destroy_storage +} - website { - index_document = "index.html" - error_document = "error.html" +resource "aws_s3_bucket_website_configuration" "client_origin" { + bucket = aws_s3_bucket.client_origin.id + index_document { + suffix = "index.html" + } + error_document { + key = "error.html" } - - force_destroy = true } resource "aws_s3_bucket_public_access_block" "client_origin" { - bucket = aws_s3_bucket.client_origin.id + bucket = aws_s3_bucket.client_origin.id block_public_acls = false block_public_policy = false ignore_public_acls = false restrict_public_buckets = false -} \ No newline at end of file +} From a7d3a9489d879a790183679887160ad00a93ee76 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:37:25 -0700 Subject: [PATCH 05/54] set ecr repo name with local --- .../terraform/aws/modules/project-storage/v001/ecr.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf index 01cc4617..0567f4a6 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf @@ -1,9 +1,8 @@ resource "aws_ecr_repository" "go_migrate" { - name = "go-migrate-${local.ID_ENV}" + name = "${local.GO_MIGRATE}-${local.ID_ENV}" image_tag_mutability = "MUTABLE" - force_delete = true - + force_delete = var.force_destroy_storage image_scanning_configuration { scan_on_push = true } -} +} \ No newline at end of file From d9b47ed523dc03e1e76d6334c345693692b04ce1 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:57:08 -0700 Subject: [PATCH 06/54] remove comment --- infrastructure/terraform/aws/environments/init-dev/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/terraform/aws/environments/init-dev/main.tf b/infrastructure/terraform/aws/environments/init-dev/main.tf index c8417129..d355df7e 100644 --- a/infrastructure/terraform/aws/environments/init-dev/main.tf +++ b/infrastructure/terraform/aws/environments/init-dev/main.tf @@ -16,7 +16,6 @@ provider "aws" { } } -// creates 3 buckets and 1 dynamodb table module "project_storage_dev" { source = "../../modules/project-storage/v001" force_destroy_storage = true From bd9ca72d119af9c54e57a0107a909a0526902aa8 Mon Sep 17 00:00:00 2001 From: max funk Date: Sun, 31 Mar 2024 22:57:48 -0700 Subject: [PATCH 07/54] add ecr repos for remaining services --- .../aws/modules/project-storage/v001/ecr.tf | 93 ++++++++++++++++++- .../modules/project-storage/v001/locals.tf | 77 +++++++++++++-- 2 files changed, 161 insertions(+), 9 deletions(-) diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf index 0567f4a6..02557c6e 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf @@ -5,4 +5,95 @@ resource "aws_ecr_repository" "go_migrate" { image_scanning_configuration { scan_on_push = true } -} \ No newline at end of file +} + +resource "aws_ecr_repository" "auto_confirm" { + name = "${local.AUTO_CONFIRM}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + + +resource "aws_ecr_repository" "balance_by_account" { + name = "${local.BALANCE_BY_ACCOUNT}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "graphql" { + name = "${local.GRAPHQL}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "request_approve" { + name = "${local.REQUEST_APPROVE}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "request_by_id" { + name = "${local.REQUEST_BY_ID}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "requests_by_account" { + name = "${local.REQUESTS_BY_ACCOUNT}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "request_create" { + name = "${local.REQUEST_CREATE}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "rule" { + name = "${local.RULE}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "transaction_by_id" { + name = "${local.TRANSACTION_BY_ID}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_repository" "transactions_by_account" { + name = "${local.TRANSACTIONS_BY_ACCOUNT}-${local.ID_ENV}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf index 4ed8c896..a6190a73 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf @@ -1,19 +1,80 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" - PROJECT_CONF = "project.yaml" - CONF_FILE = yamldecode(file("../../../../../${local.PROJECT_CONF}")) - STORAGE_ENV_VAR = local.CONF_FILE.infrastructure.terraform.aws.modules.project-storage.env_var.set - DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default - DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default - GO_MIGRATE = "go-migrate" + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF = "project.yaml" + CONF_FILE = yamldecode(file("../../../../../${local.PROJECT_CONF}")) + STORAGE_ENV_VAR = local.CONF_FILE.infrastructure.terraform.aws.modules.project-storage.env_var.set + DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default + DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + GO_MIGRATE = "go-migrate" + AUTO_CONFIRM = "auto-confirm" + BALANCE_BY_ACCOUNT = "balance-by-account" + GRAPHQL = "graphql" + REQUEST_APPROVE = "request-approve" + REQUEST_BY_ID = "request-by-id" + REQUESTS_BY_ACCOUNT = "requests-by-account" + REQUEST_CREATE = "request-create" + RULE = "rule" + TRANSACTION_BY_ID = "transaction-by-id" + TRANSACTIONS_BY_ACCOUNT = "transactions-by-account" + } -// fails if services are not found in project.yaml +// fails if services not found in project.yaml resource "terraform_data" "locals_test" { lifecycle { precondition { condition = lookup(local.CONF_FILE.migrations, local.GO_MIGRATE, null) != null error_message = "${local.GO_MIGRATE} not found in ${local.PROJECT_CONF}" } + + precondition { + condition = lookup(local.CONF_FILE.services, local.AUTO_CONFIRM, null) != null + error_message = "${local.AUTO_CONFIRM} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.BALANCE_BY_ACCOUNT, null) != null + error_message = "${local.BALANCE_BY_ACCOUNT} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.GRAPHQL, null) != null + error_message = "${local.GRAPHQL} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.REQUEST_APPROVE, null) != null + error_message = "${local.REQUEST_APPROVE} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.REQUEST_BY_ID, null) != null + error_message = "${local.REQUEST_BY_ID} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.REQUESTS_BY_ACCOUNT, null) != null + error_message = "${local.REQUESTS_BY_ACCOUNT} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.REQUEST_CREATE, null) != null + error_message = "${local.REQUEST_CREATE} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.RULE, null) != null + error_message = "${local.RULE} not found in ${local.PROJECT_CONF}" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.TRANSACTION_BY_ID, null) != null + error_message = "${local.TRANSACTION_BY_ID} not found" + } + + precondition { + condition = lookup(local.CONF_FILE.services, local.TRANSACTIONS_BY_ACCOUNT, null) != null + error_message = "${local.TRANSACTIONS_BY_ACCOUNT} not found in ${local.PROJECT_CONF}" + } } } From f1440893c2bae2a12bf8d3b06305735b2650f55c Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 17:33:01 -0700 Subject: [PATCH 08/54] remove docker subdirectories --- docker/{dev => }/balance-by-account.Dockerfile | 0 docker/{dev => }/bitnami-postgres.Dockerfile | 4 ++-- docker/{prod => }/client-base.Dockerfile | 0 docker/{dev => }/client.Dockerfile | 0 docker/{dev => }/graphql.Dockerfile | 0 docker/{dev => }/request-approve.Dockerfile | 0 docker/{dev => }/request-by-id.Dockerfile | 0 docker/{dev => }/request-create.Dockerfile | 0 docker/{dev => }/requests-by-account.Dockerfile | 0 docker/{dev => }/rule.Dockerfile | 0 docker/{dev => }/transaction-by-id.Dockerfile | 0 docker/{dev => }/transactions-by-account.Dockerfile | 0 12 files changed, 2 insertions(+), 2 deletions(-) rename docker/{dev => }/balance-by-account.Dockerfile (100%) rename docker/{dev => }/bitnami-postgres.Dockerfile (74%) rename docker/{prod => }/client-base.Dockerfile (100%) rename docker/{dev => }/client.Dockerfile (100%) rename docker/{dev => }/graphql.Dockerfile (100%) rename docker/{dev => }/request-approve.Dockerfile (100%) rename docker/{dev => }/request-by-id.Dockerfile (100%) rename docker/{dev => }/request-create.Dockerfile (100%) rename docker/{dev => }/requests-by-account.Dockerfile (100%) rename docker/{dev => }/rule.Dockerfile (100%) rename docker/{dev => }/transaction-by-id.Dockerfile (100%) rename docker/{dev => }/transactions-by-account.Dockerfile (100%) diff --git a/docker/dev/balance-by-account.Dockerfile b/docker/balance-by-account.Dockerfile similarity index 100% rename from docker/dev/balance-by-account.Dockerfile rename to docker/balance-by-account.Dockerfile diff --git a/docker/dev/bitnami-postgres.Dockerfile b/docker/bitnami-postgres.Dockerfile similarity index 74% rename from docker/dev/bitnami-postgres.Dockerfile rename to docker/bitnami-postgres.Dockerfile index 9b56c615..1dca8334 100644 --- a/docker/dev/bitnami-postgres.Dockerfile +++ b/docker/bitnami-postgres.Dockerfile @@ -6,7 +6,7 @@ ENV ALLOW_EMPTY_PASSWORD=yes ENV POSTGRESQL_USERNAME=root ENV POSTGRESQL_DATABASE=mxfactorial -COPY docker/bitnami-postgres/docker-entrypoint-initdb.d/migrate.sh /docker-entrypoint-initdb.d/migrate.sh +COPY docker/bitnami-postgres/docker-entrypoint-initdb.d/up-migrate.sh /docker-entrypoint-initdb.d/up-migrate.sh COPY migrations /tmp/migrations RUN apt update && \ @@ -16,4 +16,4 @@ RUN apt update && \ rm migrate.linux-amd64.deb && \ apt clean && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives && \ - chmod +x /docker-entrypoint-initdb.d/migrate.sh \ No newline at end of file + chmod +x /docker-entrypoint-initdb.d/up-migrate.sh \ No newline at end of file diff --git a/docker/prod/client-base.Dockerfile b/docker/client-base.Dockerfile similarity index 100% rename from docker/prod/client-base.Dockerfile rename to docker/client-base.Dockerfile diff --git a/docker/dev/client.Dockerfile b/docker/client.Dockerfile similarity index 100% rename from docker/dev/client.Dockerfile rename to docker/client.Dockerfile diff --git a/docker/dev/graphql.Dockerfile b/docker/graphql.Dockerfile similarity index 100% rename from docker/dev/graphql.Dockerfile rename to docker/graphql.Dockerfile diff --git a/docker/dev/request-approve.Dockerfile b/docker/request-approve.Dockerfile similarity index 100% rename from docker/dev/request-approve.Dockerfile rename to docker/request-approve.Dockerfile diff --git a/docker/dev/request-by-id.Dockerfile b/docker/request-by-id.Dockerfile similarity index 100% rename from docker/dev/request-by-id.Dockerfile rename to docker/request-by-id.Dockerfile diff --git a/docker/dev/request-create.Dockerfile b/docker/request-create.Dockerfile similarity index 100% rename from docker/dev/request-create.Dockerfile rename to docker/request-create.Dockerfile diff --git a/docker/dev/requests-by-account.Dockerfile b/docker/requests-by-account.Dockerfile similarity index 100% rename from docker/dev/requests-by-account.Dockerfile rename to docker/requests-by-account.Dockerfile diff --git a/docker/dev/rule.Dockerfile b/docker/rule.Dockerfile similarity index 100% rename from docker/dev/rule.Dockerfile rename to docker/rule.Dockerfile diff --git a/docker/dev/transaction-by-id.Dockerfile b/docker/transaction-by-id.Dockerfile similarity index 100% rename from docker/dev/transaction-by-id.Dockerfile rename to docker/transaction-by-id.Dockerfile diff --git a/docker/dev/transactions-by-account.Dockerfile b/docker/transactions-by-account.Dockerfile similarity index 100% rename from docker/dev/transactions-by-account.Dockerfile rename to docker/transactions-by-account.Dockerfile From 58b4b6c5d919934725fd7abd443f157ffce249e5 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 17:33:39 -0700 Subject: [PATCH 09/54] consolidate docker compose services --- docker/compose.balance-by-account.yaml | 21 --- docker/compose.bitnami-postgres.yaml | 14 -- docker/compose.client.yaml | 10 - docker/compose.graphql.yaml | 20 -- docker/compose.request-approve.yaml | 21 --- docker/compose.request-by-id.yaml | 21 --- docker/compose.request-create.yaml | 22 --- docker/compose.requests-by-account.yaml | 22 --- docker/compose.rule.yaml | 22 --- docker/compose.transaction-by-id.yaml | 21 --- docker/compose.transactions-by-account.yaml | 22 --- docker/compose.yaml | 196 ++++++++++++++++++++ 12 files changed, 196 insertions(+), 216 deletions(-) delete mode 100644 docker/compose.balance-by-account.yaml delete mode 100644 docker/compose.bitnami-postgres.yaml delete mode 100644 docker/compose.client.yaml delete mode 100644 docker/compose.graphql.yaml delete mode 100644 docker/compose.request-approve.yaml delete mode 100644 docker/compose.request-by-id.yaml delete mode 100644 docker/compose.request-create.yaml delete mode 100644 docker/compose.requests-by-account.yaml delete mode 100644 docker/compose.rule.yaml delete mode 100644 docker/compose.transaction-by-id.yaml delete mode 100644 docker/compose.transactions-by-account.yaml create mode 100644 docker/compose.yaml diff --git a/docker/compose.balance-by-account.yaml b/docker/compose.balance-by-account.yaml deleted file mode 100644 index 765b385d..00000000 --- a/docker/compose.balance-by-account.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: mxf -services: - balance-by-account: - build: - context: ../ - dockerfile: ./docker/dev/balance-by-account.Dockerfile - ports: - - "10004:10004" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - BALANCE_BY_ACCOUNT_PORT: 10004 - depends_on: - - postgres diff --git a/docker/compose.bitnami-postgres.yaml b/docker/compose.bitnami-postgres.yaml deleted file mode 100644 index 91cca7d2..00000000 --- a/docker/compose.bitnami-postgres.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: mxf -services: - postgres: - build: - context: ../ - dockerfile: ./docker/dev/bitnami-postgres.Dockerfile - ports: - - "5432:5432" - healthcheck: - test: [ "CMD-SHELL", "pg_isready -U postgres" ] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s diff --git a/docker/compose.client.yaml b/docker/compose.client.yaml deleted file mode 100644 index c3354cf6..00000000 --- a/docker/compose.client.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: mxf -services: - client: - build: - context: ../ - dockerfile: ./docker/dev/client.Dockerfile - args: - - GRAPHQL_URI=${GRAPHQL_URI} - ports: - - "10009:80" diff --git a/docker/compose.graphql.yaml b/docker/compose.graphql.yaml deleted file mode 100644 index 365d3429..00000000 --- a/docker/compose.graphql.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: mxf -services: - graphql: - build: - context: ../ - dockerfile: ./docker/dev/graphql.Dockerfile - ports: - - "10000:10000" - environment: - RULE_URL: "http://rule:10001" - REQUEST_CREATE_URL: "http://request-create:10002" - REQUEST_APPROVE_URL: "http://request-approve:10003" - REQUEST_BY_ID_URL: "http://request-by-id:10005" - REQUESTS_BY_ACCOUNT_URL: "http://requests-by-account:10006" - TRANSACTIONS_BY_ACCOUNT_URL: "http://transactions-by-account:10008" - TRANSACTION_BY_ID_URL: "http://transaction-by-id:10007" - BALANCE_BY_ACCOUNT_URL: "http://balance-by-account:10004" - READINESS_CHECK_PATH: "/healthz" - RUST_LOG: info - GRAPHQL_PORT: 10000 diff --git a/docker/compose.request-approve.yaml b/docker/compose.request-approve.yaml deleted file mode 100644 index 25da8e51..00000000 --- a/docker/compose.request-approve.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: mxf -services: - request-approve: - build: - context: ../ - dockerfile: ./docker/dev/request-approve.Dockerfile - ports: - - "10003:10003" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - REQUEST_APPROVE_PORT: 10003 - depends_on: - - postgres diff --git a/docker/compose.request-by-id.yaml b/docker/compose.request-by-id.yaml deleted file mode 100644 index a891904f..00000000 --- a/docker/compose.request-by-id.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: mxf -services: - request-by-id: - build: - context: ../ - dockerfile: ./docker/dev/request-by-id.Dockerfile - ports: - - "10005:10005" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - REQUEST_BY_ID_PORT: 10005 - depends_on: - - postgres diff --git a/docker/compose.request-create.yaml b/docker/compose.request-create.yaml deleted file mode 100644 index e4b8da6f..00000000 --- a/docker/compose.request-create.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: mxf -services: - request-create: - build: - context: ../ - dockerfile: ./docker/dev/request-create.Dockerfile - ports: - - "10002:10002" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - RULE_URL: http://rule:10001 - REQUEST_CREATE_PORT: 10002 - depends_on: - - postgres diff --git a/docker/compose.requests-by-account.yaml b/docker/compose.requests-by-account.yaml deleted file mode 100644 index fbb2e014..00000000 --- a/docker/compose.requests-by-account.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: mxf -services: - requests-by-account: - build: - context: ../ - dockerfile: ./docker/dev/requests-by-account.Dockerfile - ports: - - "10006:10006" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - RETURN_RECORD_LIMIT: 20 - REQUESTS_BY_ACCOUNT_PORT: 10006 - depends_on: - - postgres diff --git a/docker/compose.rule.yaml b/docker/compose.rule.yaml deleted file mode 100644 index 55f52b91..00000000 --- a/docker/compose.rule.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: mxf -services: - rule: - build: - context: ../ - dockerfile: ./docker/dev/rule.Dockerfile - ports: - - "10001:10001" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - RUST_LOG: info - RULE_PORT: 10001 - depends_on: - - postgres diff --git a/docker/compose.transaction-by-id.yaml b/docker/compose.transaction-by-id.yaml deleted file mode 100644 index 86bdcc68..00000000 --- a/docker/compose.transaction-by-id.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: mxf -services: - transaction-by-id: - build: - context: ../ - dockerfile: ./docker/dev/transaction-by-id.Dockerfile - ports: - - "10007:10007" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - TRANSACTION_BY_ID_PORT: 10007 - depends_on: - - postgres diff --git a/docker/compose.transactions-by-account.yaml b/docker/compose.transactions-by-account.yaml deleted file mode 100644 index efaefc2f..00000000 --- a/docker/compose.transactions-by-account.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: mxf -services: - transactions-by-account: - build: - context: ../ - dockerfile: ./docker/dev/transactions-by-account.Dockerfile - ports: - - "10008:10008" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - PG_MAX_CONNECTIONS: 20 - PG_IDLE_TIMEOUT: 10000 - PG_CONN_TIMEOUT: 500 - READINESS_CHECK_PATH: /healthz - RETURN_RECORD_LIMIT: 20 - TRANSACTIONS_BY_ACCOUNT_PORT: 10008 - depends_on: - - postgres diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 00000000..24ba4155 --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,196 @@ +name: mxf +services: + postgres: + build: + context: ../ + dockerfile: ./docker/bitnami-postgres.Dockerfile + ports: + - "5432:5432" + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + graphql: + build: + context: ../ + dockerfile: ./docker/graphql.Dockerfile + ports: + - "10000:10000" + environment: + RULE_URL: "http://rule:10001" + REQUEST_CREATE_URL: "http://request-create:10002" + REQUEST_APPROVE_URL: "http://request-approve:10003" + REQUEST_BY_ID_URL: "http://request-by-id:10005" + REQUESTS_BY_ACCOUNT_URL: "http://requests-by-account:10006" + TRANSACTIONS_BY_ACCOUNT_URL: "http://transactions-by-account:10008" + TRANSACTION_BY_ID_URL: "http://transaction-by-id:10007" + BALANCE_BY_ACCOUNT_URL: "http://balance-by-account:10004" + READINESS_CHECK_PATH: "/healthz" + RUST_LOG: info + GRAPHQL_PORT: 10000 + balance-by-account: + build: + context: ../ + dockerfile: ./docker/balance-by-account.Dockerfile + ports: + - "10004:10004" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + BALANCE_BY_ACCOUNT_PORT: 10004 + depends_on: + - postgres + request-create: + build: + context: ../ + dockerfile: ./docker/request-create.Dockerfile + ports: + - "10002:10002" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + RULE_URL: http://rule:10001 + REQUEST_CREATE_PORT: 10002 + depends_on: + - postgres + request-approve: + build: + context: ../ + dockerfile: ./docker/request-approve.Dockerfile + ports: + - "10003:10003" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + REQUEST_APPROVE_PORT: 10003 + depends_on: + - postgres + rule: + build: + context: ../ + dockerfile: ./docker/rule.Dockerfile + ports: + - "10001:10001" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + RUST_LOG: info + RULE_PORT: 10001 + depends_on: + - postgres + request-by-id: + build: + context: ../ + dockerfile: ./docker/request-by-id.Dockerfile + ports: + - "10005:10005" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + REQUEST_BY_ID_PORT: 10005 + depends_on: + - postgres + requests-by-account: + build: + context: ../ + dockerfile: ./docker/requests-by-account.Dockerfile + ports: + - "10006:10006" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + RETURN_RECORD_LIMIT: 20 + REQUESTS_BY_ACCOUNT_PORT: 10006 + depends_on: + - postgres + transaction-by-id: + build: + context: ../ + dockerfile: ./docker/transaction-by-id.Dockerfile + ports: + - "10007:10007" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + TRANSACTION_BY_ID_PORT: 10007 + depends_on: + - postgres + transactions-by-account: + build: + context: ../ + dockerfile: ./docker/transactions-by-account.Dockerfile + ports: + - "10008:10008" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + PG_MAX_CONNECTIONS: 20 + PG_IDLE_TIMEOUT: 10000 + PG_CONN_TIMEOUT: 500 + READINESS_CHECK_PATH: /healthz + RETURN_RECORD_LIMIT: 20 + TRANSACTIONS_BY_ACCOUNT_PORT: 10008 + depends_on: + - postgres + client: + build: + context: ../ + dockerfile: ./docker/client.Dockerfile + args: + - GRAPHQL_URI=${GRAPHQL_URI} + ports: + - "10009:80" From 7206d707faab8a7cdb53d17fceb7d8b35998f9c1 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 17:33:48 -0700 Subject: [PATCH 10/54] file naming --- .../docker-entrypoint-initdb.d/{migrate.sh => up-migrate.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/bitnami-postgres/docker-entrypoint-initdb.d/{migrate.sh => up-migrate.sh} (100%) diff --git a/docker/bitnami-postgres/docker-entrypoint-initdb.d/migrate.sh b/docker/bitnami-postgres/docker-entrypoint-initdb.d/up-migrate.sh similarity index 100% rename from docker/bitnami-postgres/docker-entrypoint-initdb.d/migrate.sh rename to docker/bitnami-postgres/docker-entrypoint-initdb.d/up-migrate.sh From fca5c79d51573c735ccf69bdbcd56b2d2c7854c9 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 18:41:19 -0700 Subject: [PATCH 11/54] reference single compose file --- .../workflows/prod-client-docker-base.yaml | 2 +- client/makefile | 4 ++-- migrations/makefile | 24 ++++++++++++------- scripts/compose.sh | 15 +++--------- scripts/rebuild-client-image.sh | 6 +++-- scripts/rebuild-service.sh | 13 ++++------ 6 files changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/prod-client-docker-base.yaml b/.github/workflows/prod-client-docker-base.yaml index 56945aa6..1c92e52f 100644 --- a/.github/workflows/prod-client-docker-base.yaml +++ b/.github/workflows/prod-client-docker-base.yaml @@ -19,6 +19,6 @@ jobs: - name: docker login run: echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u mxfactorial --password-stdin - name: docker build - run: docker build -f ./docker/prod/client-base.Dockerfile -t "$IMAGE_NAME" . + run: docker build -f ./docker/client-base.Dockerfile -t "$IMAGE_NAME" . - name: docker push run: docker image push "$IMAGE_NAME" \ No newline at end of file diff --git a/client/makefile b/client/makefile index 26eca61e..403a549c 100644 --- a/client/makefile +++ b/client/makefile @@ -154,10 +154,10 @@ endif ###################### docker ###################### up: - docker compose -f ../docker/compose.client.yaml up -d --renew-anon-volumes --force-recreate --build + docker-compose --log-level ERROR -f ../docker/compose.yaml up -d client --renew-anon-volumes --force-recreate --build down: - docker compose -f ../docker/compose.client.yaml down + docker-compose --log-level ERROR -f ../docker/compose.yaml down ###################### utils ###################### diff --git a/migrations/makefile b/migrations/makefile index e29014f8..11127286 100644 --- a/migrations/makefile +++ b/migrations/makefile @@ -27,35 +27,43 @@ run: # postgres docker start: @COMPOSE_IGNORE_ORPHANS=true \ - docker compose \ - -f ../docker/compose.bitnami-postgres.yaml \ + docker-compose \ + --log-level ERROR \ + -f ../docker/compose.yaml \ up \ -d \ + postgres \ --renew-anon-volumes \ --force-recreate \ --build reset: @COMPOSE_IGNORE_ORPHANS=true \ - docker compose \ - -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.bitnami-postgres.yaml \ + docker-compose \ + --log-level ERROR \ + -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ up \ -d \ + postgres \ --renew-anon-volumes \ --force-recreate rebuild: - docker compose \ - -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.bitnami-postgres.yaml \ + @docker-compose \ + --log-level ERROR \ + -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ build \ + postgres \ --no-cache up: @$(MAKE) run down: - docker compose \ - -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.bitnami-postgres.yaml down + @docker-compose \ + --log-level ERROR \ + -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ + down stop: @$(MAKE) down diff --git a/scripts/compose.sh b/scripts/compose.sh index bb0ef7d8..22d36920 100644 --- a/scripts/compose.sh +++ b/scripts/compose.sh @@ -34,18 +34,9 @@ source ./scripts/manage-cde-ports.sh COMPOSE_DIR=./docker INIT_CMD="GRAPHQL_URI=$B64_GRAPHQL_URI \\ -docker compose \\ - -f $COMPOSE_DIR/compose.bitnami-postgres.yaml \\ - -f $COMPOSE_DIR/compose.rule.yaml \\ - -f $COMPOSE_DIR/compose.request-create.yaml \\ - -f $COMPOSE_DIR/compose.request-approve.yaml \\ - -f $COMPOSE_DIR/compose.transaction-by-id.yaml \\ - -f $COMPOSE_DIR/compose.transactions-by-account.yaml \\ - -f $COMPOSE_DIR/compose.request-by-id.yaml \\ - -f $COMPOSE_DIR/compose.requests-by-account.yaml \\ - -f $COMPOSE_DIR/compose.balance-by-account.yaml \\ - -f $COMPOSE_DIR/compose.graphql.yaml \\ - -f $COMPOSE_DIR/compose.client.yaml" +docker-compose \\ + --log-level ERROR \\ + -f $COMPOSE_DIR/compose.yaml" if [[ $UP ]]; then diff --git a/scripts/rebuild-client-image.sh b/scripts/rebuild-client-image.sh index c536752d..7061bfaf 100644 --- a/scripts/rebuild-client-image.sh +++ b/scripts/rebuild-client-image.sh @@ -4,7 +4,9 @@ source ./scripts/set-uri-vars.sh GRAPHQL_URI=$B64_GRAPHQL_URI \ - docker compose \ - -f ./docker/compose.client.yaml \ + docker-compose \ + --log-level ERROR \ + -f ./docker/compose.yaml \ build \ + client \ --no-cache \ No newline at end of file diff --git a/scripts/rebuild-service.sh b/scripts/rebuild-service.sh index c350c917..80a7c346 100644 --- a/scripts/rebuild-service.sh +++ b/scripts/rebuild-service.sh @@ -3,21 +3,19 @@ set -e # print use -if [[ "$#" -ne 2 ]] && [[ "$#" -ne 3 ]]; then +if [[ "$#" -ne 2 ]]; then cat <<- 'EOF' use: - bash scripts/rebuild-service.sh --name transactions-by-account # OPTIONAL: --no-db + bash scripts/rebuild-service.sh --name transactions-by-account EOF exit 1 fi -INCLUDE_DB='-f ./docker/compose.bitnami-postgres.yaml' # assign vars to script args while [[ "$#" -gt 0 ]]; do case $1 in --name) NAME="$2"; shift ;; - --no-db) unset INCLUDE_DB ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac shift @@ -28,13 +26,12 @@ source ./scripts/set-uri-vars.sh COMPOSE_IGNORE_ORPHANS=true \ GRAPHQL_URI=$B64_GRAPHQL_URI \ - docker compose \ - ${INCLUDE_DB} \ - -f ./docker/compose.$NAME.yaml \ + docker-compose \ + --log-level ERROR \ + -f ./docker/compose.yaml \ up \ -d \ --force-recreate \ --renew-anon-volumes \ - --no-deps \ --build \ $NAME \ No newline at end of file From 609cf32f70141dfe0a44e4d6864b69b61b413a80 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 18:41:31 -0700 Subject: [PATCH 12/54] remove script arg --- makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index 3b857014..12ac1231 100644 --- a/makefile +++ b/makefile @@ -203,10 +203,10 @@ rebuild-transactions-by-account: @bash scripts/rebuild-service.sh --name transactions-by-account rebuild-graphql: - @bash scripts/rebuild-service.sh --name graphql --no-db + @bash scripts/rebuild-service.sh --name graphql rebuild-client: - @bash scripts/rebuild-service.sh --name client --no-db + @bash scripts/rebuild-service.sh --name client ###################### demo ###################### From 934969967ac5bf7e20604dd2f2905ea12efdb13f Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 1 Apr 2024 19:24:12 -0700 Subject: [PATCH 13/54] use compose plugin --- .github/workflows/dev-client.yaml | 4 ++-- client/makefile | 4 ++-- docker/compose.yaml | 2 +- migrations/makefile | 12 ++++-------- scripts/compose.sh | 3 +-- scripts/rebuild-client-image.sh | 3 +-- scripts/rebuild-service.sh | 3 +-- tests/README.md | 2 +- 8 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/dev-client.yaml b/.github/workflows/dev-client.yaml index d4424fa9..862fa471 100644 --- a/.github/workflows/dev-client.yaml +++ b/.github/workflows/dev-client.yaml @@ -14,7 +14,7 @@ jobs: env: CI: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: start services run: make start - name: e2e test client @@ -29,7 +29,7 @@ jobs: AWS_DEFAULT_REGION: us-east-1 CI: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: make install working-directory: client diff --git a/client/makefile b/client/makefile index 403a549c..ae9305ea 100644 --- a/client/makefile +++ b/client/makefile @@ -154,10 +154,10 @@ endif ###################### docker ###################### up: - docker-compose --log-level ERROR -f ../docker/compose.yaml up -d client --renew-anon-volumes --force-recreate --build + docker compose -f ../docker/compose.yaml up -d client --renew-anon-volumes --force-recreate --build down: - docker-compose --log-level ERROR -f ../docker/compose.yaml down + docker compose -f ../docker/compose.yaml down ###################### utils ###################### diff --git a/docker/compose.yaml b/docker/compose.yaml index 24ba4155..e22437e6 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -191,6 +191,6 @@ services: context: ../ dockerfile: ./docker/client.Dockerfile args: - - GRAPHQL_URI=${GRAPHQL_URI} + - GRAPHQL_URI=${GRAPHQL_URI:-aHR0cDovL2xvY2FsaG9zdDoxMDAwMAo=} ports: - "10009:80" diff --git a/migrations/makefile b/migrations/makefile index 11127286..f8c299e1 100644 --- a/migrations/makefile +++ b/migrations/makefile @@ -27,8 +27,7 @@ run: # postgres docker start: @COMPOSE_IGNORE_ORPHANS=true \ - docker-compose \ - --log-level ERROR \ + docker compose \ -f ../docker/compose.yaml \ up \ -d \ @@ -39,8 +38,7 @@ start: reset: @COMPOSE_IGNORE_ORPHANS=true \ - docker-compose \ - --log-level ERROR \ + docker compose \ -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ up \ -d \ @@ -49,8 +47,7 @@ reset: --force-recreate rebuild: - @docker-compose \ - --log-level ERROR \ + @docker compose \ -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ build \ postgres \ @@ -60,8 +57,7 @@ up: @$(MAKE) run down: - @docker-compose \ - --log-level ERROR \ + @docker compose \ -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/compose.yaml \ down diff --git a/scripts/compose.sh b/scripts/compose.sh index 22d36920..0e99ffab 100644 --- a/scripts/compose.sh +++ b/scripts/compose.sh @@ -34,8 +34,7 @@ source ./scripts/manage-cde-ports.sh COMPOSE_DIR=./docker INIT_CMD="GRAPHQL_URI=$B64_GRAPHQL_URI \\ -docker-compose \\ - --log-level ERROR \\ +docker compose \\ -f $COMPOSE_DIR/compose.yaml" if [[ $UP ]]; then diff --git a/scripts/rebuild-client-image.sh b/scripts/rebuild-client-image.sh index 7061bfaf..307c5ff8 100644 --- a/scripts/rebuild-client-image.sh +++ b/scripts/rebuild-client-image.sh @@ -4,8 +4,7 @@ source ./scripts/set-uri-vars.sh GRAPHQL_URI=$B64_GRAPHQL_URI \ - docker-compose \ - --log-level ERROR \ + docker compose \ -f ./docker/compose.yaml \ build \ client \ diff --git a/scripts/rebuild-service.sh b/scripts/rebuild-service.sh index 80a7c346..7d6eebf3 100644 --- a/scripts/rebuild-service.sh +++ b/scripts/rebuild-service.sh @@ -26,8 +26,7 @@ source ./scripts/set-uri-vars.sh COMPOSE_IGNORE_ORPHANS=true \ GRAPHQL_URI=$B64_GRAPHQL_URI \ - docker-compose \ - --log-level ERROR \ + docker compose \ -f ./docker/compose.yaml \ up \ -d \ diff --git a/tests/README.md b/tests/README.md index 03eae36e..b660e8e0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,7 +11,7 @@ local: 1. `make start` 1. `make test-local` in a separate shell -docker-compose: +docker compose: 1. `make compose-up` 1. `make test-docker` in a separate shell From 884fb721a2d060dad0f725d5259122b078a2f2a9 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 16:10:09 -0700 Subject: [PATCH 14/54] auto-confirm dockerfile --- docker/auto-confirm.Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docker/auto-confirm.Dockerfile diff --git a/docker/auto-confirm.Dockerfile b/docker/auto-confirm.Dockerfile new file mode 100644 index 00000000..7422ef8a --- /dev/null +++ b/docker/auto-confirm.Dockerfile @@ -0,0 +1,20 @@ +FROM rust:latest as builder + +WORKDIR /app + +COPY . ./ + +RUN rustup target add x86_64-unknown-linux-musl +RUN apt update && \ + apt install -y musl-tools perl make + +RUN USER=root cargo build \ + --manifest-path=services/auto-confirm/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release + +FROM public.ecr.aws/lambda/provided:al2023 + +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/auto-confirm ./auto-confirm + +ENTRYPOINT [ "./auto-confirm" ] \ No newline at end of file From 63a2ab102e29ffdb123760a5821792a2ef18a5e4 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 16:11:46 -0700 Subject: [PATCH 15/54] add web adapter to service images --- docker/balance-by-account.Dockerfile | 30 +++++---------------- docker/graphql.Dockerfile | 30 +++++---------------- docker/request-approve.Dockerfile | 30 +++++---------------- docker/request-by-id.Dockerfile | 30 +++++---------------- docker/request-create.Dockerfile | 30 +++++---------------- docker/requests-by-account.Dockerfile | 30 +++++---------------- docker/rule.Dockerfile | 30 +++++---------------- docker/transaction-by-id.Dockerfile | 30 +++++---------------- docker/transactions-by-account.Dockerfile | 32 ++++++----------------- 9 files changed, 64 insertions(+), 208 deletions(-) diff --git a/docker/balance-by-account.Dockerfile b/docker/balance-by-account.Dockerfile index b52f6152..5f0f7cf2 100644 --- a/docker/balance-by-account.Dockerfile +++ b/docker/balance-by-account.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=balance-by-account -ENV UID=10004 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/balance-by-account/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/balance-by-account/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/balance-by-account /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/balance-by-account /app/balance-by-account EXPOSE 10004 -USER balance-by-account:balance-by-account - -CMD [ "/usr/local/bin/balance-by-account" ] \ No newline at end of file +CMD [ "/app/balance-by-account" ] \ No newline at end of file diff --git a/docker/graphql.Dockerfile b/docker/graphql.Dockerfile index f0e2e18a..0a795544 100644 --- a/docker/graphql.Dockerfile +++ b/docker/graphql.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=graphql -ENV UID=10001 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/graphql/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/graphql/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/graphql /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/graphql /app/graphql EXPOSE 10000 -USER graphql:graphql - -CMD [ "/usr/local/bin/graphql" ] \ No newline at end of file +CMD [ "/app/graphql" ] \ No newline at end of file diff --git a/docker/request-approve.Dockerfile b/docker/request-approve.Dockerfile index d1d01e9d..df7f8e2c 100644 --- a/docker/request-approve.Dockerfile +++ b/docker/request-approve.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=request-approve -ENV UID=10003 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/request-approve/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/request-approve/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-approve /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-approve /app/request-approve EXPOSE 10003 -USER request-approve:request-approve - -CMD [ "/usr/local/bin/request-approve" ] \ No newline at end of file +CMD [ "/app/request-approve" ] \ No newline at end of file diff --git a/docker/request-by-id.Dockerfile b/docker/request-by-id.Dockerfile index 674dc6a9..a8edc5ec 100644 --- a/docker/request-by-id.Dockerfile +++ b/docker/request-by-id.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=request-by-id -ENV UID=10005 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/request-by-id/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/request-by-id/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-by-id /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-by-id /app/request-by-id EXPOSE 10005 -USER request-by-id:request-by-id - -CMD [ "/usr/local/bin/request-by-id" ] \ No newline at end of file +CMD [ "/app/request-by-id" ] \ No newline at end of file diff --git a/docker/request-create.Dockerfile b/docker/request-create.Dockerfile index 8d5c7bce..b76b0a0c 100644 --- a/docker/request-create.Dockerfile +++ b/docker/request-create.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=request-create -ENV UID=10002 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/request-create/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/request-create/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-create /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-create /app/request-create EXPOSE 10002 -USER request-create:request-create - -CMD [ "/usr/local/bin/request-create" ] \ No newline at end of file +CMD [ "/app/request-create" ] \ No newline at end of file diff --git a/docker/requests-by-account.Dockerfile b/docker/requests-by-account.Dockerfile index 99f6e5f4..92290a1c 100644 --- a/docker/requests-by-account.Dockerfile +++ b/docker/requests-by-account.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=requests-by-account -ENV UID=10006 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/requests-by-account/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/requests-by-account/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/requests-by-account /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/requests-by-account /app/requests-by-account EXPOSE 10006 -USER requests-by-account:requests-by-account - -CMD [ "/usr/local/bin/requests-by-account" ] \ No newline at end of file +CMD [ "/app/requests-by-account" ] \ No newline at end of file diff --git a/docker/rule.Dockerfile b/docker/rule.Dockerfile index d1caf92a..c3bfbfc7 100644 --- a/docker/rule.Dockerfile +++ b/docker/rule.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=rule -ENV UID=10001 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/rule/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/rule/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/rule /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/rule /app/rule EXPOSE 10001 -USER rule:rule - -CMD [ "/usr/local/bin/rule" ] \ No newline at end of file +CMD [ "/app/rule" ] \ No newline at end of file diff --git a/docker/transaction-by-id.Dockerfile b/docker/transaction-by-id.Dockerfile index 8867048f..16313042 100644 --- a/docker/transaction-by-id.Dockerfile +++ b/docker/transaction-by-id.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=transaction-by-id -ENV UID=10007 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/transaction-by-id/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/transaction-by-id/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transaction-by-id /usr/local/bin +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transaction-by-id /app/transaction-by-id EXPOSE 10007 -USER transaction-by-id:transaction-by-id - -CMD [ "/usr/local/bin/transaction-by-id" ] \ No newline at end of file +CMD [ "/app/transaction-by-id" ] \ No newline at end of file diff --git a/docker/transactions-by-account.Dockerfile b/docker/transactions-by-account.Dockerfile index 8f55b8de..0f2c61b5 100644 --- a/docker/transactions-by-account.Dockerfile +++ b/docker/transactions-by-account.Dockerfile @@ -6,34 +6,18 @@ COPY . ./ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ - apt install -y musl-tools perl make -RUN update-ca-certificates - -ENV USER=transactions-by-account -ENV UID=10007 - -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + apt install -y musl-tools perl make RUN USER=root cargo build \ - --manifest-path=services/transactions-by-account/Cargo.toml \ - --target x86_64-unknown-linux-musl \ - --release + --manifest-path=services/transactions-by-account/Cargo.toml \ + --target x86_64-unknown-linux-musl \ + --release FROM alpine -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transactions-by-account /usr/local/bin - -EXPOSE 10007 +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transactions-by-account /app/transactions-by-account -USER transactions-by-account:transactions-by-account +EXPOSE 10008 -CMD [ "/usr/local/bin/transactions-by-account" ] \ No newline at end of file +CMD [ "/app/transactions-by-account" ] \ No newline at end of file From ee4b3b43ab8e7e6c0eb648ddd49f01a9af48512e Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 16:18:30 -0700 Subject: [PATCH 16/54] ecr and lambda targets in separate makefile --- make/ecr-lambda.mk | 44 +++++++++++++++++++++++++++++ make/shared.mk | 70 +--------------------------------------------- 2 files changed, 45 insertions(+), 69 deletions(-) create mode 100644 make/ecr-lambda.mk diff --git a/make/ecr-lambda.mk b/make/ecr-lambda.mk new file mode 100644 index 00000000..72ab4eb5 --- /dev/null +++ b/make/ecr-lambda.mk @@ -0,0 +1,44 @@ +BUILD_CTX?=. + +build-image: + @cd $(RELATIVE_PROJECT_ROOT_PATH); \ + bash scripts/build-image.sh --app-name $(APP_NAME) --build-ctx $(BUILD_CTX) + +tag-dev-image: + @cd $(RELATIVE_PROJECT_ROOT_PATH); \ + bash scripts/tag-dev-image.sh --app-name $(APP_NAME) + +push-dev-image: + @cd $(RELATIVE_PROJECT_ROOT_PATH); \ + bash scripts/push-dev-image.sh --app-name $(APP_NAME) + +deploy-dev-image: + @cd $(RELATIVE_PROJECT_ROOT_PATH); \ + bash scripts/deploy-dev-image.sh --app-name $(APP_NAME) + +update-dev-function: + @$(MAKE) -s build-image + @$(MAKE) -s tag-dev-image + @$(MAKE) -s push-dev-image + @$(MAKE) -s deploy-dev-image + +clean-image: + @for i in $$(docker image ls | grep '$(APP_NAME)' | awk '{print $$3}'); do docker rmi -f "$$i"; done; + +###################### globally required ###################### + +initial-deploy: + @$(MAKE) -s build-image + @$(MAKE) -s tag-dev-image + @$(MAKE) -s push-dev-image + +deploy: + @$(MAKE) -s update-dev-function + +deploy-only: + @$(MAKE) -s tag-dev-image + @$(MAKE) -s push-dev-image + @$(MAKE) -s deploy-dev-image + +now: + @$(MAKE) -s update-dev-function \ No newline at end of file diff --git a/make/shared.mk b/make/shared.mk index f16c6531..b47d9123 100644 --- a/make/shared.mk +++ b/make/shared.mk @@ -5,8 +5,6 @@ PROJECT_CONF=$(RELATIVE_PROJECT_ROOT_PATH)/$(PROJECT_CONF_FILE_NAME) ROOT_PATH=$(shell cd $(RELATIVE_PROJECT_ROOT_PATH); pwd) SUB_PATH=$(shell printf '%s' $(CURDIR) | awk -F'$(ROOT_PATH)' '{print substr($$NF, 2)}') REGION=$(shell yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $(PROJECT_CONF)) -EXECUTABLE_NAME=$(shell yq '.services.env_var.set.BINARY_NAME.default' $(PROJECT_CONF)) -ARTIFACT_NAME=$(APP_NAME)-src.zip LAMBDA_NAME=$(APP_NAME)-$(ENV) ENV_FILE_NAME=$(shell yq '.env_var.set.ENV_FILE_NAME.default' $(PROJECT_CONF)) ENV_FILE=$(CURDIR)/$(ENV_FILE_NAME) @@ -14,12 +12,6 @@ NOHUP_LOG=$(RELATIVE_PROJECT_ROOT_PATH)/$(shell yq '.env_var.set.NOHUP_LOG.defau LOCAL_ADDRESS=$(shell yq '.env_var.set.LOCAL_ADDRESS.default' $(PROJECT_CONF)) HOST=http://$(LOCAL_ADDRESS) -DOCKER_ENV_VARS=PGDATABASE=mxfactorial \ -PGUSER=test \ -PGPASSWORD=test \ -PGHOST=localhost \ -PGPORT=5432 - test-env-file: ifeq (,$(wildcard $(ENV_FILE))) $(error no .env file, run 'make get-secrets ENV=dev') @@ -30,33 +22,11 @@ ifndef ENV $(error trailing ENV assignment missing, e.g. make test ENV=dev) endif -test-acc-arg: -ifndef ACC - $(error trailing ACC assignment missing, e.g. make createuser ACC=testuser) -endif - -clean-artifact: - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/clean-artifact.sh \ - --app-name $(APP_NAME) \ - --artifact-name $(ARTIFACT_NAME) - clean-log: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ bash scripts/clean-invoke-log.sh \ --app-name $(APP_NAME) -clean: - @$(MAKE) clean-build - @$(MAKE) clean-artifact - @$(MAKE) clean-log - -build: - @$(MAKE) clean - @$(MAKE) install - @$(MAKE) compile - @$(MAKE) zip - env: @$(MAKE) get-secrets @@ -72,48 +42,10 @@ clean-env: bash scripts/clean-env.sh \ --app-name $(APP_NAME) -put-object: - @$(MAKE) -s test-env-arg - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/put-object.sh \ - --app-name $(APP_NAME) \ - --artifact-name $(ARTIFACT_NAME) \ - --env $(ENV) - -update-function: - @$(MAKE) -s test-env-arg - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/update-function.sh \ - --app-name $(APP_NAME) \ - --artifact-name $(ARTIFACT_NAME) \ - --env $(ENV) - -initial-deploy: - @$(MAKE) -s test-env-arg - $(MAKE) build - $(MAKE) put-object - -deploy: - @$(MAKE) -s test-env-arg - $(MAKE) build - $(MAKE) put-object - $(MAKE) update-function - -deploy-only: - @$(MAKE) -s test-env-arg - $(MAKE) put-object - $(MAKE) update-function - -now: - @$(MAKE) -s clean - $(MAKE) compile - $(MAKE) zip - $(MAKE) deploy-only ENV=dev - invoke: @$(MAKE) invoke-local -invoke-function: +invoke-lambda: @$(MAKE) -s test-env-arg @cd $(RELATIVE_PROJECT_ROOT_PATH); \ bash scripts/invoke-function-url.sh \ From 490a3004e840bab2fc9744be71a1caa689360d31 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 16:59:14 -0700 Subject: [PATCH 17/54] test services separately then build and push their images to dev registry --- .github/workflows/auto-confirm.yaml | 50 ++++++++++ .github/workflows/balance-by-account.yaml | 90 ++++++++++++++++++ .github/workflows/graphql.yaml | 79 ++++++++++++++++ .github/workflows/request-approve.yaml | 91 ++++++++++++++++++ .github/workflows/request-by-id.yaml | 91 ++++++++++++++++++ .github/workflows/request-create.yaml | 91 ++++++++++++++++++ .github/workflows/requests-by-account.yaml | 91 ++++++++++++++++++ .github/workflows/rule.yaml | 93 +++++++++++++++++++ .github/workflows/transaction-by-id.yaml | 90 ++++++++++++++++++ .../workflows/transactions-by-account.yaml | 90 ++++++++++++++++++ 10 files changed, 856 insertions(+) create mode 100644 .github/workflows/auto-confirm.yaml create mode 100644 .github/workflows/balance-by-account.yaml create mode 100644 .github/workflows/graphql.yaml create mode 100644 .github/workflows/request-approve.yaml create mode 100644 .github/workflows/request-by-id.yaml create mode 100644 .github/workflows/request-create.yaml create mode 100644 .github/workflows/requests-by-account.yaml create mode 100644 .github/workflows/rule.yaml create mode 100644 .github/workflows/transaction-by-id.yaml create mode 100644 .github/workflows/transactions-by-account.yaml diff --git a/.github/workflows/auto-confirm.yaml b/.github/workflows/auto-confirm.yaml new file mode 100644 index 00000000..7ec6d951 --- /dev/null +++ b/.github/workflows/auto-confirm.yaml @@ -0,0 +1,50 @@ +name: auto-confirm + +on: + push: + paths: + - 'services/auto-confirm/**' + branches-ignore: + - 'master' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/auto-confirm + needs: [lint_test, unit_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml new file mode 100644 index 00000000..33fa1391 --- /dev/null +++ b/.github/workflows/balance-by-account.yaml @@ -0,0 +1,90 @@ +name: balance-by-account + +on: + push: + paths: + - 'services/balance-by-account/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/balance-by-account + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml new file mode 100644 index 00000000..112929d6 --- /dev/null +++ b/.github/workflows/graphql.yaml @@ -0,0 +1,79 @@ +name: graphql + +on: + push: + paths: + - 'services/graphql/**' + - 'crates/**' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/graphql + needs: [lint_test, unit_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image + diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml new file mode 100644 index 00000000..c9c22b5a --- /dev/null +++ b/.github/workflows/request-approve.yaml @@ -0,0 +1,91 @@ +name: request-approve + +on: + push: + paths: + - 'services/request-approve/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/request-approve + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image + diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml new file mode 100644 index 00000000..b9b287a7 --- /dev/null +++ b/.github/workflows/request-by-id.yaml @@ -0,0 +1,91 @@ +name: request-by-id + +on: + push: + paths: + - 'services/request-by-id/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/request-by-id + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image + diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml new file mode 100644 index 00000000..f8273404 --- /dev/null +++ b/.github/workflows/request-create.yaml @@ -0,0 +1,91 @@ +name: request-create + +on: + push: + paths: + - 'services/request-create/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/request-create + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image + diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml new file mode 100644 index 00000000..64aeccff --- /dev/null +++ b/.github/workflows/requests-by-account.yaml @@ -0,0 +1,91 @@ +name: requests-by-account + +on: + push: + paths: + - 'services/requests-by-account/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/requests-by-account + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image + diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml new file mode 100644 index 00000000..720b8549 --- /dev/null +++ b/.github/workflows/rule.yaml @@ -0,0 +1,93 @@ +name: rule + +on: + push: + paths: + - 'services/rule/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + - name: coverage report + run: | + make rust-coverage RUST_PKG=rule + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/rule + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml new file mode 100644 index 00000000..a1e22aff --- /dev/null +++ b/.github/workflows/transaction-by-id.yaml @@ -0,0 +1,90 @@ +name: transaction-by-id + +on: + push: + paths: + - 'services/transaction-by-id/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/transaction-by-id + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml new file mode 100644 index 00000000..314f49a6 --- /dev/null +++ b/.github/workflows/transactions-by-account.yaml @@ -0,0 +1,90 @@ +name: transactions-by-account + +on: + push: + paths: + - 'services/transactions-by-account/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +jobs: + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: unit test + run: cargo test + database_test: + name: database test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: test database + run: make -C crates/pg test-db + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + client_test: + name: client test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-c + - name: clean up + run: make stop + push_image: + name: push image to dev ecr + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APP_DIR: services/transactions-by-account + needs: [lint_test, unit_test, database_test, integration_test, client_test] + steps: + - name: build image + run: make -C $APP_DIR build-image + - name: tag image + run: make -C $APP_DIR tag-dev-image + - name: push image + run: make -C $APP_DIR push-dev-image \ No newline at end of file From 7af2cab71f3fc9e95d45f091e5f59ae132f47f7c Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:01:08 -0700 Subject: [PATCH 18/54] workflow file naming --- .github/workflows/{dev-client.yaml => client.yaml} | 5 +---- .../{dev-crates-postgres.yaml => crates-postgres.yaml} | 2 +- .github/workflows/{dev-crates.yaml => crates.yaml} | 2 +- .github/workflows/{dev-migrations.yaml => migrations.yaml} | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) rename .github/workflows/{dev-client.yaml => client.yaml} (95%) rename .github/workflows/{dev-crates-postgres.yaml => crates-postgres.yaml} (95%) rename .github/workflows/{dev-crates.yaml => crates.yaml} (98%) rename .github/workflows/{dev-migrations.yaml => migrations.yaml} (95%) diff --git a/.github/workflows/dev-client.yaml b/.github/workflows/client.yaml similarity index 95% rename from .github/workflows/dev-client.yaml rename to .github/workflows/client.yaml index 862fa471..e216201a 100644 --- a/.github/workflows/dev-client.yaml +++ b/.github/workflows/client.yaml @@ -1,4 +1,4 @@ -name: dev-client +name: client on: push: @@ -11,8 +11,6 @@ jobs: test: name: test client runs-on: ubuntu-latest - env: - CI: true steps: - uses: actions/checkout@v4 - name: start services @@ -27,7 +25,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 - CI: true steps: - uses: actions/checkout@v4 - name: install dependencies diff --git a/.github/workflows/dev-crates-postgres.yaml b/.github/workflows/crates-postgres.yaml similarity index 95% rename from .github/workflows/dev-crates-postgres.yaml rename to .github/workflows/crates-postgres.yaml index 143500f3..98ddeff1 100644 --- a/.github/workflows/dev-crates-postgres.yaml +++ b/.github/workflows/crates-postgres.yaml @@ -1,4 +1,4 @@ -name: dev-crates-postgres +name: crates-postgres on: push: diff --git a/.github/workflows/dev-crates.yaml b/.github/workflows/crates.yaml similarity index 98% rename from .github/workflows/dev-crates.yaml rename to .github/workflows/crates.yaml index f77d9cc0..54d4b414 100644 --- a/.github/workflows/dev-crates.yaml +++ b/.github/workflows/crates.yaml @@ -1,4 +1,4 @@ -name: dev-crates +name: crates on: push: diff --git a/.github/workflows/dev-migrations.yaml b/.github/workflows/migrations.yaml similarity index 95% rename from .github/workflows/dev-migrations.yaml rename to .github/workflows/migrations.yaml index 7f13fbee..3563a925 100644 --- a/.github/workflows/dev-migrations.yaml +++ b/.github/workflows/migrations.yaml @@ -1,4 +1,4 @@ -name: dev-migrations +name: migrations on: push: @@ -22,7 +22,7 @@ jobs: POSTGRES_PASSWORD: test POSTGRESQL_DATABASE: mxfactorial steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19.x' From a4ee1f1215d29e9978fae1eee052af7340f27ba0 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:03:19 -0700 Subject: [PATCH 19/54] remove dev service workflows --- .github/workflows/dev-auto-confirm.yaml | 39 ------------------- .github/workflows/dev-balance-by-account.yaml | 39 ------------------- .github/workflows/dev-go-migrate.yaml | 23 ----------- .github/workflows/dev-graphql.yaml | 39 ------------------- .github/workflows/dev-request-approve.yaml | 39 ------------------- .github/workflows/dev-request-by-id.yaml | 39 ------------------- .github/workflows/dev-request-create.yaml | 39 ------------------- .../workflows/dev-requests-by-account.yaml | 39 ------------------- .github/workflows/dev-rule.yaml | 33 ---------------- .github/workflows/dev-transaction-by-id.yaml | 39 ------------------- .../dev-transactions-by-account.yaml | 39 ------------------- 11 files changed, 407 deletions(-) delete mode 100644 .github/workflows/dev-auto-confirm.yaml delete mode 100644 .github/workflows/dev-balance-by-account.yaml delete mode 100644 .github/workflows/dev-go-migrate.yaml delete mode 100644 .github/workflows/dev-graphql.yaml delete mode 100644 .github/workflows/dev-request-approve.yaml delete mode 100644 .github/workflows/dev-request-by-id.yaml delete mode 100644 .github/workflows/dev-request-create.yaml delete mode 100644 .github/workflows/dev-requests-by-account.yaml delete mode 100644 .github/workflows/dev-rule.yaml delete mode 100644 .github/workflows/dev-transaction-by-id.yaml delete mode 100644 .github/workflows/dev-transactions-by-account.yaml diff --git a/.github/workflows/dev-auto-confirm.yaml b/.github/workflows/dev-auto-confirm.yaml deleted file mode 100644 index 8c309860..00000000 --- a/.github/workflows/dev-auto-confirm.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-auto-confirm - -on: - push: - paths: - - 'services/auto-confirm/**' - branches-ignore: - - 'master' - -jobs: - build: - name: auto-confirm - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/auto-confirm - - name: compile - run: make compile - working-directory: services/auto-confirm - - name: zip - run: make zip - working-directory: services/auto-confirm - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/auto-confirm \ No newline at end of file diff --git a/.github/workflows/dev-balance-by-account.yaml b/.github/workflows/dev-balance-by-account.yaml deleted file mode 100644 index 1935657b..00000000 --- a/.github/workflows/dev-balance-by-account.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-balance-by-account - -on: - push: - paths: - - 'services/balance-by-account/**' - branches-ignore: - - 'master' - -jobs: - build: - name: balance-by-account - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/balance-by-account - - name: compile - run: make compile - working-directory: services/balance-by-account - - name: zip - run: make zip - working-directory: services/balance-by-account - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/balance-by-account \ No newline at end of file diff --git a/.github/workflows/dev-go-migrate.yaml b/.github/workflows/dev-go-migrate.yaml deleted file mode 100644 index 64827526..00000000 --- a/.github/workflows/dev-go-migrate.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: dev-go-migrate - -on: - push: - paths: - - 'migrations/go-migrate/**' - branches-ignore: - - 'master' - -jobs: - build: - name: go-migrate - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - name: deploy - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy ENV=dev - working-directory: migrations/go-migrate \ No newline at end of file diff --git a/.github/workflows/dev-graphql.yaml b/.github/workflows/dev-graphql.yaml deleted file mode 100644 index 6e1e60a2..00000000 --- a/.github/workflows/dev-graphql.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-graphql - -on: - push: - paths: - - 'services/graphql/**' - branches-ignore: - - 'master' - -jobs: - build: - name: graphql - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/graphql - - name: compile - run: make compile - working-directory: services/graphql - - name: zip - run: make zip - working-directory: services/graphql - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/graphql \ No newline at end of file diff --git a/.github/workflows/dev-request-approve.yaml b/.github/workflows/dev-request-approve.yaml deleted file mode 100644 index 45dc4f48..00000000 --- a/.github/workflows/dev-request-approve.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-request-approve - -on: - push: - paths: - - 'services/request-approve/**' - branches-ignore: - - 'master' - -jobs: - build: - name: request-approve - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/request-approve - - name: compile - run: make compile - working-directory: services/request-approve - - name: zip - run: make zip - working-directory: services/request-approve - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/request-approve \ No newline at end of file diff --git a/.github/workflows/dev-request-by-id.yaml b/.github/workflows/dev-request-by-id.yaml deleted file mode 100644 index e317cdc4..00000000 --- a/.github/workflows/dev-request-by-id.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-request-by-id - -on: - push: - paths: - - 'services/request-by-id/**' - branches-ignore: - - 'master' - -jobs: - build: - name: request-by-id - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/request-by-id - - name: compile - run: make compile - working-directory: services/request-by-id - - name: zip - run: make zip - working-directory: services/request-by-id - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/request-by-id \ No newline at end of file diff --git a/.github/workflows/dev-request-create.yaml b/.github/workflows/dev-request-create.yaml deleted file mode 100644 index 8ead20f2..00000000 --- a/.github/workflows/dev-request-create.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-request-create - -on: - push: - paths: - - 'services/request-create/**' - branches-ignore: - - 'master' - -jobs: - build: - name: request-create - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/request-create - - name: compile - run: make compile - working-directory: services/request-create - - name: zip - run: make zip - working-directory: services/request-create - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/request-create \ No newline at end of file diff --git a/.github/workflows/dev-requests-by-account.yaml b/.github/workflows/dev-requests-by-account.yaml deleted file mode 100644 index 7cc29985..00000000 --- a/.github/workflows/dev-requests-by-account.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-requests-by-account - -on: - push: - paths: - - 'services/requests-by-account/**' - branches-ignore: - - 'master' - -jobs: - build: - name: requests-by-account - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/requests-by-account - - name: compile - run: make compile - working-directory: services/requests-by-account - - name: zip - run: make zip - working-directory: services/requests-by-account - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/requests-by-account \ No newline at end of file diff --git a/.github/workflows/dev-rule.yaml b/.github/workflows/dev-rule.yaml deleted file mode 100644 index 6c0db20c..00000000 --- a/.github/workflows/dev-rule.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: dev-rule - -on: - push: - paths: - - 'services/rule/**' - branches-ignore: - - 'master' - -jobs: - test: - name: rule - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/rule - - name: install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: services/rule unit tests - run: cargo test - working-directory: services/rule - - name: services/rule coverage report - run: | - make rust-coverage RUST_PKG=rule \ No newline at end of file diff --git a/.github/workflows/dev-transaction-by-id.yaml b/.github/workflows/dev-transaction-by-id.yaml deleted file mode 100644 index eee7f1f7..00000000 --- a/.github/workflows/dev-transaction-by-id.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-transaction-by-id - -on: - push: - paths: - - 'services/transaction-by-id/**' - branches-ignore: - - 'master' - -jobs: - build: - name: transaction-by-id - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/transaction-by-id - - name: compile - run: make compile - working-directory: services/transaction-by-id - - name: zip - run: make zip - working-directory: services/transaction-by-id - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/transaction-by-id \ No newline at end of file diff --git a/.github/workflows/dev-transactions-by-account.yaml b/.github/workflows/dev-transactions-by-account.yaml deleted file mode 100644 index 1d2e8841..00000000 --- a/.github/workflows/dev-transactions-by-account.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: dev-transactions-by-account - -on: - push: - paths: - - 'services/transactions-by-account/**' - branches-ignore: - - 'master' - -jobs: - build: - name: transactions-by-account - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: taiki-e/install-action@nextest - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - working-directory: services/transactions-by-account - - name: compile - run: make compile - working-directory: services/transactions-by-account - - name: zip - run: make zip - working-directory: services/transactions-by-account - - name: deploy to dev - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make deploy-only ENV=dev - working-directory: services/transactions-by-account \ No newline at end of file From 53e0721a6c36b632dea200dd7dd444f4932d1000 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:04:45 -0700 Subject: [PATCH 20/54] deploy dev images and test cloud integration on develop branch merge --- .github/workflows/dev-integration.yaml | 52 ++++++++++---------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index 03ec55ab..b2c2d71b 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -1,4 +1,4 @@ -name: dev-integration +name: integration on: push: @@ -12,47 +12,29 @@ on: - 'services/requests-by-account/**' - 'services/transaction-by-id/**' - 'services/transactions-by-account/**' - - 'migrations/schema/**' + - 'migrations/schema/*' - 'tests/**' - branches-ignore: - - 'master' + branches: + - 'develop' jobs: - integration: - name: integration test + integration_test: + name: cloud integration test runs-on: ubuntu-latest env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 - CI: true steps: - uses: actions/checkout@v4 - - name: install latest psql client + - name: deploy to dev cloud environment run: | - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo apt-get update - sudo apt-get install --yes --no-install-recommends postgresql-client - - name: install golang-migrate - run: | - curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb - sudo dpkg -i migrate.linux-amd64.deb - rm migrate.linux-amd64.deb - - name: start services - run: make start - - name: test service integration - run: make -C ./tests test-local - - name: reset db - run: make reset-db - - name: e2e test client - run: make -C ./client test-ci - - name: set current env-id for cloud - run: make resume-dev ENV_ID=${{ secrets.DEV_ENV_ID }} - - name: install cargo cross - run: cargo install cross --git https://github.com/cross-rs/cross - - name: deploy to cloud - run: bash scripts/deploy-all.sh --env dev --transaction-services-only + for app in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do + # tag newly pushed dev images with develop merge commit sha + bash scripts/tag-merge-commit.sh --app-name $app --env dev --env-id ${{ secrets.DEV_ENV_ID }}; + # deploy newly tagged images to dev cloud + bash scripts/deploy-last-image.sh --app-name $app --env dev --env-id ${{ secrets.DEV_ENV_ID }}; + done - name: reset rds database for integration tests run: make --no-print-directory -C ./migrations resetrds ENV=dev DB=test - name: dump rds database locally for restore between integration tests @@ -60,4 +42,10 @@ jobs: - name: get secrets for dev integration tests run: make --no-print-directory -C ./tests get-secrets ENV=dev - name: run dev cloud integration tests - run: cargo test --manifest-path ./tests/Cargo.toml --features integration_tests -- --test-threads=1 \ No newline at end of file + run: cargo test --manifest-path ./tests/Cargo.toml --features integration_tests -- --test-threads=1 + - name: tag and push dev images to prod ecr + run: | + for app in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do + # tag newly tested dev images with prod and push to prod ecr + bash scripts/push-prod-image.sh --app-name $app --env dev --env-id ${{ secrets.DEV_ENV_ID }}; + done \ No newline at end of file From 206b58fd16dd1b86e66e80b4780c0dc58e2f8367 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:05:12 -0700 Subject: [PATCH 21/54] bump to v4 checkout action --- .github/workflows/prod-client.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/prod-client.yaml b/.github/workflows/prod-client.yaml index 8340ea9d..dbfef248 100644 --- a/.github/workflows/prod-client.yaml +++ b/.github/workflows/prod-client.yaml @@ -15,9 +15,8 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 - CI: true steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: install dependencies run: make install working-directory: client From e2ef61bb69706823cf099f04f25e36a633ae95a1 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:06:17 -0700 Subject: [PATCH 22/54] deploy prod services on master merge --- .github/workflows/prod-auto-confirm.yaml | 33 ------------------- .../workflows/prod-balance-by-account.yaml | 33 ------------------- .github/workflows/prod-go-migrate.yaml | 23 ------------- .github/workflows/prod-graphql.yaml | 33 ------------------- .github/workflows/prod-request-approve.yaml | 33 ------------------- .github/workflows/prod-request-by-id.yaml | 33 ------------------- .github/workflows/prod-request-create.yaml | 33 ------------------- .../workflows/prod-requests-by-account.yaml | 33 ------------------- .github/workflows/prod-rule.yaml | 27 --------------- .github/workflows/prod-services-deploy.yaml | 21 ++++++++++++ .github/workflows/prod-transaction-by-id.yaml | 33 ------------------- .../prod-transactions-by-account.yaml | 33 ------------------- 12 files changed, 21 insertions(+), 347 deletions(-) delete mode 100644 .github/workflows/prod-auto-confirm.yaml delete mode 100644 .github/workflows/prod-balance-by-account.yaml delete mode 100644 .github/workflows/prod-go-migrate.yaml delete mode 100644 .github/workflows/prod-graphql.yaml delete mode 100644 .github/workflows/prod-request-approve.yaml delete mode 100644 .github/workflows/prod-request-by-id.yaml delete mode 100644 .github/workflows/prod-request-create.yaml delete mode 100644 .github/workflows/prod-requests-by-account.yaml delete mode 100644 .github/workflows/prod-rule.yaml create mode 100644 .github/workflows/prod-services-deploy.yaml delete mode 100644 .github/workflows/prod-transaction-by-id.yaml delete mode 100644 .github/workflows/prod-transactions-by-account.yaml diff --git a/.github/workflows/prod-auto-confirm.yaml b/.github/workflows/prod-auto-confirm.yaml deleted file mode 100644 index 6f92b2c5..00000000 --- a/.github/workflows/prod-auto-confirm.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-auto-confirm - -on: - push: - paths: - - 'services/auto-confirm/**' - branches: - - 'master' -# todo: promote artifacts with s3 cp instead of rebuilding -jobs: - build: - name: auto-confirm - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/auto-confirm - - name: zip - run: make zip - working-directory: services/auto-confirm - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/auto-confirm \ No newline at end of file diff --git a/.github/workflows/prod-balance-by-account.yaml b/.github/workflows/prod-balance-by-account.yaml deleted file mode 100644 index e025ed83..00000000 --- a/.github/workflows/prod-balance-by-account.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-balance-by-account - -on: - push: - paths: - - 'services/balance-by-account/**' - branches: - - 'master' - -jobs: - test: - name: balance-by-account in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/balance-by-account - - name: zip - run: make zip - working-directory: services/balance-by-account - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/balance-by-account \ No newline at end of file diff --git a/.github/workflows/prod-go-migrate.yaml b/.github/workflows/prod-go-migrate.yaml deleted file mode 100644 index e2bc8ba2..00000000 --- a/.github/workflows/prod-go-migrate.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: prod-go-migrate - -on: - push: - paths: - - 'migrations/go-migrate/**' - branches: - - 'master' - -jobs: - build: - name: go-migrate - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - name: deploy - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy ENV=prod - working-directory: migrations/go-migrate \ No newline at end of file diff --git a/.github/workflows/prod-graphql.yaml b/.github/workflows/prod-graphql.yaml deleted file mode 100644 index ef17f169..00000000 --- a/.github/workflows/prod-graphql.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-graphql - -on: - push: - paths: - - 'services/graphql/**' - branches: - - 'master' - -jobs: - test: - name: graphql - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/graphql - - name: zip - run: make zip - working-directory: services/graphql - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/graphql \ No newline at end of file diff --git a/.github/workflows/prod-request-approve.yaml b/.github/workflows/prod-request-approve.yaml deleted file mode 100644 index 88fb9bfa..00000000 --- a/.github/workflows/prod-request-approve.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-request-approve - -on: - push: - paths: - - 'services/request-approve/**' - branches: - - 'master' - -jobs: - build: - name: request-approve in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/request-approve - - name: zip - run: make zip - working-directory: services/request-approve - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/request-approve \ No newline at end of file diff --git a/.github/workflows/prod-request-by-id.yaml b/.github/workflows/prod-request-by-id.yaml deleted file mode 100644 index aeb9c8c9..00000000 --- a/.github/workflows/prod-request-by-id.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-request-by-id - -on: - push: - paths: - - 'services/request-by-id/**' - branches: - - 'master' - -jobs: - build: - name: request-by-id in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/request-by-id - - name: zip - run: make zip - working-directory: services/request-by-id - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/request-by-id \ No newline at end of file diff --git a/.github/workflows/prod-request-create.yaml b/.github/workflows/prod-request-create.yaml deleted file mode 100644 index 46de1f17..00000000 --- a/.github/workflows/prod-request-create.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-request-create - -on: - push: - paths: - - 'services/request-create/**' - branches: - - 'master' - -jobs: - build: - name: request-create in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/request-create - - name: zip - run: make zip - working-directory: services/request-create - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/request-create \ No newline at end of file diff --git a/.github/workflows/prod-requests-by-account.yaml b/.github/workflows/prod-requests-by-account.yaml deleted file mode 100644 index 0347f4af..00000000 --- a/.github/workflows/prod-requests-by-account.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-requests-by-account - -on: - push: - paths: - - 'services/requests-by-account/**' - branches: - - 'master' - -jobs: - build: - name: requests-by-account in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/requests-by-account - - name: zip - run: make zip - working-directory: services/requests-by-account - - name: deploy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/requests-by-account \ No newline at end of file diff --git a/.github/workflows/prod-rule.yaml b/.github/workflows/prod-rule.yaml deleted file mode 100644 index 4327aa75..00000000 --- a/.github/workflows/prod-rule.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: prod-rule - -on: - push: - paths: - - 'services/rule/**' - branches: - - 'master' - -jobs: - deploy: - name: deploy rule to prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: deploy - working-directory: services/rule - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy ENV=prod \ No newline at end of file diff --git a/.github/workflows/prod-services-deploy.yaml b/.github/workflows/prod-services-deploy.yaml new file mode 100644 index 00000000..c4ff7b50 --- /dev/null +++ b/.github/workflows/prod-services-deploy.yaml @@ -0,0 +1,21 @@ +name: prod-services-deploy + +on: + push: + branches: + - 'master' + +jobs: + deploy_prod: + name: deploy services to prod cloud environment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: deploy services to prod cloud environment + run: | + for app in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do + # tag newly pushed prod images with master merge commit sha + bash scripts/tag-merge-commit.sh --app-name $app --env prod --env-id ${{ secrets.PROD_ENV_ID }}; + # deploy newly tagged images to prod cloud + bash scripts/deploy-last-image.sh --app-name $app --env prod --env-id ${{ secrets.PROD_ENV_ID }}; + done \ No newline at end of file diff --git a/.github/workflows/prod-transaction-by-id.yaml b/.github/workflows/prod-transaction-by-id.yaml deleted file mode 100644 index a9ffef6b..00000000 --- a/.github/workflows/prod-transaction-by-id.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-transaction-by-id - -on: - push: - paths: - - 'services/transaction-by-id/**' - branches: - - 'master' - -jobs: - build: - name: transaction-by-id in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/transaction-by-id - - name: zip - run: make zip - working-directory: services/transaction-by-id - - name: deloy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/transaction-by-id \ No newline at end of file diff --git a/.github/workflows/prod-transactions-by-account.yaml b/.github/workflows/prod-transactions-by-account.yaml deleted file mode 100644 index 7d0f69d6..00000000 --- a/.github/workflows/prod-transactions-by-account.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: prod-transactions-by-account - -on: - push: - paths: - - 'services/transactions-by-account/**' - branches: - - 'master' - -jobs: - build: - name: transactions-by-account in prod - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - CI: true - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - name: compile - run: make compile - working-directory: services/transactions-by-account - - name: zip - run: make zip - working-directory: services/transactions-by-account - - name: deloy to prod - run: ENV_ID=${{ secrets.PROD_ENV_ID }} make deploy-only ENV=prod - working-directory: services/transactions-by-account \ No newline at end of file From aedaab23bceda452aac7997cde44b20b6fb96424 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:06:35 -0700 Subject: [PATCH 23/54] doc image promotion --- .github/workflows/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000..e979b5fe --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,29 @@ +

+ systemaccounting +

+ +dev to prod docker image promotion + +# service workflow files +1. test code: `cargo test` (todo: test in Dockerfile builder step) +2. build image: `make -C services/rule build-image` +3. tag image with git sha A: `make -C services/rule tag-dev-image` +4. push git sha A image to dev ecr repo: `make -C services/rule push-dev-image` + +# dev-integration.yaml +5. merge to develop +6. test if currently deployed image in each dev function is last image in ecr repo: `tag-merge-commit.sh` +7. get git sha B from merge commit: `tag-merge-commit.sh` +8. tag each last undeployed image with git sha B: `tag-merge-commit.sh` +9. deploy each image with git sha B: `deploy-last-image.sh` +10. integration test in dev: `cargo test --manifest-path ./tests/Cargo.toml --features integration_tests -- --test-threads=1` +11. query dev ecr images for git sha B tags: `push-prod-image.sh` +12. tag dev git sha B images with prod: `push-prod-image.sh` +13. push git sha B images to prod: `push-prod-image.sh` + +#### prod-services-deploy.yaml +14. merge to master +15. test if currently deployed image in each prod function is last image in ecr repo: `tag-merge-commit.sh` +16. get git sha C from merge commit: `tag-merge-commit.sh` +17. tag each last undeployed image with git sha C: `tag-merge-commit.sh` +18. deploy each image with git sha C: `deploy-last-image.sh` \ No newline at end of file From c020c91fdbafc9f43d415ccfb1e90e2073c7a8c0 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:08:55 -0700 Subject: [PATCH 24/54] remove unused cloudformation template --- infrastructure/cloudformation/README.md | 62 -- infrastructure/cloudformation/websockets.yaml | 832 ------------------ 2 files changed, 894 deletions(-) delete mode 100644 infrastructure/cloudformation/README.md delete mode 100644 infrastructure/cloudformation/websockets.yaml diff --git a/infrastructure/cloudformation/README.md b/infrastructure/cloudformation/README.md deleted file mode 100644 index b2b9624f..00000000 --- a/infrastructure/cloudformation/README.md +++ /dev/null @@ -1,62 +0,0 @@ -

- systemaccounting -

- -## current stacks -*none, websockets [converted to terraform](https://github.com/systemaccounting/mxfactorial/commit/348ae2bc6a99ae366c21b7719d6154eaa0715457)* - -## example per environment cloudformation use - -aws resources not available in terraform provisioned through cloudformation -dependencies: aws cli, credentials - -1. [create](https://docs.aws.amazon.com/cli/latest/reference/s3api/create-bucket.html) standalone `$APP-artifacts-$ENV` bucket to store and deploy artifacts -1. `make initial-deploy ENV=dev` to deploy artifacts to bucket -1. author template, e.g. `websockets.yaml` -1. prepare inline variable assignments for reusable cloudformation commands: `STACK=notification-websockets ENV=dev` -1. [display](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/list-stacks.html) current stacks: `aws cloudformation list-stacks --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE` -1. [test template](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/validate-template.html): `aws cloudformation validate-template --template-body file://$(pwd)/websockets.yaml` -1. [create](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/create-stack.html) stack: - ```bash - STACK=notification-websockets ENV=dev; \ - aws cloudformation create-stack \ - --timeout-in-minutes 5 \ - --capabilities CAPABILITY_NAMED_IAM \ - --stack-name $STACK-$ENV \ - --template-body file://$(pwd)/websockets.yaml \ - --parameters ParameterKey=Environment,ParameterValue=$ENV - ``` -1. [get](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/get-template.html) current template: `STACK=notification-websockets ENV=dev; aws cloudformation get-template --stack-name $STACK-$ENV` -1. [create template change set](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/create-change-set.html): - ```bash - export CHANGE_SET_ID=$(\ - STACK=notification-websockets ENV=dev UTC_TIME=$(date -u '+%Y-%m-%d-%H-%M-%S'); \ - aws cloudformation create-change-set \ - --capabilities CAPABILITY_NAMED_IAM \ - --stack-name $STACK-$ENV \ - --change-set-name $STACK-$ENV-$UTC_TIME \ - --template-body file://$(pwd)/websockets.yaml \ - --parameters ParameterKey=Environment,ParameterValue=$ENV \ - --query Id --output text \ - ) - ``` - **OR** skip next 2 steps to [update](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/create-change-set.html) stack **WITHOUT** first creating change set: - ```bash - STACK=notification-websockets ENV=dev; \ - aws cloudformation update-stack \ - --capabilities CAPABILITY_NAMED_IAM \ - --stack-name $STACK-$ENV \ - --template-body file://$(pwd)/websockets.yaml \ - --parameters ParameterKey=Environment,ParameterValue=$ENV - ``` -1. [describe intended stack change](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/describe-change-set.html) by passing exported `CHANGE_SET_ID` variable from previous step (**OR** visit cloudformation in aws console): `aws cloudformation describe-change-set --change-set-name $CHANGE_SET_ID` -1. [execute](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/execute-change-set.html) change set by passing exported `CHANGE_SET_ID` variable from previous 'create template change set' step: `aws cloudformation execute-change-set --change-set-name $CHANGE_SET_ID` -1. [describe stack outputs](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/describe-stacks.html) to access values required by application: - ```bash - STACK=notification-websockets ENV=dev; \ - aws cloudformation describe-stacks --stack-name $STACK-$ENV \ - --query 'Stacks[?StackName==`'$STACK-$ENV'`].Outputs' - ``` -1. document new `template.yaml: $STACK-$ENV` managed by cloudformation in section below -1. [detect drift](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/detect-stack-drift.html) IF non-cloudformation change expected in stack: `STACK=notification-websockets ENV=dev; aws cloudformation detect-stack-drift --stack-name $STACK-$ENV` -1. [delete](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/delete-stack.html) stack: `STACK=notification-websockets ENV=dev; aws cloudformation delete-stack --stack-name $STACK-$ENV` \ No newline at end of file diff --git a/infrastructure/cloudformation/websockets.yaml b/infrastructure/cloudformation/websockets.yaml deleted file mode 100644 index 7b97f067..00000000 --- a/infrastructure/cloudformation/websockets.yaml +++ /dev/null @@ -1,832 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Description: websocket api -Parameters: - Environment: - Type: String - S3EventNotificationCustomResourceArtifact: - Type: String - Default: s3-event-src.zip - WebsocketConnectFunctionArtifact: - Type: String - Default: wss-notif-connect-src.zip - NotificationGetFunctionArtifact: - Type: String - Default: notification-get-src.zip - NotificationSendFunctionArtifact: - Type: String - Default: notification-send-src.zip - NotificationsClearFunctionArtifact: - Type: String - Default: notification-clear-src.zip - NotificationRetrievalLimitCount: - Type: Number - Default: 20 -Resources: - NotificationWebsocket: - Type: AWS::ApiGatewayV2::Api - Properties: - Name: !Sub "notification-websocket-${Environment}" - Description: !Sub "realtime notifications in ${Environment}" - DisableSchemaValidation: true - ProtocolType: WEBSOCKET - RouteSelectionExpression: $request.body.action - Tags: - environment: !Sub "${Environment}" - Deployment00002: - Type: AWS::ApiGatewayV2::Deployment - DependsOn: - - ConnectRoute - - DisconnectRoute - - GetNotificationsRoute - - ClearNotificationsRoute - - DefaultRoute - Properties: - Description: !Sub "notififcation websocket in ${Environment} v1" - ApiId: !Ref NotificationWebsocket - StageName: "" # https://forums.aws.amazon.com/thread.jspa?threadID=236830 - Stage: - Type: AWS::ApiGatewayV2::Stage - Properties: - StageName: !Sub "${Environment}" - Description: !Sub "${Environment} stage" - DeploymentId: !Ref Deployment00002 - ApiId: !Ref NotificationWebsocket - DefaultRouteSettings: - DataTraceEnabled: true - LoggingLevel: ERROR - WebsocketNotificationConnectLambda: - Type: AWS::Lambda::Function - Properties: - FunctionName: !Sub "websocket-notification-connect-faas-${Environment}" - Description: records connected websocket clients in dynamodb - Handler: index.handler - MemorySize: 256 - Runtime: nodejs10.x - Layers: - - 'arn:aws:lambda:us-east-1:170110910042:layer:pg-sequelize-nodejs-deps:1' - Code: - S3Bucket: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3Key: !Ref WebsocketConnectFunctionArtifact - Environment: - Variables: - PGDATABASE: !Sub "{{resolve:secretsmanager:${Environment}/PGDATABASE}}" - PGUSER: !Sub "{{resolve:secretsmanager:${Environment}/PGUSER}}" - PGPASSWORD: !Sub "{{resolve:secretsmanager:${Environment}/PGPASSWORD}}" - PGHOST: !Sub "{{resolve:secretsmanager:${Environment}/PGHOST}}" - PGPORT: !Sub "{{resolve:secretsmanager:${Environment}/PGPORT}}" - Role: !GetAtt WebsocketNotificationConnectLambdaRole.Arn - WebsocketNotificationConnectLambdaRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "websocket-notification-connect-role-${Environment}" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - "lambda.amazonaws.com" - Action: - - "sts:AssumeRole" - Policies: - - PolicyName: !Sub "allow-logs-websocket-notification-connect-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: arn:aws:logs:*:*:* - WebsocketNotificationConnectLambdaLogGroup: - Type: AWS::Logs::LogGroup - DependsOn: - - WebsocketNotificationConnectLambda - Properties: - RetentionInDays: 90 - LogGroupName: !Sub /aws/lambda/${WebsocketNotificationConnectLambda} - ApiGatewayWebsocketConnectLambdaInvokePermission: - Type: AWS::Lambda::Permission - DependsOn: - - NotificationWebsocket - - WebsocketNotificationConnectLambda - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref WebsocketNotificationConnectLambda - Principal: apigateway.amazonaws.com - ApiGatewayNotificationGetLambdaInvokePermission: - Type: AWS::Lambda::Permission - DependsOn: - - NotificationGetLambda - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref NotificationGetLambda - Principal: apigateway.amazonaws.com - ApiGatewayNotificationClearLambdaInvokePermission: - Type: AWS::Lambda::Permission - DependsOn: - - NotificationClearLambda - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref NotificationClearLambda - Principal: apigateway.amazonaws.com - NotificationGetLambda: - Type: AWS::Lambda::Function - Properties: - FunctionName: !Sub "notification-get-faas-${Environment}" - Description: !Sub "gets notifications after receiving websocket request in ${Environment}" - Handler: index.handler - MemorySize: 256 - Runtime: nodejs10.x - Role: !GetAtt NotificationGetLambdaExecutionRole.Arn - Layers: - - 'arn:aws:lambda:us-east-1:170110910042:layer:pg-sequelize-nodejs-deps:1' - Environment: - Variables: - NOTIFICATIONS_TABLE_NAME: !Select [1, !Split ["/", !GetAtt NotificationsTable.Arn]] - WSS_CONNECTION_URL: !Sub "https://${NotificationWebsocket}.execute-api.${AWS::Region}.amazonaws.com/${Environment}" - POOL_NAME: !Sub "mxfactorial-${Environment}" - PGDATABASE: !Sub "{{resolve:secretsmanager:${Environment}/PGDATABASE}}" - PGUSER: !Sub "{{resolve:secretsmanager:${Environment}/PGUSER}}" - PGPASSWORD: !Sub "{{resolve:secretsmanager:${Environment}/PGPASSWORD}}" - PGHOST: !Sub "{{resolve:secretsmanager:${Environment}/PGHOST}}" - PGPORT: !Sub "{{resolve:secretsmanager:${Environment}/PGPORT}}" - NOTIFICATION_RETRIEVAL_LIMIT_COUNT: !Ref NotificationRetrievalLimitCount - Code: - S3Bucket: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3Key: !Ref NotificationGetFunctionArtifact - NotificationGetLambdaExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "notification-get-lambda-role-${Environment}" - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: !Sub "allow-logs-notification-get-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: arn:aws:logs:*:*:* - - PolicyName: !Sub "allow-notification-table-access-notification-get-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'dynamodb:BatchGetItem' - - 'dynamodb:ConditionCheck' - - 'dynamodb:GetItem' - - 'dynamodb:GetRecords' - - 'dynamodb:Query' - - 'dynamodb:Scan' - Resource: !GetAtt NotificationsTable.Arn - - PolicyName: !Sub "allow-notifications-table-index-access-notification-get-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'dynamodb:Query' - - 'dynamodb:Scan' - Resource: !Join [ '/', [!GetAtt NotificationsTable.Arn, 'index', '*'] ] - - PolicyName: !Sub "api-manage-connections-get-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'execute-api:ManageConnections' - - 'execute-api:Invoke' - Resource: - - !Sub 'arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${NotificationWebsocket}/${Environment}/*' - - PolicyName: !Sub "allow-cognito-pool-list-get-notification-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'cognito-idp:ListUserPools' - Resource: '*' - NotificationGetLambdaLogGroup: - Type: AWS::Logs::LogGroup - DependsOn: - - NotificationGetLambda - Properties: - RetentionInDays: 90 - LogGroupName: !Sub /aws/lambda/${NotificationGetLambda} - NotificationClearLambda: - Type: AWS::Lambda::Function - Properties: - FunctionName: !Sub "notification-clear-faas-${Environment}" - Description: !Sub "clears notifications from websocket message in ${Environment}" - Handler: index.handler - MemorySize: 256 - Runtime: nodejs10.x - Role: !GetAtt NotificationClearLambdaExecutionRole.Arn - Layers: - - 'arn:aws:lambda:us-east-1:170110910042:layer:pg-sequelize-nodejs-deps:1' - Environment: - Variables: - NOTIFICATIONS_TABLE_NAME: !Select [1, !Split ["/", !GetAtt NotificationsTable.Arn]] - WSS_CONNECTION_URL: !Sub "https://${NotificationWebsocket}.execute-api.${AWS::Region}.amazonaws.com/${Environment}" - POOL_NAME: !Sub "mxfactorial-${Environment}" - PGDATABASE: !Sub "{{resolve:secretsmanager:${Environment}/PGDATABASE}}" - PGUSER: !Sub "{{resolve:secretsmanager:${Environment}/PGUSER}}" - PGPASSWORD: !Sub "{{resolve:secretsmanager:${Environment}/PGPASSWORD}}" - PGHOST: !Sub "{{resolve:secretsmanager:${Environment}/PGHOST}}" - PGPORT: !Sub "{{resolve:secretsmanager:${Environment}/PGPORT}}" - Code: - S3Bucket: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3Key: !Ref NotificationsClearFunctionArtifact - NotificationClearLambdaExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "notification-clear-lambda-role-${Environment}" - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: !Sub "allow-logs-notification-clear-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: arn:aws:logs:*:*:* - - PolicyName: !Sub "allow-notification-table-access-notification-clear-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'dynamodb:BatchGetItem' - - 'dynamodb:BatchWriteItem' - - 'dynamodb:ConditionCheck' - - 'dynamodb:DeleteItem' - - 'dynamodb:GetItem' - - 'dynamodb:GetRecords' - - 'dynamodb:PutItem' - - 'dynamodb:Query' - - 'dynamodb:Scan' - - 'dynamodb:UpdateItem' - Resource: !GetAtt NotificationsTable.Arn - - PolicyName: !Sub "allow-api-invoke-clear-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'execute-api:ManageConnections' - - 'execute-api:Invoke' - Resource: - - !Sub 'arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${NotificationWebsocket}/${Environment}/*' - - PolicyName: !Sub "allow-cognito-pool-list-clear-notification-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'cognito-idp:ListUserPools' - Resource: '*' - NotificationClearLambdaLogGroup: - Type: AWS::Logs::LogGroup - DependsOn: - - NotificationClearLambda - Properties: - RetentionInDays: 90 - LogGroupName: !Sub /aws/lambda/${NotificationClearLambda} - NotificationTopicSubscription: # sns topic provisioned in infrastructure/terraform/aws/modules/environment/sns.tf - Type: AWS::SNS::Subscription - Properties: - Endpoint: !GetAtt NotificationSendLambda.Arn - Protocol: lambda - TopicArn: !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:notifications-${Environment}" - NotificationSendLambda: - Type: AWS::Lambda::Function - Properties: - FunctionName: !Sub "notification-send-faas-${Environment}" - Description: !Sub "stores and sends notifications in ${Environment}" - Handler: index.handler - MemorySize: 256 - Runtime: nodejs10.x - Layers: - - 'arn:aws:lambda:us-east-1:170110910042:layer:pg-sequelize-nodejs-deps:1' - Code: - S3Bucket: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3Key: !Ref NotificationSendFunctionArtifact - Environment: - Variables: - NOTIFICATIONS_TABLE_NAME: !Select [1, !Split ["/", !GetAtt NotificationsTable.Arn]] - ENVIRONMENT: !Sub ${Environment - WSS_CONNECTION_URL: !Sub "https://${NotificationWebsocket}.execute-api.${AWS::Region}.amazonaws.com/${Environment}" - PGDATABASE: !Sub "{{resolve:secretsmanager:${Environment}/PGDATABASE}}" - PGUSER: !Sub "{{resolve:secretsmanager:${Environment}/PGUSER}}" - PGPASSWORD: !Sub "{{resolve:secretsmanager:${Environment}/PGPASSWORD}}" - PGHOST: !Sub "{{resolve:secretsmanager:${Environment}/PGHOST}}" - PGPORT: !Sub "{{resolve:secretsmanager:${Environment}/PGPORT}}" - Role: !GetAtt NotificationSendLambdaRole.Arn - NotificationSendLambdaRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "notification-send-lambda-role-${Environment}" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - "lambda.amazonaws.com" - Action: - - "sts:AssumeRole" - Policies: - - PolicyName: !Sub "allow-logs-notification-send-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: arn:aws:logs:*:*:* - - PolicyName: !Sub "allow-dynamodb-access-notification-send-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'dynamodb:BatchGetItem' - - 'dynamodb:BatchWriteItem' - - 'dynamodb:ConditionCheck' - - 'dynamodb:DeleteItem' - - 'dynamodb:GetItem' - - 'dynamodb:GetRecords' - - 'dynamodb:PutItem' - - 'dynamodb:Query' - - 'dynamodb:Scan' - - 'dynamodb:UpdateItem' - Resource: !GetAtt NotificationsTable.Arn - - PolicyName: !Sub "allow-notfication-ddb-index-access-notification-send-lambda-${Environment}" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - 'dynamodb:Query' - - 'dynamodb:Scan' - Resource: !Join [ '/', [!GetAtt NotificationsTable.Arn, 'index', '*'] ] - - PolicyName: !Sub "allow-notification-table-access-notification-send-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'dynamodb:BatchGetItem' - - 'dynamodb:ConditionCheck' - - 'dynamodb:GetItem' - - 'dynamodb:GetRecords' - - 'dynamodb:Query' - - 'dynamodb:Scan' - Resource: !GetAtt NotificationsTable.Arn - - PolicyName: !Sub "api-manage-connections-send-lambda-${Environment}" - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'execute-api:ManageConnections' - - 'execute-api:Invoke' - Resource: - - !Sub 'arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${NotificationWebsocket}/${Environment}/*' - NotificationSendLambdaLogGroup: - Type: AWS::Logs::LogGroup - DependsOn: - - NotificationSendLambda - Properties: - RetentionInDays: 90 - LogGroupName: !Sub /aws/lambda/${NotificationSendLambda} - NotificationSendLambdaInvokePermission: - Type: AWS::Lambda::Permission - DependsOn: - - NotificationSendLambda - - NotificationTopicSubscription - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref NotificationSendLambda - Principal: sns.amazonaws.com - SourceArn: !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:notifications-${Environment}" - NotificationsTable: - Type: AWS::DynamoDB::Table - Properties: - TableName: !Sub "notifications-${Environment}" - AttributeDefinitions: - - AttributeName: "uuid" - AttributeType: "S" - - AttributeName: "timestamp" - AttributeType: "N" - - AttributeName: "account" - AttributeType: "S" - KeySchema: - - AttributeName: "uuid" - KeyType: "HASH" - - AttributeName: "timestamp" - KeyType: "RANGE" - ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 - GlobalSecondaryIndexes: - - IndexName: "account-index" - KeySchema: - - AttributeName: "account" - KeyType: HASH - - AttributeName: "timestamp" - KeyType: "RANGE" - Projection: - ProjectionType: ALL - ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 - NotificationsTableWriteCapacityScalableTarget: - Type: AWS::ApplicationAutoScaling::ScalableTarget - Properties: - MaxCapacity: 15 - MinCapacity: 5 - ResourceId: !Join - - / - - - table - - !Ref NotificationsTable - RoleARN: !GetAtt NotificationsTable.Arn - ScalableDimension: dynamodb:table:WriteCapacityUnits - ServiceNamespace: dynamodb - NotificationsTableWriteScalingPolicy: - Type: AWS::ApplicationAutoScaling::ScalingPolicy - Properties: - PolicyName: WriteAutoScalingPolicy - PolicyType: TargetTrackingScaling - ScalingTargetId: !Ref NotificationsTableWriteCapacityScalableTarget - TargetTrackingScalingPolicyConfiguration: - TargetValue: 50.0 - ScaleInCooldown: 60 - ScaleOutCooldown: 60 - PredefinedMetricSpecification: - PredefinedMetricType: DynamoDBWriteCapacityUtilization - DynamoDBScalingRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - - Effect: "Allow" - Principal: - Service: - - application-autoscaling.amazonaws.com - Action: - - "sts:AssumeRole" - Path: "/" - Policies: - - - PolicyName: "root" - PolicyDocument: - Version: "2012-10-17" - Statement: - - - Effect: "Allow" - Action: - - "dynamodb:DescribeTable" - - "dynamodb:UpdateTable" - - "cloudwatch:PutMetricAlarm" - - "cloudwatch:DescribeAlarms" - - "cloudwatch:GetMetricStatistics" - - "cloudwatch:SetAlarmState" - - "cloudwatch:DeleteAlarms" - Resource: "*" - ConnectRoute: - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref NotificationWebsocket - RouteKey: $connect - AuthorizationType: NONE - OperationName: ConnectRoute - Target: !Join - - '/' - - - 'integrations' - - !Ref ConnectInteg - ConnectInteg: - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref NotificationWebsocket - Description: "integration called from websocket connect event in ${Environment}" - IntegrationType: AWS_PROXY - IntegrationUri: - Fn::Sub: - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${WebsocketNotificationConnectLambda.Arn}/invocations - # todo: customize 'error: Unexpected server response: 502' api gateway response when token auth fails - DisconnectRoute: - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref NotificationWebsocket - RouteKey: $disconnect - AuthorizationType: NONE - OperationName: DisconnectRoute - Target: !Join - - '/' - - - 'integrations' - - !Ref DisconnectInteg - DisconnectInteg: - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref NotificationWebsocket - Description: Disconnect Integration - IntegrationType: AWS_PROXY - IntegrationUri: - Fn::Sub: - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${WebsocketNotificationConnectLambda.Arn}/invocations - GetNotificationsRoute: - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref NotificationWebsocket - RouteKey: getnotifications - AuthorizationType: NONE - OperationName: Get Notifications - Target: !Join - - '/' - - - 'integrations' - - !Ref GetNotificationsInteg - GetNotificationsInteg: - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref NotificationWebsocket - Description: GetNotifications Integration - IntegrationType: AWS_PROXY - IntegrationUri: - Fn::Sub: - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${NotificationGetLambda.Arn}/invocations - ClearNotificationsRoute: - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref NotificationWebsocket - RouteKey: clearnotifications - AuthorizationType: NONE - OperationName: ClearNotifications - Target: !Join - - '/' - - - 'integrations' - - !Ref ClearNotificationsInteg - ClearNotificationsInteg: - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref NotificationWebsocket - Description: Clear Notifications Integration - IntegrationType: AWS_PROXY - IntegrationUri: - Fn::Sub: - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${NotificationClearLambda.Arn}/invocations - DefaultRoute: - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref NotificationWebsocket - RouteKey: '$default' - RouteResponseSelectionExpression: '$default' - AuthorizationType: NONE - OperationName: default - Target: !Join - - '/' - - - 'integrations' - - !Ref DefaultIntegration - DefaultIntegration: - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref NotificationWebsocket - Description: returns error describing available actions - IntegrationType: MOCK - PassthroughBehavior: WHEN_NO_MATCH - RequestTemplates: - "200": "{\"statusCode\": 200}" - TemplateSelectionExpression: "200" - DefaultIntegrationResponse: - Type: AWS::ApiGatewayV2::IntegrationResponse - Properties: - ApiId: !Ref NotificationWebsocket - IntegrationId: !Ref DefaultIntegration - IntegrationResponseKey: "$default" - ResponseTemplates: - "404": "\"only getnotifications and clearnotifications actions available\"" - TemplateSelectionExpression: "404" - DefaultRouteResponse: - Type: AWS::ApiGatewayV2::RouteResponse - Properties: - ApiId: !Ref NotificationWebsocket - RouteId: !Ref DefaultRoute - RouteResponseKey: '$default' - S3EventNotificationLifecycleManager: - Type: AWS::Lambda::Function - Properties: - FunctionName: !Sub "s3-event-faas-${Environment}" - Description: !Sub "manages s3 events for cloudformation in ${Environment}" - Handler: main.lambda_handler - Role: !GetAtt S3EventNotificationLifecycleManagerRole.Arn - Runtime: python3.8 - Timeout: 50 - Layers: - - !Sub "arn:aws:lambda:${AWS::Region}:770693421928:layer:Klayers-python38-requests:5" - Code: - S3Bucket: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3Key: !Ref S3EventNotificationCustomResourceArtifact - Environment: - Variables: - ENVIRONMENT: !Sub ${Environment} - ARTIFACTS_BUCKET: !Sub "mxfactorial-websocket-artifacts-${Environment}" - S3EventNotificationLifecycleManagerRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - 'sts:AssumeRole' - Path: / - Policies: - - PolicyName: !Sub AllowS3EventCustomLambdaInvoke${Environment} - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - s3:GetBucketNotification - - s3:PutBucketNotification - Resource: '*' - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: 'arn:aws:logs:*:*:*' - S3EventNotificationLifecycleManagerLogGroup: - Type: AWS::Logs::LogGroup - DependsOn: - - S3EventNotificationLifecycleManager - Properties: - RetentionInDays: 90 - LogGroupName: !Sub /aws/lambda/${S3EventNotificationLifecycleManager} - CustomResourceLambdaDeployS3EventConfiguration: - # depend on: 1) lambda to deploy, 2) preceding s3 event configurations, - # 3) S3EventNotificationLifecycleManagerLogGroup (avoids lingering log - # group after destroy). depending on preceding s3 event configurations avoids - # 'PutBucketNotificationConfiguration operation: A conflicting conditional - # operation is currently in progress against this resource. Please try again.' - DependsOn: - # lambda to deploy - - S3EventNotificationLifecycleManager - # preceding s3 event configurations (S3EventNotificationLifecycleManager first) - # - none - # S3EventNotificationLifecycleManagerLogGroup - - S3EventNotificationLifecycleManagerLogGroup - Type: Custom::CustomResourceLambdaDeployS3EventConfiguration - Properties: - ServiceToken: !GetAtt S3EventNotificationLifecycleManager.Arn - AccountId: !Ref AWS::AccountId - ObjectName: !Ref S3EventNotificationCustomResourceArtifact - ResourceName: !Ref S3EventNotificationLifecycleManager - WebsocketNotificationConnectLambdaConfiguration: - DependsOn: - # lambda to deploy - - WebsocketNotificationConnectLambda - # preceding s3 event configurations - - CustomResourceLambdaDeployS3EventConfiguration - # S3EventNotificationLifecycleManagerLogGroup - - S3EventNotificationLifecycleManagerLogGroup - Type: Custom::WebsocketNotificationConnectLambdaConfiguration - Properties: - ServiceToken: !GetAtt S3EventNotificationLifecycleManager.Arn - AccountId: !Ref AWS::AccountId - ObjectName: !Ref WebsocketConnectFunctionArtifact - ResourceName: !Ref WebsocketNotificationConnectLambda - CustomResourceNotificationGetLambdaConfiguration: - DependsOn: - # lambda to deploy - - WebsocketNotificationConnectLambda - # preceding s3 event configurations - - CustomResourceLambdaDeployS3EventConfiguration - - WebsocketNotificationConnectLambdaConfiguration - # S3EventNotificationLifecycleManagerLogGroup - - S3EventNotificationLifecycleManagerLogGroup - Type: Custom::CustomResourceNotificationGetLambdaConfiguration - Properties: - ServiceToken: !GetAtt S3EventNotificationLifecycleManager.Arn - AccountId: !Ref AWS::AccountId - ObjectName: !Ref NotificationGetFunctionArtifact - ResourceName: !Ref NotificationGetLambda - CustomResourceNotificationSendLambdaConfiguration: - DependsOn: - # lambda to deploy - - WebsocketNotificationConnectLambda - # preceding s3 event configurations - - CustomResourceLambdaDeployS3EventConfiguration - - WebsocketNotificationConnectLambdaConfiguration - - CustomResourceNotificationGetLambdaConfiguration - # S3EventNotificationLifecycleManagerLogGroup - - S3EventNotificationLifecycleManagerLogGroup - Type: Custom::CustomResourceNotificationSendLambdaConfiguration - Properties: - ServiceToken: !GetAtt S3EventNotificationLifecycleManager.Arn - AccountId: !Ref AWS::AccountId - ObjectName: !Ref NotificationSendFunctionArtifact - ResourceName: !Ref NotificationSendLambda - CustomResourceNotificationClearLambdaConfiguration: - DependsOn: - # lambda to deploy - - WebsocketNotificationConnectLambda - # preceding s3 event configurations - - CustomResourceLambdaDeployS3EventConfiguration - - WebsocketNotificationConnectLambdaConfiguration - - CustomResourceNotificationGetLambdaConfiguration - - CustomResourceNotificationSendLambdaConfiguration - # S3EventNotificationLifecycleManagerLogGroup - - S3EventNotificationLifecycleManagerLogGroup - Type: Custom::CustomResourceNotificationClearLambdaConfiguration - Properties: - ServiceToken: !GetAtt S3EventNotificationLifecycleManager.Arn - AccountId: !Ref AWS::AccountId - ObjectName: !Ref NotificationsClearFunctionArtifact - ResourceName: !Ref NotificationClearLambda - NotificationWebsocketEndpoint: - Type: AWS::SecretsManager::Secret - Properties: - Description: !Sub "notification websocket endoint in ${Environment}" - Name: !Sub "${Environment}/WSS_CLIENT_URL" - SecretString: - Fn::Join: - - '' - - - wss:// - - !Ref NotificationWebsocket - - .execute-api. - - !Ref AWS::Region - - .amazonaws.com/ - - !Ref Stage - NotificationWebsocketConnectionUrl: - Type: AWS::SecretsManager::Secret - Properties: - Description: !Sub "notification websocket connection url in ${Environment}" - Name: !Sub "${Environment}/WSS_CONNECTION_URL" - SecretString: - Fn::Join: - - '' - - - https:// - - !Ref NotificationWebsocket - - .execute-api. - - !Ref AWS::Region - - .amazonaws.com/ - - !Ref Stage - NotificationsTableName: - Type: AWS::SecretsManager::Secret - Properties: - Description: !Sub "notifications table name in ${Environment}" - Name: !Sub "${Environment}/NOTIFICATIONS_TABLE_NAME" - SecretString: !Ref NotificationsTable - NotificationRetrievalLimitCountSecret: - Type: AWS::SecretsManager::Secret - Properties: - Description: !Sub "notifications table name in ${Environment}" - Name: !Sub "${Environment}/NOTIFICATION_RETRIEVAL_LIMIT_COUNT" - SecretString: !Ref NotificationRetrievalLimitCount -Outputs: - NotificationWebsocketEndpoint: - Description: wss notification endpoint - Value: - Fn::Join: - - '' - - - wss:// - - !Ref NotificationWebsocket - - .execute-api. - - !Ref AWS::Region - - .amazonaws.com/ - - !Ref Stage - Export: - Name: !Sub "notification-websocket-endpoint-${Environment}" \ No newline at end of file From 6252ae9fbef9718fe472b28c71a29bc4eeaa08c9 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:09:33 -0700 Subject: [PATCH 25/54] add ecr terraform module --- .../terraform/aws/modules/ecr/v001/ecr.tf | 28 +++++++++++++++++++ .../aws/modules/ecr/v001/variables.tf | 8 ++++++ 2 files changed, 36 insertions(+) create mode 100644 infrastructure/terraform/aws/modules/ecr/v001/ecr.tf create mode 100644 infrastructure/terraform/aws/modules/ecr/v001/variables.tf diff --git a/infrastructure/terraform/aws/modules/ecr/v001/ecr.tf b/infrastructure/terraform/aws/modules/ecr/v001/ecr.tf new file mode 100644 index 00000000..ee95b504 --- /dev/null +++ b/infrastructure/terraform/aws/modules/ecr/v001/ecr.tf @@ -0,0 +1,28 @@ +resource "aws_ecr_repository" "default" { + name = "${var.env_id}/${var.env}/${var.service_name}" + image_tag_mutability = "MUTABLE" + force_delete = var.force_destroy_storage + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "default" { + repository = aws_ecr_repository.default.name + policy = jsonencode({ + rules = [ + { + rulePriority = 1, + description = "keep last ${var.max_image_storage_count} images", + selection = { + tagStatus = "any", + countType = "imageCountMoreThan", + countNumber = var.max_image_storage_count + }, + action = { + type = "expire" + } + } + ] + }) +} diff --git a/infrastructure/terraform/aws/modules/ecr/v001/variables.tf b/infrastructure/terraform/aws/modules/ecr/v001/variables.tf new file mode 100644 index 00000000..bbf452aa --- /dev/null +++ b/infrastructure/terraform/aws/modules/ecr/v001/variables.tf @@ -0,0 +1,8 @@ +variable "env" {} +variable "env_id" {} +variable "force_destroy_storage" { + type = bool + default = false +} +variable "service_name" {} +variable "max_image_storage_count" {} \ No newline at end of file From 485df9048fe05d5eea939b57b0a07ab16e74f7c1 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:11:37 -0700 Subject: [PATCH 26/54] remove unused go-lambda terraform module --- .../aws/modules/go-lambda/v001/lambda.tf | 117 ------------------ .../aws/modules/go-lambda/v001/outputs.tf | 11 -- .../aws/modules/go-lambda/v001/variables.tf | 24 ---- 3 files changed, 152 deletions(-) delete mode 100644 infrastructure/terraform/aws/modules/go-lambda/v001/lambda.tf delete mode 100644 infrastructure/terraform/aws/modules/go-lambda/v001/outputs.tf delete mode 100644 infrastructure/terraform/aws/modules/go-lambda/v001/variables.tf diff --git a/infrastructure/terraform/aws/modules/go-lambda/v001/lambda.tf b/infrastructure/terraform/aws/modules/go-lambda/v001/lambda.tf deleted file mode 100644 index 1fa57056..00000000 --- a/infrastructure/terraform/aws/modules/go-lambda/v001/lambda.tf +++ /dev/null @@ -1,117 +0,0 @@ -locals { - ID_ENV = "${var.env_id}-${var.env}" - TITLED_ID_ENV = replace(title(local.ID_ENV), "-", "") - SPACED_ID_ENV = replace(local.ID_ENV, "-", " ") - SERVICE_NAME_TITLE = replace(title(var.service_name), "-", "") - SERVICE_NAME_UPPER = replace(upper(var.service_name), "-", "_") - SERVICE_NAME_LOWER = replace(var.service_name, "-", "_") - LOG_GROUP_NAME = "/aws/lambda/${aws_lambda_function.default.function_name}" - PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) - BINARY_NAME = local.PROJECT_CONF.services.env_var.set.BINARY_NAME.default - LAMBDA_RUNTIME = local.PROJECT_CONF.infrastructure.terraform.aws.modules.env_var.set.LAMBDA_RUNTIME.default -} - -data "aws_s3_object" "default" { - bucket = var.artifacts_bucket_name - key = "${var.service_name}-src.zip" -} - -data "aws_region" "current" {} - -data "aws_caller_identity" "current" {} - -resource "aws_lambda_function" "default" { - function_name = "${var.service_name}-${local.ID_ENV}" - description = "${var.service_name} lambda service in ${local.SPACED_ID_ENV}" - s3_bucket = data.aws_s3_object.default.bucket - s3_key = data.aws_s3_object.default.key - s3_object_version = data.aws_s3_object.default.version_id - handler = local.BINARY_NAME - runtime = local.LAMBDA_RUNTIME - timeout = 30 - role = aws_iam_role.default.arn - environment { - variables = merge( - {}, - var.env_vars, - ) - } -} - -resource "aws_cloudwatch_log_group" "default" { - name = local.LOG_GROUP_NAME - retention_in_days = 30 -} - -resource "aws_iam_role" "default" { - name = "${var.service_name}-lambda-${local.ID_ENV}" - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "${local.SERVICE_NAME_TITLE}LambdaTrustPolicy${local.TITLED_ID_ENV}" - Action = "sts:AssumeRole" - Effect = "Allow" - Principal = { - Service = "lambda.amazonaws.com" - } - }, - ] - }) -} - -resource "aws_iam_policy" "default" { - name = "${var.service_name}-lambda-logging-${local.ID_ENV}" - description = "${aws_lambda_function.default.function_name} logging permission in ${local.SPACED_ID_ENV}" - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "${local.SERVICE_NAME_TITLE}CreateLogGroupPolicy${local.TITLED_ID_ENV}" - Effect = "Allow", - Action = [ - "logs:CreateLogGroup" - ], - Resource = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" - }, - { - Sid = "${local.SERVICE_NAME_TITLE}LogEventPolicy${local.TITLED_ID_ENV}" - Effect = "Allow", - Action = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - Resource = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.LOG_GROUP_NAME}:*" - ] - } - ] - }) -} - -resource "aws_iam_role_policy_attachment" "default" { - role = aws_iam_role.default.name - policy_arn = aws_iam_policy.default.arn -} - -resource "aws_lambda_permission" "default" { - count = length(var.invoke_arn_principals) - statement_id = "Allow${local.SERVICE_NAME_TITLE}${local.TITLED_ID_ENV}Execution${count.index}" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.default.function_name - principal = var.invoke_arn_principals[count.index] -} - -resource "aws_iam_role_policy_attachment" "extra" { - count = length(var.attached_policy_arns) - role = aws_iam_role.default.name - policy_arn = var.attached_policy_arns[count.index] -} - -resource "aws_ssm_parameter" "default" { - count = var.create_secret ? 1 : 0 - name = "/${var.ssm_prefix}/service/lambda/${local.SERVICE_NAME_LOWER}/arn" - description = "${aws_lambda_function.default.function_name} arn in ${local.SPACED_ID_ENV}" - type = "SecureString" - value = aws_lambda_function.default.arn -} diff --git a/infrastructure/terraform/aws/modules/go-lambda/v001/outputs.tf b/infrastructure/terraform/aws/modules/go-lambda/v001/outputs.tf deleted file mode 100644 index c77f4815..00000000 --- a/infrastructure/terraform/aws/modules/go-lambda/v001/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "lambda_arn" { - value = aws_lambda_function.default.arn -} - -output "lambda_invoke_arn" { - value = aws_lambda_function.default.invoke_arn -} - -output "lambda_role_name" { - value = aws_iam_role.default.name -} diff --git a/infrastructure/terraform/aws/modules/go-lambda/v001/variables.tf b/infrastructure/terraform/aws/modules/go-lambda/v001/variables.tf deleted file mode 100644 index 6aa32c7e..00000000 --- a/infrastructure/terraform/aws/modules/go-lambda/v001/variables.tf +++ /dev/null @@ -1,24 +0,0 @@ -variable "service_name" {} -variable "env" {} -variable "env_vars" { - type = map(any) - default = null -} -variable "invoke_arn_principals" { - type = list(string) - default = [] - description = "example: cognito-idp.amazonaws.com" -} -variable "attached_policy_arns" { - type = list(string) - default = [] - description = "example: aws_iam_policy.apiv2_ddb.arn" -} -variable "create_secret" { - type = bool - default = false - description = "adds lambda arn in secrets manager for local testing" -} -variable "artifacts_bucket_name" {} -variable "ssm_prefix" {} -variable "env_id" {} From 25800efca8aa42c3914edf7f3f73973174500e75 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:13:25 -0700 Subject: [PATCH 27/54] terraform for_each ecr image repositories --- .../aws/modules/project-storage/v001/ecr.tf | 105 ++---------------- .../modules/project-storage/v001/locals.tf | 36 ++++-- .../modules/project-storage/v001/variables.tf | 1 + 3 files changed, 36 insertions(+), 106 deletions(-) diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf index 02557c6e..78dc3259 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf @@ -1,99 +1,8 @@ -resource "aws_ecr_repository" "go_migrate" { - name = "${local.GO_MIGRATE}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "auto_confirm" { - name = "${local.AUTO_CONFIRM}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - - -resource "aws_ecr_repository" "balance_by_account" { - name = "${local.BALANCE_BY_ACCOUNT}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "graphql" { - name = "${local.GRAPHQL}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "request_approve" { - name = "${local.REQUEST_APPROVE}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "request_by_id" { - name = "${local.REQUEST_BY_ID}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "requests_by_account" { - name = "${local.REQUESTS_BY_ACCOUNT}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "request_create" { - name = "${local.REQUEST_CREATE}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "rule" { - name = "${local.RULE}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "transaction_by_id" { - name = "${local.TRANSACTION_BY_ID}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "transactions_by_account" { - name = "${local.TRANSACTIONS_BY_ACCOUNT}-${local.ID_ENV}" - image_tag_mutability = "MUTABLE" - force_delete = var.force_destroy_storage - image_scanning_configuration { - scan_on_push = true - } +module "ecr_repos" { + for_each = local.ECR_REPOS + source = "../../ecr/v001" + max_image_storage_count = var.max_image_storage_count + env = var.env + env_id = var.env_id + service_name = each.value } diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf index a6190a73..effb6349 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/locals.tf @@ -1,10 +1,14 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" - PROJECT_CONF = "project.yaml" - CONF_FILE = yamldecode(file("../../../../../${local.PROJECT_CONF}")) - STORAGE_ENV_VAR = local.CONF_FILE.infrastructure.terraform.aws.modules.project-storage.env_var.set - DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default - DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF = "project.yaml" + CONF_FILE = yamldecode(file("../../../../../${local.PROJECT_CONF}")) + STORAGE_ENV_VAR = local.CONF_FILE.infrastructure.terraform.aws.modules.project-storage.env_var.set + DDB_TABLE_NAME_PREFIX = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default + DDB_TABLE_HASH_KEY = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + ID_ENV_PREFIX = "${var.env_id}/${var.env}" + + // add a terraform_data precondition to fail + // if a service is not found in project.yaml GO_MIGRATE = "go-migrate" AUTO_CONFIRM = "auto-confirm" BALANCE_BY_ACCOUNT = "balance-by-account" @@ -17,16 +21,32 @@ locals { TRANSACTION_BY_ID = "transaction-by-id" TRANSACTIONS_BY_ACCOUNT = "transactions-by-account" + // used in ecr.tf + ECR_REPOS = toset([ + local.GO_MIGRATE, + local.AUTO_CONFIRM, + local.BALANCE_BY_ACCOUNT, + local.GRAPHQL, + local.REQUEST_APPROVE, + local.REQUEST_BY_ID, + local.REQUESTS_BY_ACCOUNT, + local.REQUEST_CREATE, + local.RULE, + local.TRANSACTION_BY_ID, + local.TRANSACTIONS_BY_ACCOUNT, + ]) + } -// fails if services not found in project.yaml +// 1. fails if services not found in project.yaml +// 2. single resource with multiple preconditions used to avoid increasing state file size resource "terraform_data" "locals_test" { lifecycle { + // create a precondition to fail if the services are not found in project.yaml precondition { condition = lookup(local.CONF_FILE.migrations, local.GO_MIGRATE, null) != null error_message = "${local.GO_MIGRATE} not found in ${local.PROJECT_CONF}" } - precondition { condition = lookup(local.CONF_FILE.services, local.AUTO_CONFIRM, null) != null error_message = "${local.AUTO_CONFIRM} not found in ${local.PROJECT_CONF}" diff --git a/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf b/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf index fae6caf4..f2d0ec1d 100644 --- a/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf +++ b/infrastructure/terraform/aws/modules/project-storage/v001/variables.tf @@ -9,3 +9,4 @@ variable "force_destroy_storage" { type = bool default = false } +variable "max_image_storage_count" {} \ No newline at end of file From 09e5b75fd9ec2215b7a0f674e805ecd00b962960 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:16:11 -0700 Subject: [PATCH 28/54] remove s3 artifact source from lambda terraform module --- .../modules/environment/v001/go-migrate.tf | 1 - .../modules/provided-lambda/v001/README.md | 12 ++-- .../modules/provided-lambda/v001/lambda.tf | 59 +++++++------------ .../modules/provided-lambda/v001/variables.tf | 1 - 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/infrastructure/terraform/aws/modules/environment/v001/go-migrate.tf b/infrastructure/terraform/aws/modules/environment/v001/go-migrate.tf index 3f2d7ebc..98d7ad21 100644 --- a/infrastructure/terraform/aws/modules/environment/v001/go-migrate.tf +++ b/infrastructure/terraform/aws/modules/environment/v001/go-migrate.tf @@ -8,7 +8,6 @@ module "go_migrate" { GO_MIGRATE_PASSPHRASE = random_password.go_migrate.result SQL_TYPE = local.SQL_TYPE }) - artifacts_bucket_name = null # defaults to ecr image create_secret = true attached_policy_arns = [] lambda_url_authorization_type = "NONE" diff --git a/infrastructure/terraform/aws/modules/provided-lambda/v001/README.md b/infrastructure/terraform/aws/modules/provided-lambda/v001/README.md index 9a522ea9..7678e4b3 100644 --- a/infrastructure/terraform/aws/modules/provided-lambda/v001/README.md +++ b/infrastructure/terraform/aws/modules/provided-lambda/v001/README.md @@ -4,10 +4,14 @@ #### provided runtime lambda terraform module -use: -1. assign `var.artifacts_bucket_name` a name to deploy from an s3 bucket -1. assign `var.artifacts_bucket_name` to null and add an `aws_ecr_repository` resource to `infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf` to deploy from a docker image repository -1. assign `var.aws_lwa_port` a unique project application port* to enable the [lambda web adapter](https://github.com/awslabs/aws-lambda-web-adapter) +general use: +1. requires adding an `aws_ecr_repository` resource to `infrastructure/terraform/aws/modules/project-storage/v001/ecr.tf` +1. build, tag and push image, e.g. `make/ecr-lambda.mk` +1. `terraform apply` + +lambda web adapter use: +1. add `COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.2 /lambda-adapter /opt/extensions/lambda-adapter` to app Dockerfile +1. assign `var.aws_lwa_port` a unique project application port* to configure the [lambda web adapter](https://github.com/awslabs/aws-lambda-web-adapter) examples: 1. `infrastructure/terraform/aws/modules/environment/v001/lambda-services.tf` diff --git a/infrastructure/terraform/aws/modules/provided-lambda/v001/lambda.tf b/infrastructure/terraform/aws/modules/provided-lambda/v001/lambda.tf index e28f0f84..b06742db 100644 --- a/infrastructure/terraform/aws/modules/provided-lambda/v001/lambda.tf +++ b/infrastructure/terraform/aws/modules/provided-lambda/v001/lambda.tf @@ -1,28 +1,23 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" - TITLED_ID_ENV = replace(title(local.ID_ENV), "-", "") - SPACED_ID_ENV = replace(local.ID_ENV, "-", " ") - SERVICE_NAME_TITLE = replace(title(var.service_name), "-", "") - SERVICE_NAME_UPPER = replace(upper(var.service_name), "-", "_") - SERVICE_NAME_LOWER = replace(var.service_name, "-", "_") - LOG_GROUP_NAME = "/aws/lambda/${aws_lambda_function.default.function_name}" - PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) - ENVIRONMENT_CONF = local.PROJECT_CONF.infrastructure.terraform.aws.modules.environment.env_var.set - READINESS_CHECK_PATH = local.ENVIRONMENT_CONF.READINESS_CHECK_PATH.default - WEB_ADAPTER_LAYER_VERSION = local.ENVIRONMENT_CONF.WEB_ADAPTER_LAYER_VERSION.default - BINARY_NAME = local.PROJECT_CONF.services.env_var.set.BINARY_NAME.default - LAMBDA_RUNTIME = local.PROJECT_CONF.infrastructure.terraform.aws.modules.env_var.set.LAMBDA_RUNTIME.default + ID_ENV = "${var.env_id}-${var.env}" + TITLED_ID_ENV = replace(title(local.ID_ENV), "-", "") + SPACED_ID_ENV = replace(local.ID_ENV, "-", " ") + SERVICE_NAME_TITLE = replace(title(var.service_name), "-", "") + SERVICE_NAME_UPPER = replace(upper(var.service_name), "-", "_") + SERVICE_NAME_LOWER = replace(var.service_name, "-", "_") + LOG_GROUP_NAME = "/aws/lambda/${aws_lambda_function.default.function_name}" + PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) + ENVIRONMENT_CONF = local.PROJECT_CONF.infrastructure.terraform.aws.modules.environment.env_var.set + READINESS_CHECK_PATH = local.ENVIRONMENT_CONF.READINESS_CHECK_PATH.default } -data "aws_s3_object" "default" { - count = var.artifacts_bucket_name == null ? 0 : 1 - bucket = var.artifacts_bucket_name - key = "${var.service_name}-src.zip" +data "aws_ecr_repository" "default" { + name = "${var.service_name}-${local.ID_ENV}" } -data "aws_ecr_repository" "default" { - count = var.artifacts_bucket_name == null ? 1 : 0 - name = "${var.service_name}-${local.ID_ENV}" +data "aws_ecr_image" "default" { + repository_name = data.aws_ecr_repository.default.name + most_recent = true } data "aws_region" "current" {} @@ -30,34 +25,24 @@ data "aws_region" "current" {} data "aws_caller_identity" "current" {} resource "aws_lambda_function" "default" { - function_name = "${var.service_name}-${local.ID_ENV}" - description = "${var.service_name} lambda service in ${local.SPACED_ID_ENV}" - s3_bucket = var.artifacts_bucket_name == null ? null : data.aws_s3_object.default[0].bucket - s3_key = var.artifacts_bucket_name == null ? null : data.aws_s3_object.default[0].key - s3_object_version = var.artifacts_bucket_name == null ? null : data.aws_s3_object.default[0].version_id - image_uri = var.artifacts_bucket_name == null ? "${data.aws_ecr_repository.default[0].repository_url}:latest" : null - package_type = var.artifacts_bucket_name == null ? "Image" : null - handler = var.artifacts_bucket_name == null ? null : local.BINARY_NAME - runtime = var.artifacts_bucket_name == null ? null : local.LAMBDA_RUNTIME - timeout = var.lambda_timeout - role = aws_iam_role.default.arn + function_name = "${var.service_name}-${local.ID_ENV}" + description = "${var.service_name} lambda service in ${local.SPACED_ID_ENV}" + image_uri = data.aws_ecr_image.default.image_uri + package_type = "Image" + timeout = var.lambda_timeout + role = aws_iam_role.default.arn environment { variables = merge( // add lambda web adapter env vars if aws_lwa_port set var.aws_lwa_port != null ? { READINESS_CHECK_PATH = local.READINESS_CHECK_PATH - PORT = var.aws_lwa_port + AWS_LWA_PORT = var.aws_lwa_port "${local.SERVICE_NAME_UPPER}_PORT" = var.aws_lwa_port } : {}, var.env_vars, ) } - - // add lambda web adapter layer if aws_lwa_port set - layers = concat(var.lambda_layer_arns, var.aws_lwa_port != null ? [ - "arn:aws:lambda:${data.aws_region.current.name}:753240598075:layer:LambdaAdapterLayerX86:${local.WEB_ADAPTER_LAYER_VERSION}" - ] : []) } resource "aws_lambda_function_url" "default" { diff --git a/infrastructure/terraform/aws/modules/provided-lambda/v001/variables.tf b/infrastructure/terraform/aws/modules/provided-lambda/v001/variables.tf index 4eaee90e..d446d29c 100644 --- a/infrastructure/terraform/aws/modules/provided-lambda/v001/variables.tf +++ b/infrastructure/terraform/aws/modules/provided-lambda/v001/variables.tf @@ -22,7 +22,6 @@ variable "create_secret" { default = false description = "adds lambda arn in secrets manager for local testing" } -variable "artifacts_bucket_name" {} variable "ssm_prefix" {} variable "env_id" {} variable "lambda_timeout" { default = 30 } From 9462d3eb5ea2e60cb45360041fa37352b168f2d1 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:17:56 -0700 Subject: [PATCH 29/54] move graphql to provided lambda terraform module --- .../modules/environment/v001/graphql-api.tf | 2 +- .../aws/modules/environment/v001/graphql.tf | 104 ----------------- .../environment/v001/lambda-services.tf | 108 ++++++++++-------- 3 files changed, 61 insertions(+), 153 deletions(-) delete mode 100644 infrastructure/terraform/aws/modules/environment/v001/graphql.tf diff --git a/infrastructure/terraform/aws/modules/environment/v001/graphql-api.tf b/infrastructure/terraform/aws/modules/environment/v001/graphql-api.tf index ce656d45..56f75cf6 100644 --- a/infrastructure/terraform/aws/modules/environment/v001/graphql-api.tf +++ b/infrastructure/terraform/aws/modules/environment/v001/graphql-api.tf @@ -5,7 +5,7 @@ module "graphql_apigwv2" { enable_api_auth = var.enable_api_auth api_version = 001 env = var.env - lambda_invoke_arn = aws_lambda_function.graphql.invoke_arn + lambda_invoke_arn = module.graphql.lambda_invoke_arn enable_api_auto_deploy = var.enable_api_auto_deploy cognito_client_id = aws_cognito_user_pool_client.client.id cognito_endpoint = aws_cognito_user_pool.pool.endpoint diff --git a/infrastructure/terraform/aws/modules/environment/v001/graphql.tf b/infrastructure/terraform/aws/modules/environment/v001/graphql.tf deleted file mode 100644 index bb8b3bda..00000000 --- a/infrastructure/terraform/aws/modules/environment/v001/graphql.tf +++ /dev/null @@ -1,104 +0,0 @@ -locals { - GRAPHQL = "graphql" -} - -data "aws_s3_object" "graphql" { - bucket = var.artifacts_bucket_name - key = "${local.GRAPHQL}-src.zip" -} - -resource "aws_lambda_function" "graphql" { - function_name = "${local.GRAPHQL}-${local.ID_ENV}" - description = "${local.GRAPHQL} on api gateway in ${local.SPACED_ID_ENV}" - s3_bucket = data.aws_s3_object.graphql.bucket - s3_key = data.aws_s3_object.graphql.key - s3_object_version = data.aws_s3_object.graphql.version_id - handler = local.BINARY_NAME - runtime = local.LAMBDA_RUNTIME - timeout = 30 - role = aws_iam_role.graphql_role.arn - environment { - variables = { - ENABLE_API_AUTH = var.enable_api_auth - RULE_URL = module.rule.lambda_function_url - REQUEST_CREATE_URL = module.request_create.lambda_function_url - REQUEST_APPROVE_URL = module.request_approve.lambda_function_url - REQUEST_BY_ID_URL = module.request_by_id.lambda_function_url - REQUESTS_BY_ACCOUNT_URL = module.requests_by_account.lambda_function_url - TRANSACTIONS_BY_ACCOUNT_URL = module.transactions_by_account.lambda_function_url - TRANSACTION_BY_ID_URL = module.transaction_by_id.lambda_function_url - BALANCE_BY_ACCOUNT_URL = module.balance_by_account.lambda_function_url - READINESS_CHECK_PATH = local.READINESS_CHECK_PATH - PORT = local.GRAPHQL_PORT - GRAPHQL_PORT = local.GRAPHQL_PORT - } - } - layers = [ - "arn:aws:lambda:${data.aws_region.current.name}:753240598075:layer:LambdaAdapterLayerX86:${local.WEB_ADAPTER_LAYER_VERSION}" - ] -} - -resource "aws_cloudwatch_log_group" "graphql" { - name = "/aws/lambda/${aws_lambda_function.graphql.function_name}" - retention_in_days = 30 -} - -resource "aws_iam_role" "graphql_role" { - name = "${local.GRAPHQL}-${local.ID_ENV}" - assume_role_policy = data.aws_iam_policy_document.graphql_trust_policy.json -} - -data "aws_iam_policy_document" "graphql_trust_policy" { - version = "2012-10-17" - statement { - sid = "GraphQLFaasTrustPolicy${local.TITLED_ID_ENV}" - effect = "Allow" - actions = [ - "sts:AssumeRole", - ] - principals { - type = "Service" - identifiers = ["lambda.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy" "graphql_policy" { - name = "${local.GRAPHQL}-${var.env}" - role = aws_iam_role.graphql_role.id - - policy = data.aws_iam_policy_document.graphql_policy.json -} - -data "aws_iam_policy_document" "graphql_policy" { - version = "2012-10-17" - - statement { - sid = "GraphQLInvokeLambdaPolicy${local.TITLED_ID_ENV}" - actions = [ - "lambda:InvokeFunction" - ] - resources = [ - module.rule.lambda_arn, - module.request_create.lambda_arn, - module.request_approve.lambda_arn, - module.request_by_id.lambda_arn, - module.requests_by_account.lambda_arn, - module.transactions_by_account.lambda_arn, - module.transaction_by_id.lambda_arn, - module.balance_by_account.lambda_arn, - ] - } - - statement { - sid = "GraphQLLoggingPolicy${local.TITLED_ID_ENV}" - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = [ - "*", - ] - } -} diff --git a/infrastructure/terraform/aws/modules/environment/v001/lambda-services.tf b/infrastructure/terraform/aws/modules/environment/v001/lambda-services.tf index 8265a160..413c188a 100644 --- a/infrastructure/terraform/aws/modules/environment/v001/lambda-services.tf +++ b/infrastructure/terraform/aws/modules/environment/v001/lambda-services.tf @@ -1,4 +1,43 @@ -// modules require ${service_name}-src.zip in artifacts s3 +// modules require ${service_name}-${ID_ENV} ecr image repositories in init-$ENV module + +module "graphql" { + source = "../../provided-lambda/v001" + service_name = "graphql" + env = var.env + ssm_prefix = var.ssm_prefix + env_id = var.env_id + env_vars = merge(local.POSTGRES_VARS, { + ENABLE_API_AUTH = var.enable_api_auth + RULE_URL = module.rule.lambda_function_url + REQUEST_CREATE_URL = module.request_create.lambda_function_url + REQUEST_APPROVE_URL = module.request_approve.lambda_function_url + REQUEST_BY_ID_URL = module.request_by_id.lambda_function_url + REQUESTS_BY_ACCOUNT_URL = module.requests_by_account.lambda_function_url + TRANSACTIONS_BY_ACCOUNT_URL = module.transactions_by_account.lambda_function_url + TRANSACTION_BY_ID_URL = module.transaction_by_id.lambda_function_url + BALANCE_BY_ACCOUNT_URL = module.balance_by_account.lambda_function_url + }) + aws_lwa_port = local.GRAPHQL_PORT + invoke_url_principals = [] + attached_policy_arns = [] + create_secret = true // suppports local testing +} + +module "rule" { + source = "../../provided-lambda/v001" + service_name = "rule" + env = var.env + ssm_prefix = var.ssm_prefix + env_id = var.env_id + env_vars = merge(local.POSTGRES_VARS, {}) + aws_lwa_port = local.RULE_PORT + invoke_url_principals = [ + module.graphql.lambda_role_arn, + module.request_create.lambda_role_arn, + ] + attached_policy_arns = [] + create_secret = true +} module "request_create" { source = "../../provided-lambda/v001" @@ -10,10 +49,9 @@ module "request_create" { RULE_URL = module.rule.lambda_function_url }) aws_lwa_port = local.REQUEST_CREATE_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + invoke_url_principals = [module.graphql.lambda_role_arn] attached_policy_arns = [] - create_secret = true // suppports local testing + create_secret = true } module "request_approve" { @@ -24,12 +62,23 @@ module "request_approve" { env_id = var.env_id env_vars = merge(local.POSTGRES_VARS, {}) aws_lwa_port = local.REQUEST_APPROVE_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + invoke_url_principals = [module.graphql.lambda_role_arn] create_secret = true attached_policy_arns = [] } +module "balance_by_account" { + source = "../../provided-lambda/v001" + service_name = "balance-by-account" + env = var.env + ssm_prefix = var.ssm_prefix + env_id = var.env_id + env_vars = merge(local.POSTGRES_VARS, {}) + aws_lwa_port = local.BALANCE_BY_ACCOUNT_PORT + invoke_url_principals = [module.graphql.lambda_role_arn] + create_secret = true +} + module "requests_by_account" { source = "../../provided-lambda/v001" service_name = "requests-by-account" @@ -40,8 +89,7 @@ module "requests_by_account" { RETURN_RECORD_LIMIT = local.RETURN_RECORD_LIMIT }) aws_lwa_port = local.REQUESTS_BY_ACCOUNT_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + invoke_url_principals = [module.graphql.lambda_role_arn] create_secret = true } @@ -53,8 +101,7 @@ module "request_by_id" { env_id = var.env_id env_vars = merge(local.POSTGRES_VARS, {}) aws_lwa_port = local.REQUEST_BY_ID_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + invoke_url_principals = [module.graphql.lambda_role_arn] create_secret = true } @@ -64,12 +111,11 @@ module "transactions_by_account" { env = var.env ssm_prefix = var.ssm_prefix env_id = var.env_id - aws_lwa_port = local.TRANSACTIONS_BY_ACCOUNT_PORT env_vars = merge(local.POSTGRES_VARS, { RETURN_RECORD_LIMIT = local.RETURN_RECORD_LIMIT }) - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + aws_lwa_port = local.TRANSACTIONS_BY_ACCOUNT_PORT + invoke_url_principals = [module.graphql.lambda_role_arn] create_secret = true } @@ -81,21 +127,7 @@ module "transaction_by_id" { env_id = var.env_id env_vars = merge(local.POSTGRES_VARS, {}) aws_lwa_port = local.TRANSACTION_BY_ID_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name - create_secret = true -} - -module "balance_by_account" { - source = "../../provided-lambda/v001" - service_name = "balance-by-account" - env = var.env - ssm_prefix = var.ssm_prefix - env_id = var.env_id - env_vars = merge(local.POSTGRES_VARS, {}) - aws_lwa_port = local.BALANCE_BY_ACCOUNT_PORT - invoke_url_principals = [aws_iam_role.graphql_role.arn] - artifacts_bucket_name = var.artifacts_bucket_name + invoke_url_principals = [module.graphql.lambda_role_arn] create_secret = true } @@ -108,25 +140,5 @@ module "auto_confirm" { env_vars = merge(local.POSTGRES_VARS, { INITIAL_ACCOUNT_BALANCE = var.initial_account_balance }) - artifacts_bucket_name = var.artifacts_bucket_name invoke_arn_principals = ["cognito-idp.amazonaws.com"] } - -module "rule" { - source = "../../provided-lambda/v001" - service_name = "rule" - env = var.env - ssm_prefix = var.ssm_prefix - env_id = var.env_id - env_vars = merge(local.POSTGRES_VARS, { - RUST_LOG = "info" - }) - aws_lwa_port = local.RULE_PORT - invoke_url_principals = [ - aws_iam_role.graphql_role.arn, - module.request_create.lambda_role_arn, - ] - artifacts_bucket_name = var.artifacts_bucket_name - attached_policy_arns = [] - create_secret = true // suppports local testing -} From 963de0339fbbc2e6be7b9d2dede81978fd0624d7 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:20:13 -0700 Subject: [PATCH 30/54] remove web adapter from terraform after moving it to docker --- .../terraform/aws/modules/environment/v001/locals.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/infrastructure/terraform/aws/modules/environment/v001/locals.tf b/infrastructure/terraform/aws/modules/environment/v001/locals.tf index 60990a09..ae25607a 100644 --- a/infrastructure/terraform/aws/modules/environment/v001/locals.tf +++ b/infrastructure/terraform/aws/modules/environment/v001/locals.tf @@ -6,7 +6,6 @@ locals { SERVICES_CONF = local.PROJECT_CONF.services RULE_PORT = local.SERVICES_CONF.rule.env_var.set.RULE_PORT.default GRAPHQL_PORT = local.SERVICES_CONF.graphql.env_var.set.GRAPHQL_PORT.default - READINESS_CHECK_PATH = local.PROJECT_CONF.infrastructure.terraform.aws.modules.environment.env_var.set.READINESS_CHECK_PATH.default BALANCE_BY_ACCOUNT_PORT = local.SERVICES_CONF.balance-by-account.env_var.set.BALANCE_BY_ACCOUNT_PORT.default TRANSACTION_BY_ID_PORT = local.SERVICES_CONF.transaction-by-id.env_var.set.TRANSACTION_BY_ID_PORT.default TRANSACTIONS_BY_ACCOUNT_PORT = local.SERVICES_CONF.transactions-by-account.env_var.set.TRANSACTIONS_BY_ACCOUNT_PORT.default @@ -15,9 +14,5 @@ locals { REQUEST_APPROVE_PORT = local.SERVICES_CONF.request-approve.env_var.set.REQUEST_APPROVE_PORT.default REQUEST_CREATE_PORT = local.SERVICES_CONF.request-create.env_var.set.REQUEST_CREATE_PORT.default RETURN_RECORD_LIMIT = local.SERVICES_CONF.env_var.set.RETURN_RECORD_LIMIT.default - WEB_ADAPTER_LAYER_VERSION = local.PROJECT_CONF.infrastructure.terraform.aws.modules.environment.env_var.set.WEB_ADAPTER_LAYER_VERSION.default - WEB_ADAPTER_LAYER_ARN = "arn:aws:lambda:${data.aws_region.current.name}:753240598075:layer:LambdaAdapterLayerX86:${local.WEB_ADAPTER_LAYER_VERSION}" - BINARY_NAME = local.SERVICES_CONF.env_var.set.BINARY_NAME.default - LAMBDA_RUNTIME = local.PROJECT_CONF.infrastructure.terraform.aws.modules.env_var.set.LAMBDA_RUNTIME.default SQL_TYPE = local.PROJECT_CONF.migrations.env_var.set.SQL_TYPE.default } From 1aca31af10fb8161a7e029939167c69b0ec5c5cf Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:20:57 -0700 Subject: [PATCH 31/54] configure max erc repo image count in terraform --- infrastructure/terraform/aws/environments/init-dev/main.tf | 1 + infrastructure/terraform/aws/environments/init-prod/main.tf | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/aws/environments/init-dev/main.tf b/infrastructure/terraform/aws/environments/init-dev/main.tf index d355df7e..a0c5f037 100644 --- a/infrastructure/terraform/aws/environments/init-dev/main.tf +++ b/infrastructure/terraform/aws/environments/init-dev/main.tf @@ -26,4 +26,5 @@ module "project_storage_dev" { tfstate_bucket_name_prefix = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default ddb_table_name_prefix = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default ddb_table_hash_key = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + max_image_storage_count = 10 } diff --git a/infrastructure/terraform/aws/environments/init-prod/main.tf b/infrastructure/terraform/aws/environments/init-prod/main.tf index cb949001..8d9e382e 100644 --- a/infrastructure/terraform/aws/environments/init-prod/main.tf +++ b/infrastructure/terraform/aws/environments/init-prod/main.tf @@ -1,5 +1,5 @@ locals { - ENV = "prod" + ENV = "prod" PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) INFRA_ENV_VAR = local.PROJECT_CONF.infrastructure.terraform.aws.modules.environment.env_var.set ENV_ID = local.PROJECT_CONF.infrastructure.terraform.env-id.prod.env_var.set.PROD_ENV_ID.default @@ -25,4 +25,5 @@ module "project_storage_prod" { tfstate_bucket_name_prefix = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default ddb_table_name_prefix = local.STORAGE_ENV_VAR.DDB_TABLE_NAME_PREFIX.default ddb_table_hash_key = local.STORAGE_ENV_VAR.DDB_TABLE_HASH_KEY.default + max_image_storage_count = 10 } From 7153ae5ba47087f9a77ed509dccd291c8114a191 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:21:20 -0700 Subject: [PATCH 32/54] terraform lockfile --- .../aws/environments/prod/.terraform.lock.hcl | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/infrastructure/terraform/aws/environments/prod/.terraform.lock.hcl b/infrastructure/terraform/aws/environments/prod/.terraform.lock.hcl index 76e3e09e..19b23947 100644 --- a/infrastructure/terraform/aws/environments/prod/.terraform.lock.hcl +++ b/infrastructure/terraform/aws/environments/prod/.terraform.lock.hcl @@ -2,24 +2,24 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.31.0" + version = "5.44.0" hashes = [ - "h1:2eauBmfftzGMpzFQn9aHSXiyaO3Ve5cnihmXcKGGpgU=", - "zh:0cdb9c2083bf0902442384f7309367791e4640581652dda456f2d6d7abf0de8d", - "zh:2fe4884cb9642f48a5889f8dff8f5f511418a18537a9dfa77ada3bcdad391e4e", - "zh:36d8bdd72fe61d816d0049c179f495bc6f1e54d8d7b07c45b62e5e1696882a89", - "zh:539dd156e3ec608818eb21191697b230117437a58587cbd02ce533202a4dd520", - "zh:6a53f4b57ac4eb3479fc0d8b6e301ca3a27efae4c55d9f8bd24071b12a03361c", - "zh:6faeb8ff6792ca7af1c025255755ad764667a300291cc10cea0c615479488c87", - "zh:7d9423149b323f6d0df5b90c4d9029e5455c670aea2a7eb6fef4684ba7eb2e0b", - "zh:8235badd8a5d0993421cacf5ead48fac73d3b5a25c8a68599706a404b1f70730", - "zh:860b4f60842b2879c5128b7e386c8b49adeda9287fed12c5cd74861bb659bbcd", + "h1:Cdt9DdAhuIqo/BqxybHPFRyC2Z4crxd7Xj39yHoyagk=", + "zh:1224a42bb04574785549b89815d98bda11f6e9992352fc6c36c5622f3aea91c0", + "zh:2a8d1095a2f1ab097f516d9e7e0d289337849eebb3fcc34f075070c65063f4fa", + "zh:46cce11150eb4934196d9bff693b72d0494c85917ceb3c2914d5ff4a785af861", + "zh:4a7c15d585ee747d17f4b3904851cd95cfbb920fa197aed3df78e8d7ef9609b6", + "zh:508f1a85a0b0f93bf26341207d809bd55b60c8fdeede40097d91f30111fc6f5d", + "zh:52f968ffc21240213110378d0ffb298cbd23e9157a6d01dfac5a4360492d69c2", + "zh:5e9846b48ef03eb59541049e81b15cae8bc7696a3779ae4a5412fdce60bb24e0", + "zh:850398aecaf7dc0231fc320fdd6dffe41836e07a54c8c7b40eb28e7525d3c0a9", + "zh:8f87eeb05bdd1b873b6cfb3898dfad6402ac180dfa3c8f9754df8f85dcf92ca6", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:b021fceaf9382c8fe3c6eb608c24d01dce3d11ba7e65bb443d51ca9b90e9b237", - "zh:b38b0bfc1c69e714e80cf1c9ea06e687ee86aa9f45694be28eb07adcebbe0489", - "zh:c972d155f6c01af9690a72adfb99cfc24ef5ef311ca92ce46b9b13c5c153f572", - "zh:e0dd29920ec84fdb6026acff44dcc1fb1a24a0caa093fa04cdbc713d384c651d", - "zh:e3127ebd2cb0374cd1808f911e6bffe2f4ac4d84317061381242353f3a7bc27d", + "zh:c726b87cd6ed111536f875dccedecff21abc802a4087264515ffab113cac36dc", + "zh:d57ea706d2f98b93c7b05b0c6bc3420de8e8cf2d0b6703085dc15ed239b2cc49", + "zh:d5d1a21246e68c2a7a04c5619eb0ad5a81644f644c432cb690537b816a156de2", + "zh:e869904cac41114b7e4ee66bcd2ce4585ed15ca842040a60cb47119f69472c91", + "zh:f1a09f2f3ea72cbe795b865cf31ad9b1866a536a8050cf0bb93d3fa51069582e", ] } From f695c56d399acf14a8fb21cb9ac9f958327a7a1d Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:25:44 -0700 Subject: [PATCH 33/54] remove non null type declaration in graphql mutation --- tests/thunder-tests/thunderclient.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/thunder-tests/thunderclient.json b/tests/thunder-tests/thunderclient.json index 9b3dfd68..64937c18 100644 --- a/tests/thunder-tests/thunderclient.json +++ b/tests/thunder-tests/thunderclient.json @@ -138,7 +138,7 @@ "method": "POST", "sortNum": 50000, "created": "2023-01-22T03:27:31.002Z", - "modified": "2023-04-28T06:45:28.169Z", + "modified": "2024-04-03T04:15:05.853Z", "headers": [ { "name": "Content-Type", @@ -155,8 +155,8 @@ "raw": "", "form": [], "graphql": { - "query": "mutation createRequest($transaction_items: [TransactionItemInput!], $auth_account: String!) {\n createRequest(transaction_items: $transaction_items, auth_account: $auth_account) {\n id\n rule_instance_id\n author\n author_device_id\n author_device_latlng\n author_role\n sum_value\n transaction_items {\n id\n transaction_id\n item_id\n price\n rule_exec_ids\n quantity\n debitor_first\n rule_instance_id\n unit_of_measurement\n units_measured\n debitor\n creditor\n debitor_profile_id\n creditor_profile_id\n debitor_approval_time\n creditor_approval_time\n debitor_expiration_time\n creditor_expiration_time\n debitor_rejection_time\n creditor_rejection_time \n }\n }\n}", - "variables": "{\n\t\"auth_account\": \"GroceryStore\",\n\t\"transaction_items\": [\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.180\",\n\t\t\t\"quantity\": \"1.000\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.270\",\n\t\t\t\"quantity\": \"2.000\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.360\",\n\t\t\t\"quantity\": \"3.000\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"milk\",\n\t\t\t\"price\": \"2.000\",\n\t\t\t\"quantity\": \"1\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"bread\",\n\t\t\t\"price\": \"3.000\",\n\t\t\t\"quantity\": \"2\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"eggs\",\n\t\t\t\"price\": \"4.000\",\n\t\t\t\"quantity\": \"3\",\n\t\t\t\"debitor_first\": null,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t}\n\t]\n}" + "query": "mutation createRequest($transaction_items: [TransactionItemInput!]!, $auth_account: String!) {\n createRequest(transaction_items: $transaction_items, auth_account: $auth_account) {\n id\n rule_instance_id\n author\n author_device_id\n author_device_latlng\n author_role\n sum_value\n transaction_items {\n id\n transaction_id\n item_id\n price\n rule_exec_ids\n quantity\n debitor_first\n rule_instance_id\n unit_of_measurement\n units_measured\n debitor\n creditor\n debitor_profile_id\n creditor_profile_id\n debitor_approval_time\n creditor_approval_time\n debitor_expiration_time\n creditor_expiration_time\n debitor_rejection_time\n creditor_rejection_time \n }\n }\n}", + "variables": "{\n\t\"auth_account\": \"GroceryStore\",\n\t\"transaction_items\": [\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.180\",\n\t\t\t\"quantity\": \"1.000\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.270\",\n\t\t\t\"quantity\": \"2.000\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"9% state sales tax\",\n\t\t\t\"price\": \"0.360\",\n\t\t\t\"quantity\": \"3.000\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": \"1\",\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"StateOfCalifornia\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"milk\",\n\t\t\t\"price\": \"2.000\",\n\t\t\t\"quantity\": \"1\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"bread\",\n\t\t\t\"price\": \"3.000\",\n\t\t\t\"quantity\": \"2\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t},\n\t\t{\n\t\t\t\"id\": null,\n\t\t\t\"transaction_id\": null,\n\t\t\t\"item_id\": \"eggs\",\n\t\t\t\"price\": \"4.000\",\n\t\t\t\"quantity\": \"3\",\n\t\t\t\"debitor_first\": false,\n\t\t\t\"rule_instance_id\": null,\n\t\t\t\"unit_of_measurement\": null,\n\t\t\t\"units_measured\": null,\n\t\t\t\"debitor\": \"JacobWebb\",\n\t\t\t\"creditor\": \"GroceryStore\",\n\t\t\t\"debitor_profile_id\": null,\n\t\t\t\"creditor_profile_id\": null,\n\t\t\t\"debitor_approval_time\": null,\n\t\t\t\"creditor_approval_time\": null,\n\t\t\t\"debitor_expiration_time\": null,\n\t\t\t\"creditor_expiration_time\": null,\n\t\t\t\"debitor_rejection_time\": null,\n\t\t\t\"creditor_rejection_time\": null\n\t\t}\n\t]\n}" } }, "tests": [] @@ -298,7 +298,7 @@ "method": "POST", "sortNum": 100000, "created": "2023-01-22T03:27:31.007Z", - "modified": "2023-06-07T04:43:21.312Z", + "modified": "2024-04-03T05:37:27.089Z", "headers": [ { "name": "Content-Type", @@ -315,8 +315,8 @@ "raw": "", "form": [], "graphql": { - "query": "query getTransactionByID($id: String!, $auth_account: String!) {\n transactionByID(id: $id, auth_account: $auth_account) {\n id\n rule_instance_id\n author\n author_device_id\n author_device_latlng\n author_role\n transaction_items {\n id\n transaction_id\n item_id\n price\n quantity\n debitor_first\n rule_exec_ids\n rule_instance_id\n unit_of_measurement\n units_measured\n debitor\n creditor\n debitor_profile_id\n creditor_profile_id\n debitor_approval_time\n creditor_approval_time\n debitor_expiration_time\n creditor_expiration_time\n debitor_rejection_time\n creditor_rejection_time\n approvals {\n id\n rule_instance_id\n transaction_id\n transaction_item_id\n account_name\n account_role\n device_id\n device_latlng\n approval_time\n rejection_time\n expiration_time\n }\n }\n }\n}", - "variables": "{\n \"auth_account\": \"JoeCarter\",\n \"id\": \"2\"\n}" + "query": "query getTransactionByID($id: String!, $account_name: String!, $auth_account: String!) {\n transactionByID(id: $id, account_name: $account_name, auth_account: $auth_account) {\n id\n rule_instance_id\n author\n author_device_id\n author_device_latlng\n author_role\n transaction_items {\n id\n transaction_id\n item_id\n price\n quantity\n debitor_first\n rule_exec_ids\n rule_instance_id\n unit_of_measurement\n units_measured\n debitor\n creditor\n debitor_profile_id\n creditor_profile_id\n debitor_approval_time\n creditor_approval_time\n debitor_expiration_time\n creditor_expiration_time\n debitor_rejection_time\n creditor_rejection_time\n approvals {\n id\n rule_instance_id\n transaction_id\n transaction_item_id\n account_name\n account_role\n device_id\n device_latlng\n approval_time\n rejection_time\n expiration_time\n }\n }\n }\n}", + "variables": "{\n \"auth_account\": \"JoeCarter\",\n \"account_name\": \"JoeCarter\",\n \"id\": \"2\"\n}" } }, "tests": [] From 51bc1d1cd57ff1a98cc50a829da405c51900149f Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:28:30 -0700 Subject: [PATCH 34/54] remove default port assignment --- services/balance-by-account/src/main.rs | 2 +- services/graphql/src/main.rs | 2 +- services/request-approve/src/main.rs | 2 +- services/request-by-id/src/main.rs | 2 +- services/request-create/src/main.rs | 2 +- services/requests-by-account/src/main.rs | 2 +- services/rule/src/main.rs | 2 +- services/transaction-by-id/src/main.rs | 2 +- services/transactions-by-account/src/main.rs | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/balance-by-account/src/main.rs b/services/balance-by-account/src/main.rs index e7c843cc..9a7ee935 100644 --- a/services/balance-by-account/src/main.rs +++ b/services/balance-by-account/src/main.rs @@ -53,7 +53,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("BALANCE_BY_ACCOUNT_PORT").unwrap_or("10004".to_string()); + let port = env::var("BALANCE_BY_ACCOUNT_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/graphql/src/main.rs b/services/graphql/src/main.rs index 9c9eb5b5..4d87bbd7 100644 --- a/services/graphql/src/main.rs +++ b/services/graphql/src/main.rs @@ -268,7 +268,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("GRAPHQL_PORT").unwrap_or("10000".to_string()); + let port = env::var("GRAPHQL_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/request-approve/src/main.rs b/services/request-approve/src/main.rs index 03e86060..697fe66b 100644 --- a/services/request-approve/src/main.rs +++ b/services/request-approve/src/main.rs @@ -69,7 +69,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("REQUEST_APPROVE_PORT").unwrap_or("10003".to_string()); + let port = env::var("REQUEST_APPROVE_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/request-by-id/src/main.rs b/services/request-by-id/src/main.rs index ed898c40..ecd15f6e 100644 --- a/services/request-by-id/src/main.rs +++ b/services/request-by-id/src/main.rs @@ -88,7 +88,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("REQUEST_BY_ID_PORT").unwrap_or("10005".to_string()); + let port = env::var("REQUEST_BY_ID_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/request-create/src/main.rs b/services/request-create/src/main.rs index 70157f8f..5bf3b06a 100644 --- a/services/request-create/src/main.rs +++ b/services/request-create/src/main.rs @@ -183,7 +183,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("REQUEST_CREATE_PORT").unwrap_or("10002".to_string()); + let port = env::var("REQUEST_CREATE_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/requests-by-account/src/main.rs b/services/requests-by-account/src/main.rs index 9aaeb859..1df47818 100644 --- a/services/requests-by-account/src/main.rs +++ b/services/requests-by-account/src/main.rs @@ -67,7 +67,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("REQUESTS_BY_ACCOUNT_PORT").unwrap_or("10006".to_string()); + let port = env::var("REQUESTS_BY_ACCOUNT_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/rule/src/main.rs b/services/rule/src/main.rs index f8063764..9490e949 100644 --- a/services/rule/src/main.rs +++ b/services/rule/src/main.rs @@ -252,7 +252,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("RULE_PORT").unwrap_or("10001".to_string()); + let port = env::var("RULE_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/transaction-by-id/src/main.rs b/services/transaction-by-id/src/main.rs index 83bff1b0..de227652 100644 --- a/services/transaction-by-id/src/main.rs +++ b/services/transaction-by-id/src/main.rs @@ -102,7 +102,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("TRANSACTION_BY_ID_PORT").unwrap_or("10007".to_string()); + let port = env::var("TRANSACTION_BY_ID_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); diff --git a/services/transactions-by-account/src/main.rs b/services/transactions-by-account/src/main.rs index 9e01c29b..510e3a4d 100644 --- a/services/transactions-by-account/src/main.rs +++ b/services/transactions-by-account/src/main.rs @@ -67,7 +67,7 @@ async fn main() { let hostname_or_ip = env::var("HOSTNAME_OR_IP").unwrap_or("0.0.0.0".to_string()); - let port = env::var("TRANSACTIONS_BY_ACCOUNT_PORT").unwrap_or("10006".to_string()); + let port = env::var("TRANSACTIONS_BY_ACCOUNT_PORT").unwrap(); let serve_addr = format!("{hostname_or_ip}:{port}"); From 6dc13e441552ed0539a3b1caf41b991c0d28fc85 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:30:45 -0700 Subject: [PATCH 35/54] switch services to shared ecr lambda makefile --- make/rust.mk | 44 ++++++--------- migrations/go-migrate/makefile | 68 +---------------------- services/auto-confirm/makefile | 1 + services/balance-by-account/makefile | 38 +++++-------- services/graphql/makefile | 12 +--- services/request-approve/makefile | 14 +---- services/request-by-id/makefile | 14 +---- services/request-create/makefile | 14 +---- services/requests-by-account/makefile | 14 +---- services/rule/makefile | 11 +--- services/transaction-by-id/makefile | 14 +---- services/transactions-by-account/makefile | 14 +---- 12 files changed, 42 insertions(+), 216 deletions(-) diff --git a/make/rust.mk b/make/rust.mk index 99ee3a58..fcc2905d 100644 --- a/make/rust.mk +++ b/make/rust.mk @@ -1,7 +1,18 @@ -# requires include shared.mk -CARGO_BUILD_DIR=target -LAMBDA_TARGET=x86_64-unknown-linux-musl -RELEASE_DIR=$(CARGO_BUILD_DIR)/$(LAMBDA_TARGET)/release +# requires include shared.mk in $APP_NAME/makefile +MIGRATIONS_DIR=$(RELATIVE_PROJECT_ROOT_PATH)/migrations + +start: + @$(MAKE) get-secrets ENV=local + nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & + +stop: + $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop + +start-alone: + rm -f $(NOHUP_LOG) + $(MAKE) -C $(MIGRATIONS_DIR) start + $(MAKE) start + tail -F $(NOHUP_LOG) install: cargo fetch @@ -9,7 +20,6 @@ install: test: $(MAKE) test-lint $(MAKE) test-unit - $(MAKE) -C '../..' test-compose-up test-unit: cargo test @@ -21,29 +31,7 @@ test-lint: lint: $(MAKE) test-lint -clean-build: - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/clean-binary.sh \ - --app-name $(APP_NAME) \ - --binary-name $(EXECUTABLE_NAME); \ - compile: - if ! command -v cross --version &> /dev/null; then cargo install cross --git https://github.com/cross-rs/cross; fi - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - cross build \ - --manifest-path=$(SUB_PATH)/Cargo.toml \ - --target $(LAMBDA_TARGET) \ - --release - -compile-dev: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ cargo build \ - --manifest-path=$(SUB_PATH)/Cargo.toml - -zip: - @cp $(RELATIVE_PROJECT_ROOT_PATH)/$(RELEASE_DIR)/$(APP_NAME) $(EXECUTABLE_NAME) - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/zip-executable.sh \ - --app-name $(APP_NAME) \ - --artifact-name $(ARTIFACT_NAME) \ - --executable-name $(EXECUTABLE_NAME) \ No newline at end of file + --manifest-path=$(SUB_PATH)/Cargo.toml \ No newline at end of file diff --git a/migrations/go-migrate/makefile b/migrations/go-migrate/makefile index 59f1c53f..d18e1827 100644 --- a/migrations/go-migrate/makefile +++ b/migrations/go-migrate/makefile @@ -1,66 +1,4 @@ -SHELL:=/bin/bash -APP_NAME=$(shell basename $(CURDIR)) RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") -PROJECT_CONF_FILE_NAME=project.yaml -PROJECT_CONF=$(RELATIVE_PROJECT_ROOT_PATH)/$(PROJECT_CONF_FILE_NAME) -ENV_ID=$(shell (cd $(RELATIVE_PROJECT_ROOT_PATH); ENV=$(ENV) PROJECT_CONF=$(PROJECT_CONF) . ./scripts/print-env-id.sh)) -TAG_VERSION=latest -IMAGE_NAME=$(IMAGE_PREFIX)-$(ENV_ID)-$(ENV):$(TAG_VERSION) - -test-env-arg: -ifndef ENV - $(error trailing ENV assignment missing, e.g. make test ENV=dev) -endif - -build-image: - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - docker build -f ./docker/go-migrate.Dockerfile -t $(APP_NAME) ./migrations/go-migrate - -push-image: - @$(MAKE) -s test-env-arg - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/push-ecr-image.sh --image-prefix $(APP_NAME) --env $(ENV) - -deploy-image: - @$(MAKE) -s test-env-arg - @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/update-function-image.sh --app-name $(APP_NAME) --env $(ENV) - -update-image: - @$(MAKE) -s test-env-arg - @$(MAKE) -s build-image - @$(MAKE) -s push-image - @$(MAKE) -s deploy-image - -clean-artifact: - @for i in $$(docker image ls | grep '$(APP_NAME)' | awk '{print $$3}'); do docker rmi -f "$$i"; done; - -clean: - @$(MAKE) clean-artifact - -build: - @$(MAKE) clean - @$(MAKE) build-image - -###################### globally required ###################### - -initial-deploy: - @$(MAKE) -s test-env-arg - $(MAKE) build - $(MAKE) push-image - -deploy: - @$(MAKE) -s test-env-arg - $(MAKE) build - $(MAKE) push-image - $(MAKE) deploy-image - -deploy-only: - @$(MAKE) -s test-env-arg - $(MAKE) push-image - $(MAKE) deploy-image - -now: - $(MAKE) build - $(MAKE) push-image ENV=dev - $(MAKE) deploy-image ENV=dev \ No newline at end of file +include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk +BUILD_CTX=migrations/$(APP_NAME) \ No newline at end of file diff --git a/services/auto-confirm/makefile b/services/auto-confirm/makefile index dd779a9b..79c8e9bd 100644 --- a/services/auto-confirm/makefile +++ b/services/auto-confirm/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk signup: @$(MAKE) -s test-env-file diff --git a/services/balance-by-account/makefile b/services/balance-by-account/makefile index 47224868..63f2a572 100644 --- a/services/balance-by-account/makefile +++ b/services/balance-by-account/makefile @@ -1,42 +1,30 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk BALANCE_BY_ACCOUNT_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.BALANCE_BY_ACCOUNT_PORT.default' $(PROJECT_CONF)) BALANCE_BY_ACCOUNT_URL=$(HOST):$(BALANCE_BY_ACCOUNT_PORT) -TEST_ACCOUNT=JacobWebb -TEST_AUTH_ACCOUNT=$(TEST_ACCOUNT) -TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","account_name":"$(TEST_ACCOUNT)"}' -TEST_SENDER_ACCOUNT=$(TEST_ACCOUNT) - -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop +JSON_HEADER="-H 'Content-Type: application/json'" +define TEST_EVENT + '{"auth_account":"$(1)","account_name":"$(1)"}' +endef invoke-local: - @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(BALANCE_BY_ACCOUNT_URL) + @curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,JacobWebb) $(BALANCE_BY_ACCOUNT_URL) demo: @printf "*** request to %s at %s\n" $(SUB_PATH) $(BALANCE_BY_ACCOUNT_URL) - @echo $(TEST_EVENT) | yq -o=json + @echo $(call TEST_EVENT,JacobWebb) | yq -o=json @printf "*** response from %s at %s\n" $(SUB_PATH) $(BALANCE_BY_ACCOUNT_URL) @$(MAKE) invoke-local @printf "\n\n\e[33m*** OBSERVE the following post-transaction conservation test ***\e[0m\n" @printf "\ninitial balances: %s\n\n" 1000 - @printf "GroceryStore new balance: %s\n" $$(curl -s -d '{"auth_account":"GroceryStore","account_name":"GroceryStore"}' $(BALANCE_BY_ACCOUNT_URL)) - @printf "StateOfCalifornia new balance: %s\n" $$(curl -s -d '{"auth_account":"StateOfCalifornia","account_name":"StateOfCalifornia"}' $(BALANCE_BY_ACCOUNT_URL)) - @printf "JacobWebb new balance: %s\n" $$(curl -s -d $(TEST_EVENT) $(BALANCE_BY_ACCOUNT_URL)) - @GS=$$(curl -s -d '{"auth_account":"GroceryStore","account_name":"GroceryStore"}' $(BALANCE_BY_ACCOUNT_URL)); \ - SC=$$(curl -s -d '{"auth_account":"StateOfCalifornia","account_name":"StateOfCalifornia"}' $(BALANCE_BY_ACCOUNT_URL)); \ - JW=$$(curl -s -d $(TEST_EVENT) $(BALANCE_BY_ACCOUNT_URL)); \ + @printf "GroceryStore new balance: %s\n" $$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,GroceryStore) $(BALANCE_BY_ACCOUNT_URL)) + @printf "StateOfCalifornia new balance: %s\n" $$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,StateOfCalifornia) $(BALANCE_BY_ACCOUNT_URL)) + @printf "JacobWebb new balance: %s\n" $$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,JacobWebb) $(BALANCE_BY_ACCOUNT_URL)) + @GS=$$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,GroceryStore) $(BALANCE_BY_ACCOUNT_URL)); \ + SC=$$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,StateOfCalifornia) $(BALANCE_BY_ACCOUNT_URL)); \ + JW=$$(curl -s "$(JSON_HEADER)" -d $(call TEST_EVENT,JacobWebb) $(BALANCE_BY_ACCOUNT_URL)); \ printf "\nconservation test: (%s - 1000) + (%s - 1000) + (%s - 1000) = %s\n" "$$GS" "$$SC" "$$JW" "$$(echo "$$GS + $$SC + $$JW - 3000" | bc)" \ No newline at end of file diff --git a/services/graphql/makefile b/services/graphql/makefile index 357405d3..e64f96db 100644 --- a/services/graphql/makefile +++ b/services/graphql/makefile @@ -1,17 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk - -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk open: open $(GRAPHQL_URI)/ diff --git a/services/request-approve/makefile b/services/request-approve/makefile index fae8dcaa..fec715f9 100644 --- a/services/request-approve/makefile +++ b/services/request-approve/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk REQUEST_APPROVE_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.REQUEST_APPROVE_PORT.default' $(PROJECT_CONF)) REQUEST_APPROVE_URL=$(HOST):$(REQUEST_APPROVE_PORT) @@ -11,19 +12,6 @@ TEST_AUTH_ACCOUNT=$(TEST_ACCOUNT) TEST_ID=3 TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","id":"$(TEST_ID)","account_name":"$(TEST_ACCOUNT)","account_role":"$(TEST_ROLE)"}' -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(REQUEST_APPROVE_URL) | yq -o=json diff --git a/services/request-by-id/makefile b/services/request-by-id/makefile index 0596c681..6668be29 100644 --- a/services/request-by-id/makefile +++ b/services/request-by-id/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk REQUEST_BY_ID_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.REQUEST_BY_ID_PORT.default' $(PROJECT_CONF)) REQUEST_BY_ID_URL=$(HOST):$(REQUEST_BY_ID_PORT) @@ -11,19 +12,6 @@ TEST_TRANSACTION_ID=1 TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","account_name":"$(TEST_AUTH_ACCOUNT)","id":"$(TEST_TRANSACTION_ID)"}' TEST_SENDER_ACCOUNT=$(TEST_ACCOUNT) -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(REQUEST_BY_ID_URL) | yq -o=json diff --git a/services/request-create/makefile b/services/request-create/makefile index 77113e3f..e73dcb69 100644 --- a/services/request-create/makefile +++ b/services/request-create/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk REQUEST_CREATE_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.REQUEST_CREATE_PORT.default' $(PROJECT_CONF)) REQUEST_CREATE_URL=$(HOST):$(REQUEST_CREATE_PORT) @@ -10,19 +11,6 @@ TEST_DATA_FILE=transNoAppr.json TEST_DATA_DIR=$(RELATIVE_PROJECT_ROOT_PATH)/tests/testdata TEST_EVENT='$(shell cat $(TEST_DATA_DIR)/$(TEST_DATA_FILE))' -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(REQUEST_CREATE_URL) | yq -o=json diff --git a/services/requests-by-account/makefile b/services/requests-by-account/makefile index ceaccda7..2f0b8876 100644 --- a/services/requests-by-account/makefile +++ b/services/requests-by-account/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk REQUESTS_BY_ACCOUNT_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.REQUESTS_BY_ACCOUNT_PORT.default' $(PROJECT_CONF)) REQUESTS_BY_ACCOUNT_URL=$(HOST):$(REQUESTS_BY_ACCOUNT_PORT) @@ -10,19 +11,6 @@ TEST_ACCOUNT=JacobWebb TEST_AUTH_ACCOUNT=$(TEST_ACCOUNT) TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","account_name":"$(TEST_ACCOUNT)"}' -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(REQUESTS_BY_ACCOUNT_URL) | yq -o=json diff --git a/services/rule/makefile b/services/rule/makefile index 47d32065..83475182 100644 --- a/services/rule/makefile +++ b/services/rule/makefile @@ -1,24 +1,17 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk RULE_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.RULE_PORT.default' $(PROJECT_CONF)) RULE_URL=$(HOST):$(RULE_PORT) -NOHUP_LOG=$(RELATIVE_PROJECT_ROOT_PATH)/$(shell yq '.env_var.set.NOHUP_LOG.default' $(PROJECT_CONF)) TEST_DATA_FILE=preRuleTrItems.json TEST_DATA_DIR=$(RELATIVE_PROJECT_ROOT_PATH)/tests/testdata TEST_EVENT='$(shell cat $(TEST_DATA_DIR)/$(TEST_DATA_FILE))' dev: - @$(MAKE) start - -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop + @$(MAKE) start-alone run: @$(MAKE) -C ../../migrations run diff --git a/services/transaction-by-id/makefile b/services/transaction-by-id/makefile index 001cb6f3..6807b823 100644 --- a/services/transaction-by-id/makefile +++ b/services/transaction-by-id/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk TRANSACTION_BY_ID_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.TRANSACTION_BY_ID_PORT.default' $(PROJECT_CONF)) TRANSACTION_BY_ID_URL=$(HOST):$(TRANSACTION_BY_ID_PORT) @@ -11,19 +12,6 @@ TEST_TRANSACTION_ID=2 TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","account_name":"$(TEST_AUTH_ACCOUNT)","id":"$(TEST_TRANSACTION_ID)"}' TEST_SENDER_ACCOUNT=$(TEST_ACCOUNT) -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(TRANSACTION_BY_ID_URL) | yq -o=json diff --git a/services/transactions-by-account/makefile b/services/transactions-by-account/makefile index f11ffee2..73cf14f0 100644 --- a/services/transactions-by-account/makefile +++ b/services/transactions-by-account/makefile @@ -1,6 +1,7 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk include $(RELATIVE_PROJECT_ROOT_PATH)/make/rust.mk +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk TRANSACTIONS_BY_ACCOUNT_PORT=$(shell yq '.services["$(APP_NAME)"].env_var.set.TRANSACTIONS_BY_ACCOUNT_PORT.default' $(PROJECT_CONF)) TRANSACTIONS_BY_ACCOUNT_URL=$(HOST):$(TRANSACTIONS_BY_ACCOUNT_PORT) @@ -11,19 +12,6 @@ TEST_EVENT='{"auth_account":"$(TEST_AUTH_ACCOUNT)","account_name":"$(TEST_ACCOUN TEST_SENDER_ACCOUNT=$(TEST_ACCOUNT) RETURN_RECORD_LIMIT=2 -start: - @$(MAKE) get-secrets ENV=local - nohup cargo watch --env-file $(ENV_FILE) -w src -w $(RELATIVE_PROJECT_ROOT_PATH)/crates -x run >> $(NOHUP_LOG) & - -start-alone: - rm -f $(NOHUP_LOG) - $(MAKE) -C $(MIGRATIONS_DIR) run - $(MAKE) start - tail -F $(NOHUP_LOG) - -stop: - $(MAKE) -C $(RELATIVE_PROJECT_ROOT_PATH) stop - invoke-local: @curl -s -H 'Content-Type: application/json' -d $(TEST_EVENT) $(TRANSACTIONS_BY_ACCOUNT_URL) | yq -o=json From 7e0a938b55af9f8749edae3f745c2139c4eb56a9 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:34:17 -0700 Subject: [PATCH 36/54] remove unused bash scripts --- scripts/clean-artifact.sh | 19 ---------------- scripts/clean-binary.sh | 19 ---------------- scripts/put-object.sh | 45 ------------------------------------- scripts/update-function.sh | 46 -------------------------------------- scripts/zip-executable.sh | 28 ----------------------- 5 files changed, 157 deletions(-) delete mode 100644 scripts/clean-artifact.sh delete mode 100644 scripts/clean-binary.sh delete mode 100644 scripts/put-object.sh delete mode 100644 scripts/update-function.sh delete mode 100644 scripts/zip-executable.sh diff --git a/scripts/clean-artifact.sh b/scripts/clean-artifact.sh deleted file mode 100644 index 19635166..00000000 --- a/scripts/clean-artifact.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [[ "$#" -ne 4 ]]; then - echo "use: bash scripts/clean-artifact.sh --app-name request-create --artifact-name request-create-src.zip" - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - --artifact-name) ARTIFACT_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -APP_DIR_PATH=$(source scripts/list-dir-paths.sh --type app | grep --color=never "$APP_NAME") - -rm -f "$APP_DIR_PATH/$ARTIFACT_NAME" \ No newline at end of file diff --git a/scripts/clean-binary.sh b/scripts/clean-binary.sh deleted file mode 100644 index 60a45502..00000000 --- a/scripts/clean-binary.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [[ "$#" -ne 4 ]]; then - echo "use: bash scripts/clean-binary.sh --app-name request-create --binary-name bootstrap" - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - --binary-name) BINARY_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -APP_DIR_PATH=$(source scripts/list-dir-paths.sh --type app | grep --color=never "$APP_NAME") - -rm -f "$APP_DIR_PATH/$BINARY_NAME" \ No newline at end of file diff --git a/scripts/put-object.sh b/scripts/put-object.sh deleted file mode 100644 index a91e6c02..00000000 --- a/scripts/put-object.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$#" -ne 6 ]]; then - cat <<- 'EOF' - use: - bash scripts/put-object.sh \ - --app-name request-create \ - --artifact-name request-create-src.zip \ - --env dev - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - --artifact-name) ARTIFACT_NAME="$2"; shift ;; - --env) ENV="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml - -ARTIFACTS_BUCKET_PREFIX=$(yq '.infrastructure.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) - -APP_DIR_PATH=$(source scripts/list-dir-paths.sh --type app | grep --color=never "$APP_NAME") - -ENV_ID=$(source scripts/print-env-id.sh) - -REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) - -ETAG=$(aws s3api put-object \ - --bucket="$ARTIFACTS_BUCKET_PREFIX-$ENV_ID-$ENV" \ - --key=$ARTIFACT_NAME \ - --body="$PWD/$APP_DIR_PATH/$ARTIFACT_NAME" \ - --region=$REGION \ - --query='ETag' \ - --output=text \ - | xargs) - -echo "*** pushed $ARTIFACT_NAME artifact with ETag: $ETAG" \ No newline at end of file diff --git a/scripts/update-function.sh b/scripts/update-function.sh deleted file mode 100644 index 1be59fb8..00000000 --- a/scripts/update-function.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$#" -ne 6 ]]; then - cat <<- 'EOF' - use: - bash scripts/update-function.sh \ - --app-name request-create \ - --artifact-name request-create-src.zip \ - --env dev - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - --artifact-name) ARTIFACT_NAME="$2"; shift ;; - --env) ENV="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml - -ENV_ID=$(source scripts/print-env-id.sh) - -ID_ENV="$ENV_ID-$ENV" - -ARTIFACTS_BUCKET_PREFIX=$(yq '.infrastructure.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) - -LAMBDA_NAME="$APP_NAME-$ID_ENV" - -REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) - -MOD=$(aws lambda update-function-code \ - --function-name="$LAMBDA_NAME" \ - --s3-key=$ARTIFACT_NAME \ - --s3-bucket="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" \ - --region=$REGION \ - --query 'LastModified' \ - --output text) - -echo "*** $LAMBDA_NAME lambda deployed @ $MOD" \ No newline at end of file diff --git a/scripts/zip-executable.sh b/scripts/zip-executable.sh deleted file mode 100644 index 35c97100..00000000 --- a/scripts/zip-executable.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [[ "$#" -ne 6 ]]; then - cat <<- 'EOF' - use: - bash scripts/zip-executable.sh \ - --app-name request-create \ - --artifact-name request-create-src.zip \ - --executable-name bootstrap - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - --artifact-name) ARTIFACT_NAME="$2"; shift ;; - --executable-name) EXECUTABLE_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -APP_DIR_PATH=$(source scripts/list-dir-paths.sh --type app | grep --color=never "$APP_NAME") - -cd $APP_DIR_PATH - -zip $ARTIFACT_NAME ./$EXECUTABLE_NAME \ No newline at end of file From b830905c498b4157d6b6fdbfb9ad998d86cf4c7b Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:35:08 -0700 Subject: [PATCH 37/54] script naming --- scripts/auth-ecr-repo.sh | 28 ------------------- scripts/auth-ecr.sh | 7 +++++ ...ecr-repo-name.sh => print-ecr-repo-uri.sh} | 13 ++++----- 3 files changed, 13 insertions(+), 35 deletions(-) delete mode 100644 scripts/auth-ecr-repo.sh create mode 100644 scripts/auth-ecr.sh rename scripts/{print-ecr-repo-name.sh => print-ecr-repo-uri.sh} (60%) diff --git a/scripts/auth-ecr-repo.sh b/scripts/auth-ecr-repo.sh deleted file mode 100644 index 42a3cbd8..00000000 --- a/scripts/auth-ecr-repo.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [[ "$#" -ne 4 ]]; then - cat <<- 'EOF' - use: - bash scripts/auth-ecr-repo.sh --image-prefix go-migrate --env dev - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --image-prefix) IMAGE_PREFIX="$2"; shift ;; - --env) ENV="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml - -REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) - -REPO=$(source scripts/print-ecr-repo-name.sh --image-prefix $IMAGE_PREFIX --env $ENV) - -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) - -aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com \ No newline at end of file diff --git a/scripts/auth-ecr.sh b/scripts/auth-ecr.sh new file mode 100644 index 00000000..6e97aa1e --- /dev/null +++ b/scripts/auth-ecr.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +PROJECT_CONF=project.yaml +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) + +aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com \ No newline at end of file diff --git a/scripts/print-ecr-repo-name.sh b/scripts/print-ecr-repo-uri.sh similarity index 60% rename from scripts/print-ecr-repo-name.sh rename to scripts/print-ecr-repo-uri.sh index 129cf6f9..90631ed4 100644 --- a/scripts/print-ecr-repo-name.sh +++ b/scripts/print-ecr-repo-uri.sh @@ -1,28 +1,27 @@ #!/bin/bash -if [[ "$#" -ne 4 ]]; then +if [[ "$#" -ne 6 ]]; then cat <<- 'EOF' use: - bash scripts/print-ecr-repo-name.sh --image-prefix go-migrate --env dev + bash scripts/print-ecr-repo-uri.sh --app-name go-migrate --env dev --env-id 12345 EOF exit 1 fi while [[ "$#" -gt 0 ]]; do case $1 in - --image-prefix) IMAGE_PREFIX="$2"; shift ;; + --app-name) APP_NAME="$2"; shift ;; --env) ENV="$2"; shift ;; + --env-id) ENV_ID="$2"; shift ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac shift done PROJECT_CONF=project.yaml -ENV_ID=$(source scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -IMAGE_NAME=$IMAGE_PREFIX-$ID_ENV +ID_ENV_PREFIX="$ENV_ID/$ENV" +IMAGE_NAME="$ID_ENV_PREFIX/$APP_NAME" REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) aws ecr describe-repositories \ --query "repositories[?contains(repositoryUri, \`$IMAGE_NAME\`)].repositoryUri" \ From 6c60add891d148d34f5138892cadd92f3c350bdd Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:37:56 -0700 Subject: [PATCH 38/54] remove go from start script --- scripts/start-local.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/start-local.sh b/scripts/start-local.sh index 644abe2e..c4a3f400 100644 --- a/scripts/start-local.sh +++ b/scripts/start-local.sh @@ -40,10 +40,10 @@ for d in "${APP_DIRS[@]}"; do RUNTIME=$(yq "$CONF_PATH.runtime" $PROJECT_CONF) BUILD_SRC_PATH=$(yq "$CONF_PATH.build_src_path" $PROJECT_CONF) - # use non shared make target to avoid cross build for rust + # compile rust before starting if [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then echo -e -n "\n${GREEN}*** compiling $d${RESET}\n" - make --no-print-directory -C "$d" compile-dev + make --no-print-directory -C "$d" compile fi # skip starting client in workflows @@ -59,9 +59,6 @@ for d in "${APP_DIRS[@]}"; do if [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then (cd "$d"; eval $(cat $ENV_FILE_NAME) cargo run > /dev/null 2>&1 & disown $!) fi - if [[ "$RUNTIME" == 'go1.x' ]]; then - (cd "$d"; eval $(cat $ENV_FILE_NAME) go run ./$BUILD_SRC_PATH > /dev/null 2>&1 & disown $!) - fi else make --no-print-directory -C "$d" start fi From 031b1618d97d07ab62c475ae6de952e0087943aa Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:38:51 -0700 Subject: [PATCH 39/54] pass image tag to function deploy script --- scripts/update-function-image.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/update-function-image.sh b/scripts/update-function-image.sh index d70bbd91..25273c2c 100644 --- a/scripts/update-function-image.sh +++ b/scripts/update-function-image.sh @@ -2,11 +2,12 @@ set -e -if [[ "$#" -ne 4 ]]; then +if [[ "$#" -ne 6 ]]; then cat <<- 'EOF' use: bash scripts/update-function-image.sh \ --app-name go-migrate \ + --curr-tag 123456789101.dkr.ecr.us-east-1.amazonaws.com/rule-12345-dev:93496996 \ --env dev EOF exit 1 @@ -15,6 +16,7 @@ fi while [[ "$#" -gt 0 ]]; do case $1 in --app-name) APP_NAME="$2"; shift ;; + --curr-tag) CURR_TAG="$2"; shift ;; --env) ENV="$2"; shift ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac @@ -24,17 +26,14 @@ done PROJECT_CONF=project.yaml ENV_ID=$(source scripts/print-env-id.sh) ID_ENV="$ENV_ID-$ENV" -REPO=$(source scripts/print-ecr-repo-name.sh --image-prefix $APP_NAME --env $ENV) -TAG_VERSION=latest -IMAGE_NAME=$REPO:$TAG_VERSION LAMBDA_NAME="$APP_NAME-$ID_ENV" REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -MOD=$(aws lambda update-function-code \ +LAST_MOD=$(aws lambda update-function-code \ --function-name="$LAMBDA_NAME" \ - --image-uri=$IMAGE_NAME \ + --image-uri=$CURR_TAG \ --region=$REGION \ --query 'LastModified' \ --output text) -echo "*** $LAMBDA_NAME lambda deployed @ $MOD" \ No newline at end of file +echo "*** $LAMBDA_NAME lambda deployed @ $LAST_MOD" \ No newline at end of file From e73bd144c3faa2c38ca7739de6ea4085029ebbb3 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:39:48 -0700 Subject: [PATCH 40/54] switch from go to rust training step --- scripts/bootcamp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bootcamp.sh b/scripts/bootcamp.sh index cc917462..567cf065 100644 --- a/scripts/bootcamp.sh +++ b/scripts/bootcamp.sh @@ -81,7 +81,7 @@ eval_with_no_print_directory "$CMD" echo "" -echo -e -n "${GREEN}now add log.Println(\"hello cadet\") at the top of func main() in services/transactions-by-account/cmd/main.go to restart the service with a code change. press any key to continue${RESET}\n\n>" +echo -e -n "${GREEN}now add \"println!(\"hello cadet\");\" at the top of fn main() in services/transactions-by-account/src/main.rs to restart the service with a code change. press any key to continue${RESET}\n\n>" read -n 1 echo "" From d1e281c38b0b00c2b3246312ba2cd220fb1cf167 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:40:48 -0700 Subject: [PATCH 41/54] invoke function only with dev cloud environment values --- scripts/invoke-function-url.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/invoke-function-url.sh b/scripts/invoke-function-url.sh index 8af2315a..167b122f 100644 --- a/scripts/invoke-function-url.sh +++ b/scripts/invoke-function-url.sh @@ -17,7 +17,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --app-name) APP_NAME="$2"; shift ;; --payload) PAYLOAD="$2"; shift ;; - --env) ENVIRONMENT="$2"; shift ;; + --env) ENV="$2"; shift ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac shift @@ -30,7 +30,12 @@ APP_DIR_PATH=$(source scripts/list-dir-paths.sh --type app | grep --color=never ENV_FILE="$APP_DIR_PATH/$ENV_FILE_NAME" if [[ ! -f $ENV_FILE ]]; then - make get-secrets -C $APP_DIR_PATH ENV=$ENVIRONMENT + make get-secrets -C $APP_DIR_PATH ENV=$ENV +fi + +# recreate env file with cloud values if localhost values found +if [[ $(sed -n '/http:\/\/localhost:/=' $ENV_FILE | wc -l | xargs) -gt 0 ]]; then + make get-secrets -C $APP_DIR_PATH ENV=$ENV fi SNAKE_APP_NAME=$(echo $APP_NAME | sed 's/-/_/g') From c552ea1c3cd6896446adc392b99bea2f23bfa170 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:41:03 -0700 Subject: [PATCH 42/54] remove color from grep --- scripts/deploy-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy-all.sh b/scripts/deploy-all.sh index 5945b24f..9dbb3a44 100644 --- a/scripts/deploy-all.sh +++ b/scripts/deploy-all.sh @@ -25,7 +25,7 @@ while [[ "$#" -gt 0 ]]; do --initial) INITIAL='initial-' ;; --services-only) INVENTORY_LIST=$(cat inventory | grep services/) ;; # for convenience, not currently referenced in makefiles - --transaction-services-only) INVENTORY_LIST=$(cat inventory | grep -e transaction -e request -e rule -e graphql) ;; + --transaction-services-only) INVENTORY_LIST=$(cat inventory | grep --color=never -e transaction -e request -e rule -e graphql) ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac shift From 6850dfd32dcf04b645291e95ffbb8fd988e7bf2b Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 17:42:02 -0700 Subject: [PATCH 43/54] remove image tag step --- scripts/push-ecr-image.sh | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/scripts/push-ecr-image.sh b/scripts/push-ecr-image.sh index 36966aad..560cbe50 100644 --- a/scripts/push-ecr-image.sh +++ b/scripts/push-ecr-image.sh @@ -1,34 +1,24 @@ #!/bin/bash -if [[ "$#" -ne 4 ]]; then +if [[ "$#" -ne 2 ]]; then cat <<- 'EOF' use: - bash scripts/push-ecr-image.sh --image-prefix go-migrate --env dev + bash scripts/push-ecr-image.sh \ + --curr-tag 123456789101.dkr.ecr.us-east-1.amazonaws.com/rule-12345-dev:93496996 EOF exit 1 fi while [[ "$#" -gt 0 ]]; do case $1 in - --image-prefix) IMAGE_PREFIX="$2"; shift ;; - --env) ENV="$2"; shift ;; + --curr-tag) CURR_TAG="$2"; shift ;; *) echo "unknown parameter passed: $1"; exit 1 ;; esac shift done -source scripts/auth-ecr-repo.sh --image-prefix $IMAGE_PREFIX --env $ENV +source scripts/test-image-name.sh --curr-tag "$CURR_TAG" -REPO=$(source scripts/print-ecr-repo-name.sh --image-prefix $IMAGE_PREFIX --env $ENV) +source scripts/auth-ecr.sh -PROJECT_CONF=project.yaml -ENV=dev -ENV_ID=$(source scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -TAG_VERSION=latest - -IMAGE_NAME=$REPO:$TAG_VERSION - -docker tag $IMAGE_PREFIX $REPO - -docker push $REPO \ No newline at end of file +docker push $CURR_TAG \ No newline at end of file From 1f48def6f9cbd94c3d5ba250a6ac2d18e8953757 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 18:14:19 -0700 Subject: [PATCH 44/54] expectation faq --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index ce235445..facbf00a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ encryption and replication are secondary government is not above failure, nor is it entitled to steal from the private sector to conceal its failure. improving government depends on failure [predicting](https://en.wikipedia.org/wiki/Time_travel_debugging) the individuals and laws that must be replaced. flying a flag and demanding loyalty before this step is just misdirection +**q.** how does systemaccounting manage expectation? +**a.** central banks providing "forward guidance" *appease* more than they **set** expectation when they allow interest rate manipulation and money printing. systemaccounting prices capital by switching the "risk-free" rate from referencing the hackable price of debt to the immutably recorded price of equity. when the risk-free rate refers to the empirical rate, i.e. to the historical and not the expected, the economy remains protected from the catastrophic failure indulged by intended government mispricing + +removing financial appeasement guides the freedom of speech by recalibrating expectation to the empirical + **q.** will a government hosted payment app reduce my freedom? **a.** the government can already see your transactions. systemaccounting empowers you to see the transactions of your government. access to the realtime financial performance of your government helps protect you from electing individuals who exploit money printing, price manipulation and the absence of accountability to systematize the cost of their failures to everyone else From f054bf8f5c5d0179b6857c0cb68b71fd391b4a8d Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 18:15:53 -0700 Subject: [PATCH 45/54] show lambda perms with convenience script --- scripts/print-lambda-policy.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 scripts/print-lambda-policy.sh diff --git a/scripts/print-lambda-policy.sh b/scripts/print-lambda-policy.sh new file mode 100644 index 00000000..d1d80907 --- /dev/null +++ b/scripts/print-lambda-policy.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [[ "$#" -ne 4 ]]; then + cat <<- 'EOF' + use: + bash scripts/print-lambda-policy.sh --app-name rule --env dev + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --env) ENV="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +ENV_ID=$(source scripts/print-env-id.sh) +ID_ENV="$ENV_ID-$ENV" +LAMBDA_NAME=$APP_NAME-$ID_ENV +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) + +aws lambda get-policy \ + --function-name "$LAMBDA_NAME" \ + --region $REGION \ + --query "Policy" \ + | yq '. | from_json' -P -o=json \ No newline at end of file From 72463443efa7d750dfd69a1c339113c92951e4df Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 18:18:22 -0700 Subject: [PATCH 46/54] build and tag images with git commit hash --- scripts/build-image.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scripts/build-image.sh diff --git a/scripts/build-image.sh b/scripts/build-image.sh new file mode 100644 index 00000000..7d3db846 --- /dev/null +++ b/scripts/build-image.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [[ "$#" -ne 4 ]]; then + cat <<- 'EOF' + use: + bash scripts/build-image.sh --app-name rule --build-ctx . + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --build-ctx) BUILD_CTX="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +HASH=$(git rev-parse --short HEAD) +IMAGE_TAG="$APP_NAME:$HASH" +DOCKERFILE_PATH=./docker/$APP_NAME.Dockerfile + +docker build -f $DOCKERFILE_PATH -t $IMAGE_TAG "$BUILD_CTX" \ No newline at end of file From 0902fce92ae4678272a2fbe880bc222278fa6bd2 Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 18:19:26 -0700 Subject: [PATCH 47/54] ecr scripts used in makefiles and workflows --- scripts/README.md | 70 ++++++++++++++++++++++++------------ scripts/delete-ecr-repos.sh | 11 ++++++ scripts/deploy-dev-image.sh | 41 +++++++++++++++++++++ scripts/deploy-last-image.sh | 55 ++++++++++++++++++++++++++++ scripts/print-image-tag.sh | 27 ++++++++++++++ scripts/push-dev-image.sh | 24 +++++++++++++ scripts/push-prod-image.sh | 53 +++++++++++++++++++++++++++ scripts/tag-dev-image.sh | 31 ++++++++++++++++ scripts/tag-merge-commit.sh | 68 +++++++++++++++++++++++++++++++++++ scripts/test-image-name.sh | 41 +++++++++++++++++++++ 10 files changed, 399 insertions(+), 22 deletions(-) create mode 100644 scripts/delete-ecr-repos.sh create mode 100644 scripts/deploy-dev-image.sh create mode 100644 scripts/deploy-last-image.sh create mode 100644 scripts/print-image-tag.sh create mode 100644 scripts/push-dev-image.sh create mode 100644 scripts/push-prod-image.sh create mode 100644 scripts/tag-dev-image.sh create mode 100644 scripts/tag-merge-commit.sh create mode 100644 scripts/test-image-name.sh diff --git a/scripts/README.md b/scripts/README.md index 4e11c01f..c70c0025 100755 --- a/scripts/README.md +++ b/scripts/README.md @@ -33,14 +33,6 @@ ARTIFACTS_BUCKET_PREFIX=$(yq ".infrastructure.terraform.aws.modules["project-sto --- -##### `clean-artifact.sh` - -deletes `*.zip` files created in app directories - -##### `clean-binary.sh` - -deletes binaries created in app directories - ##### `clean-env.sh` deletes `.env` files created in app directories @@ -89,24 +81,12 @@ lists apps and libs in `project.yaml`, e.g. `services/request-create` prints list of environment variables set in `project.yaml` -##### `put-object.sh` - -puts artifact created in app directory in s3 bucket - ##### `set-codecov-flags.sh` sets custom [CODECOV_FLAGS](https://docs.codecov.com/docs/flags) github workflow environment variable to 1) app or package name, and 2) one standard codecov flag listed in `package.json` example: `CODECOV_FLAGS=tools,unittest` -##### `update-function.sh` - -deploys lambda function from s3 object created by `put-object.sh` - -##### `zip-executable.sh` - -adds binaries and shell scripts to `*.zip` files for deployment in app directories - ##### `insert-transactions.sh` adds a mix of requests and transactions in docker postgres (requires `cd migrations && make run`) @@ -252,7 +232,7 @@ print the value of an `env-var` in `project.yaml` send a http request to the internal `migrations/go-migrate` tool -##### `auth-ecr-repo.sh` +##### `auth-ecr.sh` authenticate with ecr @@ -270,4 +250,50 @@ imports resources into the `infrastructure/terraform/aws/environments/init-$ENV` ##### `rust-coverage.sh` -prints rust crate test coverage \ No newline at end of file +prints rust crate test coverage + +#### `print-lambda-policy.sh` + +prints policy attached to lambda function + +#### `print-ecr-repo-uri.sh` + +prints uri of ecr repo + +#### `print-image-tag.sh` + +prints service image tag with ecr repo uri and current git sha added as tag version + +#### `delete-ecr-repos.sh` + +convenience script to delete all dev ecr repos + +### `push-dev-image.sh` + +pushes local docker image to dev ecr repo (assumes local image already tagged) + +### `deploy-dev-image.sh` + +deploys "last" dev ecr image to lambda function. "latest" tag convention not used in ecr image tagging + +### `tag-merge-commit.sh` + +1. gets tag from image deployed to lambda +1. gets tag(s) from last image pushed to ecr (ecr images may have multiple tags) +1. tests if currently deployed lambda image tag matches any tag belonging to last imaged pushed to ecr +1. tags last ecr image with merge commit sha if last ecr image tag is NOT matched with currently deployed function image tag, OR +1. exits before tagging last ecr image tag with merge commit sha if last ecr image tag IS matched with currently deployed function image tag (avoids retagging) + +### `deploy-last-image.sh` + +1. gets tag from image deployed to lambda +1. gets tags from last image pushed to ecr (ecr images may have multiple tags) +1. tests currently deployed lambda image tag matches any tag belonging to last imaged pushed to ecr +1. deploys last ecr image if last ecr image tag is NOT matched with currently deployed function image tag, OR +1. exits before deploying if last ecr image tag IS matched with currently deployed function image tag (avoids redeploying) + +### `push-prod-image.sh` +used in integration test workflow after cloud integration tests pass +1. tests if current dev image tagged with merge commit +1. adds prod tag if current dev image tagged with merge commit, then pushes to prod ecr +1. exits if current dev image NOT tagged with merge commit (prod image not tagged and pushed) \ No newline at end of file diff --git a/scripts/delete-ecr-repos.sh b/scripts/delete-ecr-repos.sh new file mode 100644 index 00000000..fdf31028 --- /dev/null +++ b/scripts/delete-ecr-repos.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +PROJECT_CONF=project.yaml +ENV=dev +ENV_ID=$(source scripts/print-env-id.sh) +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) + +for APP_NAME in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do + IMAGE_NAME="$ENV_ID/$ENV/$APP_NAME" + aws ecr delete-repository --repository-name $IMAGE_NAME --region $REGION --force +done \ No newline at end of file diff --git a/scripts/deploy-dev-image.sh b/scripts/deploy-dev-image.sh new file mode 100644 index 00000000..c2023048 --- /dev/null +++ b/scripts/deploy-dev-image.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +if [[ "$#" -ne 2 ]]; then + cat <<- 'EOF' + use: + bash scripts/deploy-dev-image.sh --app-name go-migrate + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +ENV=dev +ENV_ID=$(source scripts/print-env-id.sh) +ID_ENV="$ENV_ID/$ENV" +REPO_NAME="$ID_ENV/$APP_NAME" + +REPO_URI=$(aws ecr describe-repositories \ + --query "repositories[?contains(repositoryUri, \`$REPO_NAME\`)].repositoryUri" \ + --output text \ + --region $REGION) + +LATEST_ECR_IMAGE_TAG_VERSIONS=($(aws ecr describe-images --repository-name $IMAGE_NAME --output text --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags' | xargs)) +LATEST_ECR_IMAGE_TAG_VERSION="${LATEST_ECR_IMAGE_TAG_VERSIONS[0]}" + +IMAGE_TAG=$REPO_URI:$LATEST_ECR_IMAGE_TAG_VERSION + +source scripts/update-function-image.sh \ + --app-name $APP_NAME \ + --curr-tag $IMAGE_TAG \ + --env dev \ No newline at end of file diff --git a/scripts/deploy-last-image.sh b/scripts/deploy-last-image.sh new file mode 100644 index 00000000..d9bff7fb --- /dev/null +++ b/scripts/deploy-last-image.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +if [[ "$#" -ne 6 ]]; then + cat <<- 'EOF' + use: + bash scripts/deploy-last-image.sh --app-name go-migrate --env dev --env-id 12345 + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --env) ENV="$2"; shift ;; + --env-id) ENV_ID="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +ID_ENV="$ENV_ID-$ENV" +ID_ENV_PREFIX="$ENV_ID/$ENV" +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) + +IMAGE_NAME="$ID_ENV_PREFIX/$APP_NAME" +LAMBDA_NAME="$APP_NAME-$ID_ENV" + +DEPLOYED_IMAGE=$(aws lambda get-function --function-name $LAMBDA_NAME --region $REGION --query 'Code.ImageUri' --output text) + +# test for image manifest sha tag +if [[ $(tr -dc '@' <<< "$DEPLOYED_IMAGE" | wc -c | xargs) -gt 0 ]]; then + IFS='@' read -ra DEPLOYED_IMAGE <<< "$DEPLOYED_IMAGE" + REPO_NAME=${DEPLOYED_IMAGE[0]} + DEPLOYED_IMAGE_TAG_VERSION=${DEPLOYED_IMAGE[1]} +else # assume git sha image tag + IFS=':' read -ra DEPLOYED_IMAGE <<< "$DEPLOYED_IMAGE" + REPO_NAME=${DEPLOYED_IMAGE[0]} + DEPLOYED_IMAGE_TAG_VERSION=${DEPLOYED_IMAGE[1]} +fi + +LATEST_ECR_IMAGE_TAG_VERSIONS=($(aws ecr describe-images --repository-name $IMAGE_NAME --output text --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags' | xargs)) + +for TAG_VERSION in "${LATEST_ECR_IMAGE_TAG_VERSIONS[@]}"; do + if [[ "$TAG_VERSION" == "$DEPLOYED_IMAGE_TAG_VERSION" ]]; then + echo "*** $LAMBDA_NAME has latest image tag $DEPLOYED_IMAGE_TAG_VERSION deployed. skipping deployment" + exit 0 + fi +done + +LATEST_ECR_IMAGE_TAG_VERSION="${LATEST_ECR_IMAGE_TAG_VERSIONS[0]}" + +LATEST_ECR_IMAGE="${REPO_NAME}:${LATEST_ECR_IMAGE_TAG_VERSION}" +LAST_MOD=$(aws lambda update-function-code --function-name $LAMBDA_NAME --image-uri $LATEST_ECR_IMAGE --region $REGION --query 'LastModified' --output text) +echo "*** $LATEST_ECR_IMAGE image deployed to lambda @ $LAST_MOD" \ No newline at end of file diff --git a/scripts/print-image-tag.sh b/scripts/print-image-tag.sh new file mode 100644 index 00000000..36f03b76 --- /dev/null +++ b/scripts/print-image-tag.sh @@ -0,0 +1,27 @@ +#/bin/bash + +if [[ "$#" -ne 6 ]] && [[ "$#" -ne 8 ]]; then + cat <<- 'EOF' + use: + bash scripts/print-image-tag.sh --app-name rule --env dev --env-id 12345 # OPTIONAL: --hash 12345678 + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --env) ENV="$2"; shift ;; + --env-id) ENV_ID="$2"; shift ;; + --hash) HASH="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [[ -z $HASH ]]; then + HASH=$(git rev-parse --short HEAD) +fi + +REPO=$(source scripts/print-ecr-repo-uri.sh --app-name $APP_NAME --env $ENV --env-id $ENV_ID) +echo "$REPO:$HASH" \ No newline at end of file diff --git a/scripts/push-dev-image.sh b/scripts/push-dev-image.sh new file mode 100644 index 00000000..a3f0f27c --- /dev/null +++ b/scripts/push-dev-image.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [[ "$#" -ne 2 ]]; then + cat <<- 'EOF' + use: + bash scripts/push-ecr-image.sh --app-name rule + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +ENV=dev +ENV_ID=$(source scripts/print-env-id.sh) +IMAGE_TAG=$(source scripts/print-image-tag.sh --app-name $APP_NAME --env $ENV --env-id $ENV_ID) + +source scripts/push-ecr-image.sh --curr-tag $IMAGE_TAG \ No newline at end of file diff --git a/scripts/push-prod-image.sh b/scripts/push-prod-image.sh new file mode 100644 index 00000000..01e66cb5 --- /dev/null +++ b/scripts/push-prod-image.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +if [[ "$#" -ne 6 ]]; then + cat <<- 'EOF' + use: + bash scripts/push-prod-image.sh --app-name rule --env dev --env-id 12345 + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --env) ENV="$2"; shift ;; + --env-id) ENV_ID="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +ID_ENV="$ENV_ID-$ENV" +ID_ENV_PREFIX="$ENV_ID/$ENV" +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +REPO_NAME="$ID_ENV_PREFIX/$APP_NAME" +PROD_ENV_ID=$(yq '.infrastructure.terraform.env-id.prod.env_var.set.PROD_ENV_ID.default' $PROJECT_CONF) + +MERGE_COMMIT_HASH=$(git rev-parse --short HEAD) + +TAG_COUNT=$(aws ecr batch-get-image \ + --repository-name=$REPO_NAME \ + --image-ids=imageTag=$MERGE_COMMIT_HASH \ + --region $REGION \ + --query 'length(images[*])' \ + --output text) + +if [[ $TAG_COUNT -gt 0 ]]; then + DEV_TAG=$(source scripts/print-ecr-repo-uri.sh --app-name $APP_NAME --env $ENV --env-id $ENV_ID):$MERGE_COMMIT_HASH + PROD_TAG=$(source scripts/print-ecr-repo-uri.sh --app-name $APP_NAME --env prod --env-id $PROD_ENV_ID):$MERGE_COMMIT_HASH + + # auth + source scripts/auth-ecr.sh + + # pull dev image + docker pull $DEV_TAG + + # tag dev image as prod image + docker tag $DEV_TAG $PROD_TAG + echo "\"$DEV_TAG\" tagged as \"$PROD_TAG\"" + + # push prod image + source scripts/push-ecr-image.sh --curr-tag $PROD_TAG +fi \ No newline at end of file diff --git a/scripts/tag-dev-image.sh b/scripts/tag-dev-image.sh new file mode 100644 index 00000000..5867186b --- /dev/null +++ b/scripts/tag-dev-image.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +if [[ "$#" -ne 2 ]]; then + cat <<- 'EOF' + use: + bash scripts/tag-dev-image.sh --app-name go-migrate + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +HASH=$(git rev-parse --short HEAD) +LOCAL_IMAGE_TAG="$APP_NAME:$HASH" + +ENV=dev +PROJECT_CONF=project.yaml +ENV_ID=$(source scripts/print-env-id.sh) + +DEV_REPO=$(source scripts/print-ecr-repo-uri.sh --app-name $APP_NAME --env dev --env-id $ENV_ID) +DEV_IMAGE_TAG="$DEV_REPO:$HASH" + +docker tag $LOCAL_IMAGE_TAG $DEV_IMAGE_TAG + +echo "tagged \"$LOCAL_IMAGE_TAG\" as \"$DEV_IMAGE_TAG\"" \ No newline at end of file diff --git a/scripts/tag-merge-commit.sh b/scripts/tag-merge-commit.sh new file mode 100644 index 00000000..acbe79aa --- /dev/null +++ b/scripts/tag-merge-commit.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +if [[ "$#" -ne 6 ]]; then + cat <<- 'EOF' + use: + bash scripts/tag-merge-commit.sh --app-name go-migrate --env dev --env-id 12345 + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --app-name) APP_NAME="$2"; shift ;; + --env) ENV="$2"; shift ;; + --env-id) ENV_ID="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +PROJECT_CONF=project.yaml +ID_ENV="$ENV_ID-$ENV" +ID_ENV_PREFIX="$ENV_ID/$ENV" +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) + +IMAGE_NAME="$ID_ENV_PREFIX/$APP_NAME" +LAMBDA_NAME="$APP_NAME-$ID_ENV" + +DEPLOYED_IMAGE=$(aws lambda get-function --function-name $LAMBDA_NAME --region $REGION --query 'Code.ImageUri' --output text) + +# test for image manifest sha tag +if [[ $(tr -dc '@' <<< "$DEPLOYED_IMAGE" | wc -c | xargs) -gt 0 ]]; then + IFS='@' read -ra DEPLOYED_IMAGE <<< "$DEPLOYED_IMAGE" + REPO_NAME=${DEPLOYED_IMAGE[0]} + DEPLOYED_IMAGE_TAG_VERSION=${DEPLOYED_IMAGE[1]} +else # assume git sha image tag + IFS=':' read -ra DEPLOYED_IMAGE <<< "$DEPLOYED_IMAGE" + REPO_NAME=${DEPLOYED_IMAGE[0]} + DEPLOYED_IMAGE_TAG_VERSION=${DEPLOYED_IMAGE[1]} +fi + +LATEST_ECR_IMAGE_TAG_VERSIONS=($(aws ecr describe-images --repository-name $IMAGE_NAME --output text --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags' | xargs)) + +# get the commit hash after merge +MERGE_COMMIT_HASH=$(git rev-parse --short HEAD) + +for TAG_VERSION in "${LATEST_ECR_IMAGE_TAG_VERSIONS[@]}"; do + if [[ "$TAG_VERSION" == "$DEPLOYED_IMAGE_TAG_VERSION" ]]; then + echo "*** $LAMBDA_NAME has latest image tag $DEPLOYED_IMAGE_TAG_VERSION deployed. skipping $MERGE_COMMIT_HASH retag" + exit 0 + fi +done + +# pick first image tag version +LATEST_ECR_IMAGE_TAG_VERSION="${LATEST_ECR_IMAGE_TAG_VERSIONS[0]}" + +# https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-retag.html +MANIFEST=$(aws ecr batch-get-image \ + --repository-name $IMAGE_NAME \ + --region $REGION \ + --image-ids imageTag=$LATEST_ECR_IMAGE_TAG_VERSION \ + --output text \ + --query 'images[].imageManifest') + +# retag image with merge commit hash +aws ecr put-image --repository-name $IMAGE_NAME --image-tag $MERGE_COMMIT_HASH --region $REGION --image-manifest "$MANIFEST" 1>/dev/null + +echo "*** $REPO_NAME image retagged with $MERGE_COMMIT_HASH merge commit hash" \ No newline at end of file diff --git a/scripts/test-image-name.sh b/scripts/test-image-name.sh new file mode 100644 index 00000000..0f21a2e6 --- /dev/null +++ b/scripts/test-image-name.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +if [[ "$#" -ne 2 ]]; then + cat <<- 'EOF' + use: + bash scripts/test-image-name.sh --curr-tag 123456789101.dkr.ecr.us-east-1.amazonaws.com/12345/dev/rule:93496996 + EOF + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + --curr-tag) CURR_TAG="$2"; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +IFS=':' read -r -a CURR_TAG_ARR <<< "$CURR_TAG" + +if [[ ${#CURR_TAG_ARR[@]} -ne 2 ]]; then + echo "tag must be in the format app-name:hash" + exit 1 +fi + +declare TESTED_APP_NAME +# parse app-name from image-repo +for d in $(source scripts/list-dir-paths.sh --type app | grep -v client); do + # remove directory path + APP_NAME=$(basename $d) + # test for substring match + if [[ $CURR_TAG == *$APP_NAME* ]]; then + TESTED_APP_NAME=$APP_NAME + fi +done + +if [[ -z $TESTED_APP_NAME ]]; then + echo "error: failed to match service name with $CURR_TAG" + source scripts/list-dir-paths.sh --type app | grep -v client | xargs basename + exit 1 +fi \ No newline at end of file From 0a70d0ba3c4539c4a900af0dc8876e3593ca609b Mon Sep 17 00:00:00 2001 From: max funk Date: Fri, 12 Apr 2024 18:47:29 -0700 Subject: [PATCH 48/54] checkout, deps and current makefile target spelling --- .github/workflows/auto-confirm.yaml | 1 + .github/workflows/balance-by-account.yaml | 14 +++++++++++++- .github/workflows/dev-integration.yaml | 11 +++++++++++ .github/workflows/graphql.yaml | 14 +++++++++++++- .github/workflows/request-approve.yaml | 14 +++++++++++++- .github/workflows/request-by-id.yaml | 14 +++++++++++++- .github/workflows/request-create.yaml | 14 +++++++++++++- .github/workflows/requests-by-account.yaml | 14 +++++++++++++- .github/workflows/rule.yaml | 15 ++++++++++++++- .github/workflows/transaction-by-id.yaml | 14 +++++++++++++- .github/workflows/transactions-by-account.yaml | 14 +++++++++++++- 11 files changed, 130 insertions(+), 9 deletions(-) diff --git a/.github/workflows/auto-confirm.yaml b/.github/workflows/auto-confirm.yaml index 7ec6d951..1d21a67e 100644 --- a/.github/workflows/auto-confirm.yaml +++ b/.github/workflows/auto-confirm.yaml @@ -42,6 +42,7 @@ jobs: APP_DIR: services/auto-confirm needs: [lint_test, unit_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml index 33fa1391..d89e50ca 100644 --- a/.github/workflows/balance-by-account.yaml +++ b/.github/workflows/balance-by-account.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/balance-by-account needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index b2c2d71b..67cae90c 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -27,6 +27,17 @@ jobs: AWS_DEFAULT_REGION: us-east-1 steps: - uses: actions/checkout@v4 + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: deploy to dev cloud environment run: | for app in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml index 112929d6..e4ba3871 100644 --- a/.github/workflows/graphql.yaml +++ b/.github/workflows/graphql.yaml @@ -43,6 +43,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -57,7 +68,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -70,6 +81,7 @@ jobs: APP_DIR: services/graphql needs: [lint_test, unit_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml index c9c22b5a..c7a7d6cb 100644 --- a/.github/workflows/request-approve.yaml +++ b/.github/workflows/request-approve.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/request-approve needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml index b9b287a7..0fcad2d8 100644 --- a/.github/workflows/request-by-id.yaml +++ b/.github/workflows/request-by-id.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/request-by-id needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml index f8273404..dc1d748b 100644 --- a/.github/workflows/request-create.yaml +++ b/.github/workflows/request-create.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/request-create needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml index 64aeccff..d69835cf 100644 --- a/.github/workflows/requests-by-account.yaml +++ b/.github/workflows/requests-by-account.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/requests-by-account needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml index 720b8549..478acdaf 100644 --- a/.github/workflows/rule.yaml +++ b/.github/workflows/rule.yaml @@ -36,6 +36,7 @@ jobs: - name: unit test run: cargo test - name: coverage report + uses: taiki-e/install-action@cargo-llvm-cov run: | make rust-coverage RUST_PKG=rule database_test: @@ -58,6 +59,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -72,7 +84,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -85,6 +97,7 @@ jobs: APP_DIR: services/rule needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml index a1e22aff..864fb718 100644 --- a/.github/workflows/transaction-by-id.yaml +++ b/.github/workflows/transaction-by-id.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/transaction-by-id needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml index 314f49a6..1b8c255f 100644 --- a/.github/workflows/transactions-by-account.yaml +++ b/.github/workflows/transactions-by-account.yaml @@ -55,6 +55,17 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb - name: start services run: make start - name: test service integration @@ -69,7 +80,7 @@ jobs: - name: start services run: make start - name: e2e test client - run: make -C ./client test-c + run: make -C ./client test-ci - name: clean up run: make stop push_image: @@ -82,6 +93,7 @@ jobs: APP_DIR: services/transactions-by-account needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: + - uses: actions/checkout@v4 - name: build image run: make -C $APP_DIR build-image - name: tag image From ce5b1229abeb1dc6fe6beef36e30c1782d17a4e1 Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 07:28:12 -0700 Subject: [PATCH 49/54] skip declaring lambda env var before testing locally --- crates/httpclient/src/lib.rs | 2 +- project.yaml | 2 +- scripts/create-env-file.sh | 8 ++---- tests/src/helpers.rs | 47 +++++++++++++++++------------------- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/crates/httpclient/src/lib.rs b/crates/httpclient/src/lib.rs index a9fe104a..6b09330f 100644 --- a/crates/httpclient/src/lib.rs +++ b/crates/httpclient/src/lib.rs @@ -24,7 +24,7 @@ impl HttpClient { .body(body) .unwrap(); - // sign request if running in lambda + // sign request if testing lambda if env::var("AWS_LAMBDA_FUNCTION_NAME").ok().is_some() { HttpClient::sign(&mut http_request).await; } diff --git a/project.yaml b/project.yaml index ccdc8a3e..f825f80d 100644 --- a/project.yaml +++ b/project.yaml @@ -560,7 +560,7 @@ tests: set: AWS_LAMBDA_FUNCTION_NAME: ssm: null - default: null # switched to non null by scripts/create-env-file.sh when testing lambdas + default: 1 get: - GRAPHQL_URI - RULE_URL diff --git a/scripts/create-env-file.sh b/scripts/create-env-file.sh index dad8fb01..d2d4af3a 100644 --- a/scripts/create-env-file.sh +++ b/scripts/create-env-file.sh @@ -95,6 +95,8 @@ function set_default_values() { else echo "$s=$HOST:$PORT_VAL" >> $ENV_FILE fi + elif [[ "$s" == 'AWS_LAMBDA_FUNCTION_NAME' ]]; then + continue # skip setting when ENV=local else ENV_VAR=$(yq "... | select(has(\"$s\")).$s.default" $PROJECT_CONF) if [[ $ENV_VAR == 'null' ]]; then @@ -119,12 +121,6 @@ function set_secrets() { else ENV_VAR=$(echo "$CONF_OBJ" | yq ".default") fi - # lambdas sign their requests when a function name is detected - # so this env var is set to 1 while integration testing from - # a local machine or workflow - if [[ $s == 'AWS_LAMBDA_FUNCTION_NAME' ]]; then - ENV_VAR=1 - fi echo $s=$ENV_VAR >> $ENV_FILE unset ENV_VAR done diff --git a/tests/src/helpers.rs b/tests/src/helpers.rs index adde68c9..1467e9c1 100644 --- a/tests/src/helpers.rs +++ b/tests/src/helpers.rs @@ -8,32 +8,29 @@ use serde_json::json; use std::{env, fs::File, io::BufReader, process::Command}; pub fn restore_testseed() { - if let Ok(val) = env::var("AWS_LAMBDA_FUNCTION_NAME") { - // nested condition assumes AWS_LAMBDA_FUNCTION_NAME always present but not always empty - if val.is_empty() { - let restore_output = Command::new("make") - .arg("-C") - .arg("../migrations/dumps") - .arg("restore-testseed") - .output() - .expect("failed to execute process"); + if env::var("AWS_LAMBDA_FUNCTION_NAME").ok().is_some() { + let restore_output = Command::new("make") + .arg("-C") + .arg("../migrations/dumps") + .arg("restore-rds-testseed") + .arg("ENV=dev") // cadet todo: assigned ENV from env var + .output() + .expect("failed to execute process"); - // cargo test -- --show-output - let _restore_output_str = String::from_utf8(restore_output.stdout).expect("Not UTF8"); - // println!("{}", _restore_output_str); // comment in to print db restore output - } else { - let restore_output = Command::new("make") - .arg("-C") - .arg("../migrations/dumps") - .arg("restore-rds-testseed") - .arg("ENV=dev") // cadet todo: assigned ENV from env var - .output() - .expect("failed to execute process"); + // cargo test -- --show-output + let restore_output_str = String::from_utf8(restore_output.stdout).expect("Not UTF8"); + println!("{}", restore_output_str); + } else { + let restore_output = Command::new("make") + .arg("-C") + .arg("../migrations/dumps") + .arg("restore-testseed") + .output() + .expect("failed to execute process"); - // cargo test -- --show-output - let restore_output_str = String::from_utf8(restore_output.stdout).expect("Not UTF8"); - println!("{}", restore_output_str); - } + // cargo test -- --show-output + let _restore_output_str = String::from_utf8(restore_output.stdout).expect("Not UTF8"); + // println!("{}", _restore_output_str); // comment in to print db restore output } } @@ -72,4 +69,4 @@ pub async fn create_transaction() -> Transaction { let approve_request: IntraTransaction = serde_json::from_str(&approve_request_response_body).unwrap(); approve_request.transaction -} +} \ No newline at end of file From 3b7e10dd4e80afe3ed843a60dd6e4ff767d776f5 Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 07:28:54 -0700 Subject: [PATCH 50/54] set dev env id when tagging and pushing images --- .github/workflows/auto-confirm.yaml | 4 ++-- .github/workflows/balance-by-account.yaml | 4 ++-- .github/workflows/graphql.yaml | 4 ++-- .github/workflows/request-approve.yaml | 4 ++-- .github/workflows/request-by-id.yaml | 4 ++-- .github/workflows/request-create.yaml | 4 ++-- .github/workflows/requests-by-account.yaml | 4 ++-- .github/workflows/rule.yaml | 4 ++-- .github/workflows/transaction-by-id.yaml | 4 ++-- .github/workflows/transactions-by-account.yaml | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/auto-confirm.yaml b/.github/workflows/auto-confirm.yaml index 1d21a67e..c813b516 100644 --- a/.github/workflows/auto-confirm.yaml +++ b/.github/workflows/auto-confirm.yaml @@ -46,6 +46,6 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image \ No newline at end of file + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml index d89e50ca..8db2aee1 100644 --- a/.github/workflows/balance-by-account.yaml +++ b/.github/workflows/balance-by-account.yaml @@ -97,6 +97,6 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image \ No newline at end of file + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml index e4ba3871..5f254e67 100644 --- a/.github/workflows/graphql.yaml +++ b/.github/workflows/graphql.yaml @@ -85,7 +85,7 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml index c7a7d6cb..b82cc1c9 100644 --- a/.github/workflows/request-approve.yaml +++ b/.github/workflows/request-approve.yaml @@ -97,7 +97,7 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml index 0fcad2d8..3eb6b6cf 100644 --- a/.github/workflows/request-by-id.yaml +++ b/.github/workflows/request-by-id.yaml @@ -97,7 +97,7 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml index dc1d748b..7a330075 100644 --- a/.github/workflows/request-create.yaml +++ b/.github/workflows/request-create.yaml @@ -97,7 +97,7 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml index d69835cf..4e6c241d 100644 --- a/.github/workflows/requests-by-account.yaml +++ b/.github/workflows/requests-by-account.yaml @@ -97,7 +97,7 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml index 478acdaf..adb1de4d 100644 --- a/.github/workflows/rule.yaml +++ b/.github/workflows/rule.yaml @@ -101,6 +101,6 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image \ No newline at end of file + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml index 864fb718..fc0be3d2 100644 --- a/.github/workflows/transaction-by-id.yaml +++ b/.github/workflows/transaction-by-id.yaml @@ -97,6 +97,6 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image \ No newline at end of file + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image \ No newline at end of file diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml index 1b8c255f..b87157b5 100644 --- a/.github/workflows/transactions-by-account.yaml +++ b/.github/workflows/transactions-by-account.yaml @@ -97,6 +97,6 @@ jobs: - name: build image run: make -C $APP_DIR build-image - name: tag image - run: make -C $APP_DIR tag-dev-image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - name: push image - run: make -C $APP_DIR push-dev-image \ No newline at end of file + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image \ No newline at end of file From 20c7f2ba844cebf3500debadd79b0be11d064a83 Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 07:30:05 -0700 Subject: [PATCH 51/54] lint --- tests/src/helpers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/helpers.rs b/tests/src/helpers.rs index 1467e9c1..e2a50cfa 100644 --- a/tests/src/helpers.rs +++ b/tests/src/helpers.rs @@ -69,4 +69,4 @@ pub async fn create_transaction() -> Transaction { let approve_request: IntraTransaction = serde_json::from_str(&approve_request_response_body).unwrap(); approve_request.transaction -} \ No newline at end of file +} From 110b3b281687df08195499948052fbfd8d7ab60f Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 07:41:27 -0700 Subject: [PATCH 52/54] start workflows --- crates/httpclient/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/httpclient/src/lib.rs b/crates/httpclient/src/lib.rs index 6b09330f..4ae120ad 100644 --- a/crates/httpclient/src/lib.rs +++ b/crates/httpclient/src/lib.rs @@ -24,7 +24,7 @@ impl HttpClient { .body(body) .unwrap(); - // sign request if testing lambda + // sign request when testing lambda if env::var("AWS_LAMBDA_FUNCTION_NAME").ok().is_some() { HttpClient::sign(&mut http_request).await; } From 365d46d73694ae3bc8514490d108a6c66678961b Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 07:42:07 -0700 Subject: [PATCH 53/54] test integration locally after test changes --- .github/workflows/dev-integration.yaml | 2 +- .github/workflows/local-integration.yaml | 37 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/local-integration.yaml diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index 67cae90c..6a5d7d06 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -1,4 +1,4 @@ -name: integration +name: dev-integration on: push: diff --git a/.github/workflows/local-integration.yaml b/.github/workflows/local-integration.yaml new file mode 100644 index 00000000..13f50450 --- /dev/null +++ b/.github/workflows/local-integration.yaml @@ -0,0 +1,37 @@ +name: local-integration + +on: + push: + paths: + - 'tests/**' + branches-ignore: + - 'master' + - 'develop' + +jobs: + integration_test: + name: integration test in local docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: install latest psql client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install --yes --no-install-recommends postgresql-client + - name: install golang-migrate + run: | + curl -LO https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-amd64.deb + sudo dpkg -i migrate.linux-amd64.deb + rm migrate.linux-amd64.deb + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop \ No newline at end of file From 2cf9d9c3a31ea639b2d913c14e1569b0dbfc8a93 Mon Sep 17 00:00:00 2001 From: max funk Date: Sat, 13 Apr 2024 08:06:21 -0700 Subject: [PATCH 54/54] mask workflow values --- .github/workflows/auto-confirm.yaml | 2 ++ .github/workflows/balance-by-account.yaml | 2 ++ .github/workflows/dev-integration.yaml | 2 ++ .github/workflows/graphql.yaml | 2 ++ .github/workflows/prod-services-deploy.yaml | 2 ++ .github/workflows/request-approve.yaml | 2 ++ .github/workflows/request-by-id.yaml | 2 ++ .github/workflows/request-create.yaml | 2 ++ .github/workflows/requests-by-account.yaml | 2 ++ .github/workflows/rule.yaml | 2 ++ .github/workflows/transaction-by-id.yaml | 2 ++ .github/workflows/transactions-by-account.yaml | 2 ++ crates/types/README.md | 7 +++++++ 13 files changed, 31 insertions(+) create mode 100644 crates/types/README.md diff --git a/.github/workflows/auto-confirm.yaml b/.github/workflows/auto-confirm.yaml index c813b516..7e752011 100644 --- a/.github/workflows/auto-confirm.yaml +++ b/.github/workflows/auto-confirm.yaml @@ -43,6 +43,8 @@ jobs: needs: [lint_test, unit_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml index 8db2aee1..293e9069 100644 --- a/.github/workflows/balance-by-account.yaml +++ b/.github/workflows/balance-by-account.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index 6a5d7d06..fda00881 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -27,6 +27,8 @@ jobs: AWS_DEFAULT_REGION: us-east-1 steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: install latest psql client run: | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml index 5f254e67..c5c6300c 100644 --- a/.github/workflows/graphql.yaml +++ b/.github/workflows/graphql.yaml @@ -82,6 +82,8 @@ jobs: needs: [lint_test, unit_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/prod-services-deploy.yaml b/.github/workflows/prod-services-deploy.yaml index c4ff7b50..c4a6504f 100644 --- a/.github/workflows/prod-services-deploy.yaml +++ b/.github/workflows/prod-services-deploy.yaml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: deploy services to prod cloud environment run: | for app in $(echo $(bash scripts/list-dir-paths.sh --type app | grep -v client) | xargs basename); do diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml index b82cc1c9..5f6d798f 100644 --- a/.github/workflows/request-approve.yaml +++ b/.github/workflows/request-approve.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml index 3eb6b6cf..481ae592 100644 --- a/.github/workflows/request-by-id.yaml +++ b/.github/workflows/request-by-id.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml index 7a330075..e66e44a6 100644 --- a/.github/workflows/request-create.yaml +++ b/.github/workflows/request-create.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml index 4e6c241d..d0100135 100644 --- a/.github/workflows/requests-by-account.yaml +++ b/.github/workflows/requests-by-account.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml index adb1de4d..e97d005f 100644 --- a/.github/workflows/rule.yaml +++ b/.github/workflows/rule.yaml @@ -98,6 +98,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml index fc0be3d2..ee66356f 100644 --- a/.github/workflows/transaction-by-id.yaml +++ b/.github/workflows/transaction-by-id.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml index b87157b5..9d14d5a8 100644 --- a/.github/workflows/transactions-by-account.yaml +++ b/.github/workflows/transactions-by-account.yaml @@ -94,6 +94,8 @@ jobs: needs: [lint_test, unit_test, database_test, integration_test, client_test] steps: - uses: actions/checkout@v4 + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - name: build image run: make -C $APP_DIR build-image - name: tag image diff --git a/crates/types/README.md b/crates/types/README.md new file mode 100644 index 00000000..309b6a83 --- /dev/null +++ b/crates/types/README.md @@ -0,0 +1,7 @@ +

+ systemaccounting +

+ +### crates/types + +types for rust apps \ No newline at end of file