feat: add ArangoDB

(Optionally) adds Arango to Data Workspace

Includes Arango ECS service connecting to the main ECS cluster.  ArangoDB
instance run using EC2 to allow for Elastic Block Storage to attach to ArangoDB
instances when spun up (auto-scaling group). There is a load balancer to
connect the Data workspace tools to the ArangoDB instance.

The EBS volume mounts to /data directory within the instance, and the ArangoDB
container task definition mounts the instance's /data directory to
/var/lib/arangodb3 directory within the container (as this is where data is
stored in an Arango database).

As the Arango EC2 instance is in a private subnet and the Datasets VPC does not
have access to the internet, VPC Endpoints were used too allow the EC2
insrtance to connect to ECS. The endpoints were placed in the Datasets VPC
(which did require DNS hostnames to be enabled) and policies were created and
attach to the endpoints to restrict access to only the Arango instance.

Co-authored-by: Isobel Daley <[email protected]>
Co-authored-by: Zayn-6point6 <[email protected]>
Co-authored-by: Michal Charemza <[email protected]>
Co-authored-by: Ares Galamatis <[email protected]>
Co-authored-by: Peter Woodcock <[email protected]>
Co-authored-by: Mohizur Khan <[email protected]>

infra/ecr.tf

@@ -98,6 +98,11 @@ resource "aws_ecr_repository" "mlflow" {
   name = "${var.prefix}-mlflow"
 }
 
+resource "aws_ecr_repository" "arango" {
+  count = var.arango_on ? 1 : 0
+  name  = "${var.prefix}-arango"
+}
+
 resource "aws_vpc_endpoint" "ecr_dkr" {
   vpc_id              = aws_vpc.main.id
   service_name        = "com.amazonaws.${data.aws_region.aws_region.name}.ecr.dkr"

infra/ecs_main_admin.tf

@@ -1,5 +1,5 @@
 locals {
-  admin_container_vars = {
+  admin_container_vars = merge({
     container_image = "${aws_ecr_repository.admin.repository_url}:${data.external.admin_current_tag.result.tag}"
     container_name  = "${local.admin_container_name}"
     container_port  = "${local.admin_container_port}"
@@ -98,7 +98,16 @@ locals {
 
     jwt_private_key = "${var.jwt_private_key}"
     mlflow_port     = "${local.mlflow_port}"
-  }
+    }, var.arango_on ? {
+    arango_db__host     = "${aws_lb.arango[0].dns_name}"
+    arango_db__password = "${random_string.aws_arangodb_root_password[0].result}"
+    arango_db__port     = "${local.arango_container_port}"
+    } : {
+    arango_db__host     = ""
+    arango_db__password = ""
+    arango_db__port     = ""
+    }
+  )
 }
 
 resource "aws_ecs_service" "admin" {

infra/ecs_main_admin_container_definitions.json

@@ -55,6 +55,22 @@
       "name": "EXPLORER_DEFAULT_CONNECTION",
       "value": "datasets_1"
     },
+    {
+      "name": "ARANGO_DB__HOST",
+      "value": "${arango_db__host}"
+    },
+    {
+      "name": "ARANGO_DB__PORT",
+      "value": "${arango_db__port}"
+    },
+    {
+      "name": "ARANGO_DB__USER",
+      "value": "root"
+    },
+    {
+      "name": "ARANGO_DB__PASSWORD",
+      "value": "${arango_db__password}"
+    },
     {
       "name": "ALLOWED_HOSTS__1",
       "value": "${root_domain}"