CFE-4322: Changed how cf-remote spawn finds AMIs with AWS

Before we hard-coded AMI and had to update them fairly often and add them when we supported a new platform.

Here we change to querying for AMI based on known trusted owner IDs.

This should make it so that new platforms are automatically available and any updates will be in-place automatically as well.

This change should also enable users to specify other regions and have most tplatforms "just work".

One note: specific Windows AMI are only available in eu-west-1

Ticket: CFE-4322
Changelog: title

README.md

@@ -209,3 +209,9 @@ To install `cf-remote` so that it reflects any changes in this source directory
 ```
 $ pip install --editable .
 ```
+
+## cloud_data.py tips
+
+In order to find AWS images for a particular owner to work on cloud_data.py name_pattern list the names for an owner with the following `aws` command:
+
+aws ec2 describe-images --region us-east-2 --owners 801119661308 --query 'Images[*].[Name]' --output text

cf_remote/cloud_data.py

@@ -1,179 +1,74 @@
-aws_platforms = {
-    "ubuntu-22-04-arm64": {
-        "ami": "ami-00c50882a52d323a6",
-        "user": "ubuntu",
-        "size": "t4g.micro",
-        "xlsize": "t4g.xlarge",
-    },
-    "ubuntu-22-04-x64": {
-        "ami": "ami-01dd271720c1ba44f",
-        "user": "ubuntu",
-        "size": "t2.small",
-        "xlsize": "t3.xlarge",
-    },
-    "ubuntu-20-04-x64": {
-        "ami": "ami-0aef57767f5404a3c",
-        "user": "ubuntu",
-        "size": "t2.small",
-        "xlsize": "t3.xlarge",
-    },
-    "ubuntu-18-04-x64": {
-        "ami": "ami-0ee3436f275c4f2e8",
-        "user": "ubuntu",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
-    },
-    "ubuntu-14-04-x32": {
-        "ami": "ami-07a1e6256cb43b99c",
-        "user": "ubuntu",
-        "size": "m1.small",
-    },
-    "debian-8-x64": {
-        "ami": "ami-402f1a33",
-        "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "debian-7-x64": {
-        "ami": "ami-61e56916",
-        "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "debian-9-x64": {
-        "ami": "ami-035c67e6a9ef8f024",
-        "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "debian-10-x64": {
-        "ami": "ami-0a9d04ba7d4df6c3b",
-        "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "debian-11-arm64": {
-        "ami": "ami-0353cb95279bf4f20",
-        "user": "admin",
-        "size": "t4g.micro",
-        "xlsize": "t4g.xlarge",
-    },
-    "debian-11-x64": {
-        "ami": "ami-0293236c9a0c23a77",
-        "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "debian-12-arm64": {
-        "ami": "ami-03820227fb3e4ffad",
+aws_defaults = {
+    "architecture": "x86_64",
+    "sizes": {
+        "x86_64": {
+            "size": "t2.micro",
+            "xlsize": "t2.xlarge",
+        },
+        "arm64": {
+            "size": "t4g.micro",
+            "xlsize": "t4g.xlarge",
+        },
+    },
+    "user": "ec2-user",
+}
+aws_image_criteria = {
+    "debian-9": {
+        "owner_id": "379101102735",
+        "name_pattern": "debian-stretch-hvm-x86_64*",
         "user": "admin",
-        "size": "t4g.micro",
-        "xlsize": "t4g.xlarge",
     },
-    "debian-12-x64": {
-        "ami": "ami-07024fbdfd1aab8a0",
+    "debian": {
+        "owner_id": "136693071363",
+        "name_pattern": "debian-{version}*",
         "user": "admin",
-        "size": "t1.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "centos-6-x64": {
-        "ami": "ami-05bd23226cb7c2896",
-        "user": "centos",
-        "size": "t2.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "centos-7-x64": {
-        "ami": "ami-0f4775c518fa29365",
-        "user": "centos",
-        "size": "t2.micro",
-        "xlsize": "m3.xlarge",
-    },
-    "rhel-5-x64": {
-        "ami": "ami-ea94369d",
-        "size": "t1.micro",
-        "user": "root",
-        "xlsize": "t1.micro",
-    },
-    "rhel-6-x64": {
-        "ami": "ami-c1bb06b2",
-        "size": "t2.micro",
-        "user": "ec2-user",
-        "xlsize": "t2.large",
     },
-    "rhel-7-x64": {
-        "ami": "ami-065ec1e661d619058",
-        "size": "t2.micro",
-        "user": "ec2-user",
-        "xlsize": "t2.large",
-    },
-    "rhel-8-x64": {
-        "ami": "ami-08f4717d06813bf00",
-        "size": "t3a.micro",
-        "user": "ec2-user",
-        "xlsize": "m3.xlarge",
-    },
-    "rhel-9-x64": {
-        "ami": "ami-049b0abf844cab8d7",
-        "size": "t3a.micro",
-        "user": "ec2-user",
-        "xlsize": "m3.xlarge"
-   },
-    "centos-5-x32": {"ami": "ami-fe11398a", "user": "root", "size": "m1.small"},
-    "debian-6-x64": {"ami": "ami-879e4ff0", "user": "admin", "size": "t1.micro"},
-    "debian-5-x32": {"ami": "ami-8398b3f7", "user": "root", "size": "m1.small"},
-    "debian-7-x32": {"ami": "ami-1be06c6c", "user": "admin", "size": "t1.micro"},
-    "debian-4-x32": {"ami": "ami-8198b3f5", "user": "root", "size": "m1.small"},
-    "ubuntu-12-04-x64": {
-        "ami": "ami-d1767bb7",
+    "ubuntu-16": {
+        "owner_id": "099720109477",
+        "name_pattern": "ubuntu-pro-server/images/hvm-ssd/ubuntu-xenial-16.04-amd64-pro-server*",
         "user": "ubuntu",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
     },
-    "debian-6-x32": {"ami": "ami-8d9e4ffa", "user": "admin", "size": "t1.micro"},
-    "ubuntu-16-04-x64": {
-        "ami": "ami-0d47c52ffe8fef155",
+    "ubuntu": {
+        "owner_id": "099720109477",
+        "name_pattern": "ubuntu/images/hvm-ssd/ubuntu-*-{version}*",
         "user": "ubuntu",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
     },
-    "debian-5-x64": {"ami": "ami-8f98b3fb", "user": "root", "size": "m1.small"},
-    "debian-4-x64": {"ami": "ami-8d98b3f9", "user": "root", "size": "m1.small"},
-    "ubuntu-14-04-x64": {
-        "ami": "ami-0c68b4b8bbbdc39de",
-        "user": "ubuntu",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
+    "centos": {
+        "meta": "This owner is our nt-dev account in AWS so these are private custom images.",
+        "owner_id": "304194462000",
+        "name_pattern": "centos-{version}-x64",
+        "region": "eu-west-1",
+    },
+    "rhel": {
+        "owner_id": "309956199498",
+        "name_pattern": "RHEL-{version}*",
     },
-    "ubuntu-12-04-x32": {"ami": "ami-5c78753a", "user": "ubuntu", "size": "m1.small"},
-    "centos-5-x64": {"ami": "ami-f2113986", "user": "root", "size": "m1.small"},
-    "windows-2012-x64": {
-        "ami": "ami-045768fc2ae3fa829",
+    "windows-2008": {
+        "ami": "ami-09046e654c804633f",
         "user": "Administrator",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
+        "region": "eu-west-1",
     },
-    "windows-2016-x64": {
-        "ami": "ami-08f68fefe026532ea",
+    "windows-2012": {
+        "ami": "ami-0444b0c023c7f3671",
         "user": "Administrator",
-        "size": "m1.small",
-        "xlsize": "m3.xlarge",
+        "region": "eu-west-1",
     },
-    "windows-2019-x64": {
-        "ami": "ami-0311c2819c6a29312",
+    "windows-2016": {
+        "ami": "ami-00a7e5468b339302c",
         "user": "Administrator",
-        "size": "t2.small",
-        "xlsize": "t2.xlarge",
+        "region": "eu-west-1",
     },
-    "suse-12-x64": {
-        "ami": "ami-0d5622d69a166848b",
-        "user": "ec2-user",
-        "size": "t2.small",
-        "xlsize": "t2.xlarge",
+    "windows-2019": {
+        "ami": "ami-0311c2819c6a29312",
+        "user": "Administrator",
+        "region": "eu-west-1",
     },
-    "suse-15-x64": {
-        "ami": "ami-0e5e442298b8e7f5a",
-        "user": "ec2-user",
-        "size": "t2.small",
-        "xlsize": "t2.xlarge",
+    "windows": {
+        "meta": "Note that typically we rely on custom pre-configured windows imimages with ssh installed and pre-populated public keys so an image spawned from this criteria will not come with ssh built-in and ready to go.",
+        "owner_id": "801119661308",
+        "name_pattern": "Windows_Server-{version}-English-Core-Base*",
+        "user": "Administrator",
     },
+    "suse": {"owner_id": "013907871322", "name_pattern": "suse-sles-{version}*"},
+    "macos": {"owner_id": "634519214787", "name_pattern": "amxn-ec2-macos-{version}*"},
 }

cf_remote/commands.py

@@ -556,8 +556,8 @@ def destroy(group_name=None):
 
 
 def list_platforms():
-    print("Available platforms:")
-    for key in sorted(cloud_data.aws_platforms.keys()):
+    print("Available platforms (image criteria):")
+    for key in sorted(cloud_data.aws_image_criteria.keys()):
         print(key)
     return 0