Netflix-Skunkworks · patricksanders · Aug 26, 2020 · Sep 10, 2020 · Sep 11, 2020 · Sep 11, 2020
diff --git a/.dockerignore b/.dockerignore
@@ -3,3 +3,5 @@
 ./.travis.yml
 ./.env
 ./docker-compose.yml
+*.log
+*.db
diff --git a/.gitignore b/.gitignore
@@ -106,4 +106,5 @@ ENV/
 .mypy_cache/
 
 # config
-config.py
+.secrets.yaml
+settings.local.yaml.bak
diff --git a/.travis.yml b/.travis.yml
@@ -1,29 +1,39 @@
 language: python
 cache: pip
 python:
-- 3.7
+- 3.8
 matrix:
   allow_failures:
   - python: nightly
   - python: pypy
   - python: pypy3
 install:
 - pip install -r requirements.txt
-- python setup.py develop
-- pip install flake8
+- pip install -r requirements-test.txt
+- pip install -e .
 before_script:
 - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
 - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 script:
-- pytest --capture=sys --ignore=test/test_docker.py
+- pytest
 deploy:
-  provider: pypi
-  user: __token__
-  password:
-    secure: A+HimWMUsL9T4hN4mgHl3JPJ+/61Zd24OhZ1piDUZfp8GtfCSAd1UmxjyLSMl5+8CaALOYcndf3WWkGSLFz/DBi0Z0M+4YNwX+JfJzHXPWCcJ0lGG89VHi+TRcAEzaT95bhHGkh8OTq3aKqHlY9t6phdwb1S6zwNuvtv1FtJvBIVEsaQ9X7toOcY//7+VegmwPxUE07tP606aKLUXxNDlLiqAIRiEnRIHUB1yWzhY77k2CYEV31P4PvdAjy5wLJzd4CHpmQmZgzY7DgRTIq1AecjsCDR34O+erjvrgx31itpE9GpmPfz76ExhSx5MSYyFoE347FnsJnZXu1n2XgsvYUfx0rrQBX4ti0ozSWe7v+B96YFfr+m84AUJBDB4lFIxeNYxjhc5+9BH70WQkosOLdW/n0GMtcAvYQXo7wyLm7vqOkVgN5dlk1J29Z3F3GXJxNGo7/gytOK/m3UO3wJk8VqNeIO2CV7uRvQ7DwYoar9kyx88wN586nKahGQLdsMKcULujU3ofF0BHoylb0oKRzCf0A9fPlcbpk1M8RJv7VyDBnWlirqou8uflfs2P1tGCjgrCDtVW3U7ER4LBNvbO0wPT6/o2arcaF99Np3mIMIxNh1VUhDhGouUAfgTvKh+sZW/t9u+ZI6kloZ64UhDHODkAUwAfwledcF6GDVzOc=
-  skip_existing: true
-  on:
-    tags: true
+  - provider: pypi
+    user: __token__
+    password:
+      secure: A+HimWMUsL9T4hN4mgHl3JPJ+/61Zd24OhZ1piDUZfp8GtfCSAd1UmxjyLSMl5+8CaALOYcndf3WWkGSLFz/DBi0Z0M+4YNwX+JfJzHXPWCcJ0lGG89VHi+TRcAEzaT95bhHGkh8OTq3aKqHlY9t6phdwb1S6zwNuvtv1FtJvBIVEsaQ9X7toOcY//7+VegmwPxUE07tP606aKLUXxNDlLiqAIRiEnRIHUB1yWzhY77k2CYEV31P4PvdAjy5wLJzd4CHpmQmZgzY7DgRTIq1AecjsCDR34O+erjvrgx31itpE9GpmPfz76ExhSx5MSYyFoE347FnsJnZXu1n2XgsvYUfx0rrQBX4ti0ozSWe7v+B96YFfr+m84AUJBDB4lFIxeNYxjhc5+9BH70WQkosOLdW/n0GMtcAvYQXo7wyLm7vqOkVgN5dlk1J29Z3F3GXJxNGo7/gytOK/m3UO3wJk8VqNeIO2CV7uRvQ7DwYoar9kyx88wN586nKahGQLdsMKcULujU3ofF0BHoylb0oKRzCf0A9fPlcbpk1M8RJv7VyDBnWlirqou8uflfs2P1tGCjgrCDtVW3U7ER4LBNvbO0wPT6/o2arcaF99Np3mIMIxNh1VUhDhGouUAfgTvKh+sZW/t9u+ZI6kloZ64UhDHODkAUwAfwledcF6GDVzOc=
+    skip_existing: true
+    on:
+      tags: true
+  - provider: pypi
+    user: __token__
+    password:
+      secure: A+HimWMUsL9T4hN4mgHl3JPJ+/61Zd24OhZ1piDUZfp8GtfCSAd1UmxjyLSMl5+8CaALOYcndf3WWkGSLFz/DBi0Z0M+4YNwX+JfJzHXPWCcJ0lGG89VHi+TRcAEzaT95bhHGkh8OTq3aKqHlY9t6phdwb1S6zwNuvtv1FtJvBIVEsaQ9X7toOcY//7+VegmwPxUE07tP606aKLUXxNDlLiqAIRiEnRIHUB1yWzhY77k2CYEV31P4PvdAjy5wLJzd4CHpmQmZgzY7DgRTIq1AecjsCDR34O+erjvrgx31itpE9GpmPfz76ExhSx5MSYyFoE347FnsJnZXu1n2XgsvYUfx0rrQBX4ti0ozSWe7v+B96YFfr+m84AUJBDB4lFIxeNYxjhc5+9BH70WQkosOLdW/n0GMtcAvYQXo7wyLm7vqOkVgN5dlk1J29Z3F3GXJxNGo7/gytOK/m3UO3wJk8VqNeIO2CV7uRvQ7DwYoar9kyx88wN586nKahGQLdsMKcULujU3ofF0BHoylb0oKRzCf0A9fPlcbpk1M8RJv7VyDBnWlirqou8uflfs2P1tGCjgrCDtVW3U7ER4LBNvbO0wPT6/o2arcaF99Np3mIMIxNh1VUhDhGouUAfgTvKh+sZW/t9u+ZI6kloZ64UhDHODkAUwAfwledcF6GDVzOc=
+    skip_existing: true
+    on:
+      branch: develop
 notifications:
-  on_success: change
-  on_failure: change
+  email:
+    recipients:
+      - [email protected]
+    on_success: change
+    on_failure: change
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.8
 
 RUN apt-get update -y \
     && apt-get upgrade -y \
@@ -15,10 +15,12 @@ WORKDIR /etc/aardvark
 ENV AARDVARK_DATA_DIR=/data \
     AARDVARK_ROLE=Aardvark \
     ARN_PARTITION=aws \
-    AWS_DEFAULT_REGION=us-east-1
+    AWS_DEFAULT_REGION=us-east-1 \
+    FLASK_APP=aardvark
 
 EXPOSE 5000
 
+COPY ./settings.yaml .
 COPY ./entrypoint.sh /etc/aardvark/entrypoint.sh
 
 ENTRYPOINT [ "/etc/aardvark/entrypoint.sh" ]

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,3 @@
-include setup.py README.md MANIFEST.in LICENSE
+include setup.py README.md MANIFEST.in LICENSE aardvark/config_default.yaml
 recursive-include aardvark *.js
 global-exclude *~
diff --git a/README.md b/README.md
@@ -3,13 +3,32 @@ Aardvark
 [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/osstracker.svg)]()
 [![Discord chat](https://img.shields.io/discord/754080763070382130?logo=discord)](https://discord.gg/9kwMWa6)
 
-<img align="center" alt="Aardvark Logo" src="docs/images/aardvark_logo.jpg" width="10%" display="block">
+![Aardvark Logo](docs/images/aardvark_logo_small.png)
 
 Aardvark is a multi-account AWS IAM Access Advisor API (and caching layer).
 
-## Install:
+## New in `v1.0.0`
 
-Ensure that you have Python 3.6 or later. Python 2 is no longer supported.
+⚠️ Breaking change 
+
+✨ Enhancement
+
+- ⚠️ Upgrade to Python 3.8+
+- ⚠️ New configuration format
+- ✨ Pluggable persistence layer
+- ✨ Pluggable retrievers
+
+## Install
+
+Ensure that you have Python 3.8 or later.
+
+Use pip install Aardvark:
+
+```bash
+pip install aardvark
+```
+
+Alternatively, clone the repository and install a development version:
 
 ```bash
 git clone https://github.com/Netflix-Skunkworks/aardvark.git
@@ -19,58 +38,125 @@ python3 -m venv env
 python setup.py develop
 ```
 
-### Known Dependencies
- - libpq-dev
-
 ## Configure Aardvark
 
 The Aardvark config wizard will guide you through the setup.
-```
-% aardvark config
+```bash
+❯ aardvark config
 
-Aardvark can use SWAG to look up accounts. https://github.com/Netflix-Skunkworks/swag-client
-Do you use SWAG to track accounts? [yN]: no
-ROLENAME: Aardvark
-DATABASE [sqlite:////home/github/aardvark/aardvark.db]:
-# Threads [5]:
+Aardvark can use SWAG to look up accounts. See https://github.com/Netflix-Skunkworks/swag-client
+Do you use SWAG to track accounts? [yN]: N
+Role Name [Aardvark]: Aardvark
+Database URI [sqlite:///aardvark.db]: 
+Worker Count [5]: 5
+Config file location [settings.yaml]: settings.local.yaml
 
->> Writing to config.py
+writing config file to settings.local.yaml
 ```
 - Whether to use [SWAG](https://github.com/Netflix-Skunkworks/swag-client) to enumerate your AWS accounts. (Optional, but useful when you have many accounts.)
 - The name of the IAM Role to assume into in each account.
 - The Database connection string. (Defaults to sqlite in the current working directory. Use RDS Postgres for production.)
+- The number of workers to create.
 
 ## Create the DB tables
 
-```
+```bash
 aardvark create_db
 ```
 
 ## IAM Permissions:
 
 Aardvark needs an IAM Role in each account that will be queried.  Additionally, Aardvark needs to be launched with a role or user which can `sts:AssumeRole` into the different account roles.
 
-AardvarkInstanceProfile:
+### Hub role (`AardvarkInstanceProfile`):
+
 - Only create one.
-- Needs the ability to call `sts:AssumeRole` into all of the AardvarkRole's
+- Needs the ability to call `sts:AssumeRole` into all of the `AardvarkRole`s
+
+Inline policy example:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AssumeSpokeRoles",
+            "Effect": "Allow",
+            "Action": [
+                "sts:assumerole"
+            ],
+            "Resource": [
+                "arn:aws:iam::*:role/AardvarkRole"
+            ]
+        }
+    ]
+}
+```
+
+### Spoke roles (`AardvarkRole`):
 
-AardvarkRole:
 - Must exist in every account to be monitored.
 - Must have a trust policy allowing `AardvarkInstanceProfile`.
 - Has these permissions:
+
 ```
 iam:GenerateServiceLastAccessedDetails
 iam:GetServiceLastAccessedDetails
-iam:listrolepolicies
-iam:listroles
+iam:ListRolePolicies
+iam:ListRoles
 iam:ListUsers
 iam:ListPolicies
 iam:ListGroups
 ```
+Assume role policy document example (be sure to replace the account ID with a real one):
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AllowHubRoleAssume",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "arn:aws:iam::111111111111:role/AardvarkInstanceProfile"
+                ]
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}
+```
+
+Inline policy example:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "IAMAccess",
+            "Effect": "Allow",
+            "Action": [
+              "iam:GenerateServiceLastAccessedDetails",
+              "iam:GetServiceLastAccessedDetails",
+              "iam:ListRolePolicies",
+              "iam:ListRoles",
+              "iam:ListUsers",
+              "iam:ListPolicies",
+              "iam:ListGroups"
+            ],
+            "Resource": [
+              "*"
+            ]
+        }
+    ]
+}
+```
 
-So if you are monitoring `n` accounts, you will always need `n+1` roles. (`n` AardvarkRoles and `1` AardvarkInstanceProfile).
+So if you are monitoring `n` accounts, you will always need `n+1` roles. (one `AardvarkInstanceProfile` and n `AardvarkRole`s).
 
-Note: For locally running aardvark, you don't have to take care of the AardvarkInstanceProfile. Instead, just attach a policy which contains "sts:AssumeRole" to the user you are using on the AWS CLI to assume Aardvark Role. Also, the same user should be mentioned in the trust policy of Aardvark Role for proper assignment of the privileges.
+Note: For locally running aardvark, you don't have to take care of the AardvarkInstanceProfile. Instead, just attach a policy which contains `sts:AssumeRole` to the user you are using on the AWS CLI to assume Aardvark Role. Also, the same user should be mentioned in the trust policy of Aardvark Role for proper assignment of the privileges.
 
 ## Gather Access Advisor Data
 
@@ -80,24 +166,30 @@ You'll likely want to refresh the Access Advisor data regularly.  We recommend r
 
 If you don't have SWAG you can pass comma separated account numbers:
 
-    aardvark update -a 123456789012,210987654321
+    aardvark update -a 123456789012 -a 210987654321
 
 #### With SWAG:
 
 Aardvark can use [SWAG](https://github.com/Netflix-Skunkworks/swag-client) to look up accounts, so you can run against all with:
 
-    aardvark update
+```bash
+aardvark update
+```
 
 or by account name/tag with:
 
-    aardvark update -a dev,test,prod
+```bash
+aardvark update -a dev -a test -a prod
+```
 
 
 ## API
 
 ### Start the API
 
-    aardvark start_api -b 0.0.0.0:5000
+```bash
+FLASK_APP=aardvark flask run -b 0.0.0.0:5000
+```
 
 In production, you'll likely want to have something like supervisor starting the API for you.
 

diff --git a/TODO.md b/TODO.md
diff --git a/aardvark/__about__.py b/aardvark/__about__.py