diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index c378ddd93d..d21ff92cb5 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -10,10 +10,11 @@ import botocore.exceptions import neo4j -from . import ec2 -from . import organizations -from .resources import RESOURCE_FUNCTIONS from cartography.config import Config +from cartography.intel.aws import ec2 +from cartography.intel.aws import organizations +from cartography.intel.aws.resources import RESOURCE_FUNCTIONS +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata @@ -121,7 +122,7 @@ def _autodiscover_accounts( logger.info("Trying to autodiscover accounts.") try: # Fetch all accounts - client = boto3_session.client('organizations') + client = boto3_session.client('organizations', config=get_botocore_config()) paginator = client.get_paginator('list_accounts') accounts: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 828b5c86c5..36c17cbbfc 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -12,6 +12,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -22,7 +23,7 @@ @timeit @aws_handle_regions def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('apigateway', region_name=region) + client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('get_rest_apis') apis: List[Any] = [] for page in paginator.paginate(): @@ -38,7 +39,7 @@ def get_rest_api_details( """ Iterates over all API Gateway REST APIs. """ - client = boto3_session.client('apigateway', region_name=region) + client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config()) apis = [] for api in rest_apis: stages = get_rest_api_stages(api, client) diff --git a/cartography/intel/aws/config.py b/cartography/intel/aws/config.py index 0cf746f6c9..142bdf775c 100644 --- a/cartography/intel/aws/config.py +++ b/cartography/intel/aws/config.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -15,7 +16,7 @@ @timeit @aws_handle_regions def get_configuration_recorders(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) recorders: List[Dict] = [] response = client.describe_configuration_recorders() for recorder in response.get('ConfigurationRecorders'): @@ -26,7 +27,7 @@ def get_configuration_recorders(boto3_session: boto3.session.Session, region: st @timeit @aws_handle_regions def get_delivery_channels(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) channels: List[Dict] = [] response = client.describe_delivery_channels() for channel in response.get('DeliveryChannels'): @@ -37,7 +38,7 @@ def get_delivery_channels(boto3_session: boto3.session.Session, region: str) -> @timeit @aws_handle_regions def get_config_rules(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_config_rules') rules: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/dynamodb.py b/cartography/intel/aws/dynamodb.py index 05f39caf98..660f54d503 100644 --- a/cartography/intel/aws/dynamodb.py +++ b/cartography/intel/aws/dynamodb.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.dynamodb.gsi import DynamoDBGSISchema from cartography.models.aws.dynamodb.tables import DynamoDBTableSchema from cartography.stats import get_stats_client diff --git a/cartography/intel/aws/ec2/__init__.py b/cartography/intel/aws/ec2/__init__.py index 1c28d7e773..5def9d24dc 100644 --- a/cartography/intel/aws/ec2/__init__.py +++ b/cartography/intel/aws/ec2/__init__.py @@ -3,6 +3,7 @@ import boto3 +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) @@ -10,6 +11,6 @@ @timeit def get_ec2_regions(boto3_session: boto3.session.Session) -> List[str]: - client = boto3_session.client('ec2') + client = boto3_session.client('ec2', config=get_botocore_config()) result = client.describe_regions() return [r['RegionName'] for r in result['Regions']] diff --git a/cartography/intel/aws/ec2/auto_scaling_groups.py b/cartography/intel/aws/ec2/auto_scaling_groups.py index 3e456be736..1687fdf68d 100644 --- a/cartography/intel/aws/ec2/auto_scaling_groups.py +++ b/cartography/intel/aws/ec2/auto_scaling_groups.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/elastic_ip_addresses.py b/cartography/intel/aws/ec2/elastic_ip_addresses.py index fb9eeb35df..76abd59dc0 100644 --- a/cartography/intel/aws/ec2/elastic_ip_addresses.py +++ b/cartography/intel/aws/ec2/elastic_ip_addresses.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/images.py b/cartography/intel/aws/ec2/images.py index d5af19d657..a902a94925 100644 --- a/cartography/intel/aws/ec2/images.py +++ b/cartography/intel/aws/ec2/images.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.images import EC2ImageSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index d288c27e63..94f3e07525 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema diff --git a/cartography/intel/aws/ec2/internet_gateways.py b/cartography/intel/aws/ec2/internet_gateways.py index 6d1b73a56b..8bd8ac7b25 100644 --- a/cartography/intel/aws/ec2/internet_gateways.py +++ b/cartography/intel/aws/ec2/internet_gateways.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/key_pairs.py b/cartography/intel/aws/ec2/key_pairs.py index df6dd46219..6866ccb9a9 100644 --- a/cartography/intel/aws/ec2/key_pairs.py +++ b/cartography/intel/aws/ec2/key_pairs.py @@ -5,8 +5,8 @@ import boto3 import neo4j -from .util import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/launch_templates.py b/cartography/intel/aws/ec2/launch_templates.py index 01cd412700..7b30a1d210 100644 --- a/cartography/intel/aws/ec2/launch_templates.py +++ b/cartography/intel/aws/ec2/launch_templates.py @@ -5,9 +5,9 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/load_balancer_v2s.py b/cartography/intel/aws/ec2/load_balancer_v2s.py index bc4b4d55eb..4d200dc149 100644 --- a/cartography/intel/aws/ec2/load_balancer_v2s.py +++ b/cartography/intel/aws/ec2/load_balancer_v2s.py @@ -6,7 +6,7 @@ import botocore import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/load_balancers.py b/cartography/intel/aws/ec2/load_balancers.py index caf7f61f89..4bbfce1cc4 100644 --- a/cartography/intel/aws/ec2/load_balancers.py +++ b/cartography/intel/aws/ec2/load_balancers.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 1b6bcf069a..b6a731d22c 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -8,9 +8,9 @@ import boto3 import neo4j -from .util import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema diff --git a/cartography/intel/aws/ec2/reserved_instances.py b/cartography/intel/aws/ec2/reserved_instances.py index 3ceac7c5b1..bb824c2d1c 100644 --- a/cartography/intel/aws/ec2/reserved_instances.py +++ b/cartography/intel/aws/ec2/reserved_instances.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index c0a66c8d22..f32070def0 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -6,8 +6,8 @@ import boto3 import neo4j -from .util import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/snapshots.py b/cartography/intel/aws/ec2/snapshots.py index a235d69edf..aea1460dd3 100644 --- a/cartography/intel/aws/ec2/snapshots.py +++ b/cartography/intel/aws/ec2/snapshots.py @@ -6,6 +6,7 @@ import neo4j from botocore.exceptions import ClientError +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -27,7 +28,7 @@ def get_snapshots_in_use(neo4j_session: neo4j.Session, region: str, current_aws_ @timeit @aws_handle_regions def get_snapshots(boto3_session: boto3.session.Session, region: str, in_use_snapshot_ids: List[str]) -> List[Dict]: - client = boto3_session.client('ec2', region_name=region) + client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_snapshots') snapshots: List[Dict] = [] for page in paginator.paginate(OwnerIds=['self']): diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index d306049835..571ca9f905 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -5,8 +5,8 @@ import boto3 import neo4j -from .util import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/tgw.py b/cartography/intel/aws/ec2/tgw.py index 340d879730..a0904150b9 100644 --- a/cartography/intel/aws/ec2/tgw.py +++ b/cartography/intel/aws/ec2/tgw.py @@ -6,7 +6,7 @@ import botocore.exceptions import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/volumes.py b/cartography/intel/aws/ec2/volumes.py index 6b8610d3cc..23df61d33e 100644 --- a/cartography/intel/aws/ec2/volumes.py +++ b/cartography/intel/aws/ec2/volumes.py @@ -9,6 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.intel.aws.util.arns import build_arn +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.volumes import EBSVolumeSchema from cartography.util import aws_handle_regions from cartography.util import timeit @@ -19,7 +20,7 @@ @timeit @aws_handle_regions def get_volumes(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ec2', region_name=region) + client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_volumes') volumes: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/ec2/vpc.py b/cartography/intel/aws/ec2/vpc.py index f62194c482..a8b8665725 100644 --- a/cartography/intel/aws/ec2/vpc.py +++ b/cartography/intel/aws/ec2/vpc.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc_peerings.py b/cartography/intel/aws/ec2/vpc_peerings.py index 93e11a57a8..867ac48565 100644 --- a/cartography/intel/aws/ec2/vpc_peerings.py +++ b/cartography/intel/aws/ec2/vpc_peerings.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ecr.py b/cartography/intel/aws/ecr.py index 4569024d52..461b691b46 100644 --- a/cartography/intel/aws/ecr.py +++ b/cartography/intel/aws/ecr.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job @@ -20,7 +21,7 @@ @aws_handle_regions def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> List[Dict]: logger.info("Getting ECR repositories for region '%s'.", region) - client = boto3_session.client('ecr', region_name=region) + client = boto3_session.client('ecr', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_repositories') ecr_repositories: List[Dict] = [] for page in paginator.paginate(): @@ -32,7 +33,7 @@ def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> L @aws_handle_regions def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]: logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region) - client = boto3_session.client('ecr', region_name=region) + client = boto3_session.client('ecr', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_images') ecr_repository_images: List[Dict] = [] for page in paginator.paginate(repositoryName=repository_name): diff --git a/cartography/intel/aws/ecs.py b/cartography/intel/aws/ecs.py index 70e8d503cf..2f95f2c73d 100644 --- a/cartography/intel/aws/ecs.py +++ b/cartography/intel/aws/ecs.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import camel_to_snake from cartography.util import dict_date_to_epoch @@ -18,7 +19,7 @@ @timeit @aws_handle_regions def get_ecs_cluster_arns(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_clusters') cluster_arns: List[str] = [] for page in paginator.paginate(): @@ -33,7 +34,7 @@ def get_ecs_clusters( region: str, cluster_arns: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) # TODO: also include attachment info, and make relationships between the attachements # and the cluster. includes = ['SETTINGS', 'CONFIGURATIONS'] @@ -52,7 +53,7 @@ def get_ecs_container_instances( boto3_session: boto3.session.Session, region: str, ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_container_instances') container_instances: List[Dict[str, Any]] = [] container_instance_arns: List[str] = [] @@ -73,7 +74,7 @@ def get_ecs_container_instances( @timeit @aws_handle_regions def get_ecs_services(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_services') services: List[Dict[str, Any]] = [] service_arns: List[str] = [] @@ -96,7 +97,7 @@ def get_ecs_task_definitions( region: str, tasks: List[Dict[str, Any]], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) task_definitions: List[Dict[str, Any]] = [] for task in tasks: task_definition = client.describe_task_definition( @@ -109,7 +110,7 @@ def get_ecs_task_definitions( @timeit @aws_handle_regions def get_ecs_tasks(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_tasks') tasks: List[Dict[str, Any]] = [] task_arns: List[str] = [] diff --git a/cartography/intel/aws/eks.py b/cartography/intel/aws/eks.py index 07173eccbc..9a4f116b6c 100644 --- a/cartography/intel/aws/eks.py +++ b/cartography/intel/aws/eks.py @@ -8,6 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.eks.clusters import EKSClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit @@ -18,7 +19,7 @@ @timeit @aws_handle_regions def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('eks', region_name=region) + client = boto3_session.client('eks', region_name=region, config=get_botocore_config()) clusters: List[str] = [] paginator = client.get_paginator('list_clusters') for page in paginator.paginate(): @@ -28,7 +29,7 @@ def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[ @timeit def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict: - client = boto3_session.client('eks', region_name=region) + client = boto3_session.client('eks', region_name=region, config=get_botocore_config()) response = client.describe_cluster(name=cluster_name) return response['cluster'] diff --git a/cartography/intel/aws/elasticache.py b/cartography/intel/aws/elasticache.py index f5afb25685..a721f80df7 100644 --- a/cartography/intel/aws/elasticache.py +++ b/cartography/intel/aws/elasticache.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import merge_module_sync_metadata @@ -39,7 +40,7 @@ def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]: @aws_handle_regions def get_elasticache_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]: logger.debug(f"Getting ElastiCache Clusters in region '{region}'.") - client = boto3_session.client('elasticache', region_name=region) + client = boto3_session.client('elasticache', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_cache_clusters') clusters: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/emr.py b/cartography/intel/aws/emr.py index 3ed964e731..049ad91178 100644 --- a/cartography/intel/aws/emr.py +++ b/cartography/intel/aws/emr.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.emr import EMRClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/iam.py b/cartography/intel/aws/iam.py index eec5634a43..0759f14ee6 100644 --- a/cartography/intel/aws/iam.py +++ b/cartography/intel/aws/iam.py @@ -11,6 +11,7 @@ from cartography.intel.aws.permission_relationships import parse_statement_node from cartography.intel.aws.permission_relationships import principal_allowed_on_resource +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_cleanup_job @@ -34,7 +35,7 @@ def get_policy_name_from_arn(arn: str) -> str: @timeit def get_group_policies(boto3_session: boto3.session.Session, group_name: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_group_policies') policy_names: List[Dict] = [] for page in paginator.paginate(GroupName=group_name): @@ -46,13 +47,13 @@ def get_group_policies(boto3_session: boto3.session.Session, group_name: str) -> def get_group_policy_info( boto3_session: boto3.session.Session, group_name: str, policy_name: str, ) -> Any: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) return client.get_group_policy(GroupName=group_name, PolicyName=policy_name) @timeit def get_group_membership_data(boto3_session: boto3.session.Session, group_name: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) try: memberships = client.get_group(GroupName=group_name) return memberships @@ -187,7 +188,7 @@ def get_role_tags(boto3_session: boto3.session.Session) -> List[Dict]: @timeit def get_user_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_users') users: List[Dict] = [] @@ -198,7 +199,7 @@ def get_user_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_group_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_groups') groups: List[Dict] = [] for page in paginator.paginate(): @@ -208,7 +209,7 @@ def get_group_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_role_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_roles') roles: List[Dict] = [] for page in paginator.paginate(): @@ -218,7 +219,7 @@ def get_role_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_account_access_key_data(boto3_session: boto3.session.Session, username: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) # NOTE we can get away without using a paginator here because users are limited to two access keys access_keys: Dict = {} try: diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index 51dd064a12..3c823fe5c1 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -9,6 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema from cartography.util import aws_handle_regions @@ -34,7 +35,7 @@ def get_inspector_findings( session: boto3.session.Session, region: str, current_aws_account_id: str, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ We must list_findings by filtering the request, otherwise the request could tiemout. First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away, @@ -42,7 +43,7 @@ def get_inspector_findings( list_members will get us all the accounts that have delegated access to the account specified by current_aws_account_id. """ - client = session.client('inspector2', region_name=region) + client = session.client('inspector2', region_name=region, config=get_botocore_config()) members = aws_paginate(client, 'list_members', 'members') # the current host account may not be considered a "member", but we still fetch its findings diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index ec60cb48ba..fcb3c34fa8 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -13,6 +13,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -23,7 +24,7 @@ @timeit @aws_handle_regions def get_kms_key_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('kms', region_name=region) + client = boto3_session.client('kms', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_keys') key_list: List[Any] = [] for page in paginator.paginate(): @@ -50,7 +51,7 @@ def get_kms_key_details( """ Iterates over all KMS Keys. """ - client = boto3_session.client('kms', region_name=region) + client = boto3_session.client('kms', region_name=region, config=get_botocore_config()) for key in kms_key_data: policy = get_policy(key, client) aliases = get_aliases(key, client) diff --git a/cartography/intel/aws/lambda_function.py b/cartography/intel/aws/lambda_function.py index b9e8ca6237..ee8a239208 100644 --- a/cartography/intel/aws/lambda_function.py +++ b/cartography/intel/aws/lambda_function.py @@ -8,6 +8,7 @@ import botocore import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -21,7 +22,7 @@ def get_lambda_data(boto3_session: boto3.session.Session, region: str) -> List[D """ Create an Lambda boto3 client and grab all the lambda functions. """ - client = boto3_session.client('lambda', region_name=region) + client = boto3_session.client('lambda', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_functions') lambda_functions = [] for page in paginator.paginate(): @@ -111,7 +112,7 @@ def get_event_source_mappings(lambda_function: Dict, client: botocore.client.Bas def get_lambda_function_details( boto3_session: boto3.session.Session, data: List[Dict], region: str, ) -> List[Tuple[str, List[Any], List[Any], List[Any]]]: - client = boto3_session.client('lambda', region_name=region) + client = boto3_session.client('lambda', region_name=region, config=get_botocore_config()) details = [] for lambda_function in data: function_aliases = get_function_aliases(lambda_function, client) diff --git a/cartography/intel/aws/organizations.py b/cartography/intel/aws/organizations.py index bfcbc9cb08..1fb11ed652 100644 --- a/cartography/intel/aws/organizations.py +++ b/cartography/intel/aws/organizations.py @@ -5,6 +5,7 @@ import botocore.exceptions import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) @@ -16,7 +17,7 @@ def get_account_from_arn(arn: str) -> str: def get_caller_identity(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('sts') + client = boto3_session.client('sts', config=get_botocore_config()) return client.get_caller_identity() diff --git a/cartography/intel/aws/rds.py b/cartography/intel/aws/rds.py index 333766503e..b2bc3cd437 100644 --- a/cartography/intel/aws/rds.py +++ b/cartography/intel/aws/rds.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import aws_paginate @@ -24,7 +25,7 @@ def get_rds_cluster_data(boto3_session: boto3.session.Session, region: str) -> L """ Create an RDS boto3 client and grab all the DBClusters. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_db_clusters') instances: List[Any] = [] for page in paginator.paginate(): @@ -117,7 +118,7 @@ def get_rds_instance_data(boto3_session: boto3.session.Session, region: str) -> """ Create an RDS boto3 client and grab all the DBInstances. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_db_instances') instances: List[Any] = [] for page in paginator.paginate(): @@ -223,7 +224,7 @@ def get_rds_snapshot_data(boto3_session: boto3.session.Session, region: str) -> """ Create an RDS boto3 client and grab all the DBSnapshots. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) return aws_paginate(client, 'describe_db_snapshots', 'DBSnapshots') diff --git a/cartography/intel/aws/redshift.py b/cartography/intel/aws/redshift.py index 290ec0535b..bb25932e59 100644 --- a/cartography/intel/aws/redshift.py +++ b/cartography/intel/aws/redshift.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -15,7 +16,7 @@ @timeit @aws_handle_regions def get_redshift_cluster_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('redshift', region_name=region) + client = boto3_session.client('redshift', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_clusters') clusters: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/resourcegroupstaggingapi.py b/cartography/intel/aws/resourcegroupstaggingapi.py index b3444c7d8c..e5011fe83c 100644 --- a/cartography/intel/aws/resourcegroupstaggingapi.py +++ b/cartography/intel/aws/resourcegroupstaggingapi.py @@ -7,6 +7,7 @@ import neo4j from cartography.intel.aws.iam import get_role_tags +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job @@ -128,7 +129,7 @@ def get_tags(boto3_session: boto3.session.Session, resource_type: str, region: s if resource_type == 'iam:role': return get_role_tags(boto3_session) - client = boto3_session.client('resourcegroupstaggingapi', region_name=region) + client = boto3_session.client('resourcegroupstaggingapi', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('get_resources') resources: List[Dict] = [] for page in paginator.paginate( diff --git a/cartography/intel/aws/resources.py b/cartography/intel/aws/resources.py index 8c7879049d..52ba1aab0e 100644 --- a/cartography/intel/aws/resources.py +++ b/cartography/intel/aws/resources.py @@ -1,46 +1,46 @@ from typing import Dict -from . import apigateway -from . import config -from . import dynamodb -from . import ecr -from . import ecs -from . import eks -from . import elasticache -from . import elasticsearch -from . import emr -from . import iam -from . import inspector -from . import kms -from . import lambda_function -from . import permission_relationships -from . import rds -from . import redshift -from . import resourcegroupstaggingapi -from . import route53 -from . import s3 -from . import secretsmanager -from . import securityhub -from . import sqs -from . import ssm -from .ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups -from .ec2.elastic_ip_addresses import sync_elastic_ip_addresses -from .ec2.images import sync_ec2_images -from .ec2.instances import sync_ec2_instances -from .ec2.internet_gateways import sync_internet_gateways -from .ec2.key_pairs import sync_ec2_key_pairs -from .ec2.launch_templates import sync_ec2_launch_templates -from .ec2.load_balancer_v2s import sync_load_balancer_v2s -from .ec2.load_balancers import sync_load_balancers -from .ec2.network_interfaces import sync_network_interfaces -from .ec2.reserved_instances import sync_ec2_reserved_instances -from .ec2.security_groups import sync_ec2_security_groupinfo -from .ec2.snapshots import sync_ebs_snapshots -from .ec2.subnets import sync_subnets -from .ec2.tgw import sync_transit_gateways -from .ec2.volumes import sync_ebs_volumes -from .ec2.vpc import sync_vpc -from .ec2.vpc_peerings import sync_vpc_peerings +from cartography.intel.aws import apigateway +from cartography.intel.aws import config +from cartography.intel.aws import dynamodb +from cartography.intel.aws import ecr +from cartography.intel.aws import ecs +from cartography.intel.aws import eks +from cartography.intel.aws import elasticache +from cartography.intel.aws import elasticsearch +from cartography.intel.aws import emr +from cartography.intel.aws import iam +from cartography.intel.aws import inspector +from cartography.intel.aws import kms +from cartography.intel.aws import lambda_function +from cartography.intel.aws import permission_relationships +from cartography.intel.aws import rds +from cartography.intel.aws import redshift +from cartography.intel.aws import resourcegroupstaggingapi +from cartography.intel.aws import route53 +from cartography.intel.aws import s3 +from cartography.intel.aws import secretsmanager +from cartography.intel.aws import securityhub +from cartography.intel.aws import sqs +from cartography.intel.aws import ssm +from cartography.intel.aws.ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups +from cartography.intel.aws.ec2.elastic_ip_addresses import sync_elastic_ip_addresses +from cartography.intel.aws.ec2.images import sync_ec2_images +from cartography.intel.aws.ec2.instances import sync_ec2_instances +from cartography.intel.aws.ec2.internet_gateways import sync_internet_gateways +from cartography.intel.aws.ec2.key_pairs import sync_ec2_key_pairs +from cartography.intel.aws.ec2.launch_templates import sync_ec2_launch_templates +from cartography.intel.aws.ec2.load_balancer_v2s import sync_load_balancer_v2s +from cartography.intel.aws.ec2.load_balancers import sync_load_balancers +from cartography.intel.aws.ec2.network_interfaces import sync_network_interfaces +from cartography.intel.aws.ec2.reserved_instances import sync_ec2_reserved_instances +from cartography.intel.aws.ec2.security_groups import sync_ec2_security_groupinfo +from cartography.intel.aws.ec2.snapshots import sync_ebs_snapshots +from cartography.intel.aws.ec2.subnets import sync_subnets +from cartography.intel.aws.ec2.tgw import sync_transit_gateways +from cartography.intel.aws.ec2.volumes import sync_ebs_volumes +from cartography.intel.aws.ec2.vpc import sync_vpc +from cartography.intel.aws.ec2.vpc_peerings import sync_vpc_peerings RESOURCE_FUNCTIONS: Dict = { 'iam': iam.sync, diff --git a/cartography/intel/aws/route53.py b/cartography/intel/aws/route53.py index f7bc2eb6bc..f2c5289023 100644 --- a/cartography/intel/aws/route53.py +++ b/cartography/intel/aws/route53.py @@ -8,6 +8,7 @@ import botocore import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit @@ -421,7 +422,7 @@ def sync( update_tag: int, common_job_parameters: Dict, ) -> None: logger.info("Syncing Route53 for account '%s'.", current_aws_account_id) - client = boto3_session.client('route53') + client = boto3_session.client('route53', config=get_botocore_config()) zones = get_zones(client) load_dns_details(neo4j_session, zones, current_aws_account_id, update_tag) link_sub_zones(neo4j_session, update_tag) diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index 396c275926..e199892296 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -16,6 +16,7 @@ from botocore.exceptions import EndpointConnectionError from policyuniverse.policy import Policy +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_job @@ -30,7 +31,7 @@ @timeit def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]: - client = boto3_session.client('s3') + client = boto3_session.client('s3', config=get_botocore_config()) # NOTE no paginator available for this operation buckets = client.list_buckets() for bucket in buckets['Buckets']: @@ -66,7 +67,7 @@ async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail: # in us-east-1 region client = s3_regional_clients.get(bucket['Region']) if not client: - client = boto3_session.client('s3', bucket['Region']) + client = boto3_session.client('s3', bucket['Region'], config=get_botocore_config()) s3_regional_clients[bucket['Region']] = client ( acl, diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index 2fa5bbd0b1..f1b3a1117f 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import dict_date_to_epoch from cartography.util import run_cleanup_job @@ -16,7 +17,7 @@ @timeit @aws_handle_regions def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('secretsmanager', region_name=region) + client = boto3_session.client('secretsmanager', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_secrets') secrets: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/securityhub.py b/cartography/intel/aws/securityhub.py index 5dfff53cee..d0b1fb1c30 100644 --- a/cartography/intel/aws/securityhub.py +++ b/cartography/intel/aws/securityhub.py @@ -6,6 +6,7 @@ import neo4j from dateutil import parser +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit @@ -14,7 +15,7 @@ @timeit def get_hub(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('securityhub') + client = boto3_session.client('securityhub', config=get_botocore_config()) try: return client.describe_hub() except client.exceptions.ResourceNotFoundException: diff --git a/cartography/intel/aws/sqs.py b/cartography/intel/aws/sqs.py index 31f59a401e..0716ca7bc3 100644 --- a/cartography/intel/aws/sqs.py +++ b/cartography/intel/aws/sqs.py @@ -9,6 +9,7 @@ import neo4j from botocore.exceptions import ClientError +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -19,7 +20,7 @@ @timeit @aws_handle_regions def get_sqs_queue_list(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('sqs', region_name=region) + client = boto3_session.client('sqs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_queues') queues: List[Any] = [] for page in paginator.paginate(): @@ -36,7 +37,7 @@ def get_sqs_queue_attributes( """ Iterates over all SQS queues. Returns a dict with url as key, and attributes as value. """ - client = boto3_session.client('sqs') + client = boto3_session.client('sqs', config=get_botocore_config()) queue_attributes = [] for queue_url in queue_urls: diff --git a/cartography/intel/aws/ssm.py b/cartography/intel/aws/ssm.py index 937cba52bf..fb5b471e9e 100644 --- a/cartography/intel/aws/ssm.py +++ b/cartography/intel/aws/ssm.py @@ -8,6 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema from cartography.util import aws_handle_regions @@ -38,7 +39,7 @@ def get_instance_information( region: str, instance_ids: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ssm', region_name=region) + client = boto3_session.client('ssm', region_name=region, config=get_botocore_config()) instance_information: List[Dict[str, Any]] = [] paginator = client.get_paginator('describe_instance_information') for i in range(0, len(instance_ids), 50): @@ -67,7 +68,7 @@ def get_instance_patches( region: str, instance_ids: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ssm', region_name=region) + client = boto3_session.client('ssm', region_name=region, config=get_botocore_config()) instance_patches: List[Dict[str, Any]] = [] paginator = client.get_paginator('describe_instance_patches') for instance_id in instance_ids: diff --git a/cartography/intel/aws/ec2/util.py b/cartography/intel/aws/util/boto3.py similarity index 91% rename from cartography/intel/aws/ec2/util.py rename to cartography/intel/aws/util/boto3.py index 1f85b05124..ecebb5afd3 100644 --- a/cartography/intel/aws/ec2/util.py +++ b/cartography/intel/aws/util/boto3.py @@ -1,7 +1,6 @@ import botocore.config -# TODO memoize this def get_botocore_config() -> botocore.config.Config: return botocore.config.Config( read_timeout=360, diff --git a/tests/unit/cartography/intel/github/test_github.py b/tests/unit/cartography/intel/github/test_github.py index 0a2ffd5c05..500f842aea 100644 --- a/tests/unit/cartography/intel/github/test_github.py +++ b/tests/unit/cartography/intel/github/test_github.py @@ -16,23 +16,21 @@ from tests.data.github.rate_limit import RATE_LIMIT_RESPONSE_JSON +@patch('time.sleep') @patch('cartography.intel.github.util.handle_rate_limit_sleep') -@patch('cartography.intel.github.util.fetch_page') +@patch('cartography.intel.github.util.fetch_page', side_effect=HTTPError('my-error', response=Response())) def test_fetch_all_handles_retries( - mock_fetch_page: Mock, - mock_handle_rate_limit_sleep: Mock, + mock_fetch_page: Mock, + mock_handle_rate_limit_sleep: Mock, + mock_sleep: Mock, ) -> None: ''' Ensures that fetch_all re-reaises the same exceptions when exceeding retry limit ''' # Arrange - exception = HTTPError - response = Response() - response.status_code = 500 - mock_fetch_page.side_effect = exception('my-error', response=response) retries = 3 # Act - with pytest.raises(exception) as excinfo: + with pytest.raises(Exception) as excinfo: fetch_all('my-token', 'my-api_url', 'my-org', 'my-query', 'my-resource', retries=retries) # Assert assert mock_handle_rate_limit_sleep.call_count == retries