Specify botocore config for all boto clients

cartography/intel/aws/__init__.py

@@ -10,10 +10,11 @@
 import botocore.exceptions
 import neo4j
 
-from . import ec2
-from . import organizations
-from .resources import RESOURCE_FUNCTIONS
 from cartography.config import Config
+from cartography.intel.aws import ec2
+from cartography.intel.aws import organizations
+from cartography.intel.aws.resources import RESOURCE_FUNCTIONS
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs
 from cartography.stats import get_stats_client
 from cartography.util import merge_module_sync_metadata
@@ -121,7 +122,7 @@ def _autodiscover_accounts(
     logger.info("Trying to autodiscover accounts.")
     try:
         # Fetch all accounts
-        client = boto3_session.client('organizations')
+        client = boto3_session.client('organizations', config=get_botocore_config())
         paginator = client.get_paginator('list_accounts')
         accounts: List[Dict] = []
         for page in paginator.paginate():

cartography/intel/aws/apigateway.py

@@ -12,6 +12,7 @@
 from botocore.exceptions import ClientError
 from policyuniverse.policy import Policy
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -22,7 +23,7 @@
 @timeit
 @aws_handle_regions
 def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('apigateway', region_name=region)
+    client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('get_rest_apis')
     apis: List[Any] = []
     for page in paginator.paginate():
@@ -38,7 +39,7 @@ def get_rest_api_details(
     """
     Iterates over all API Gateway REST APIs.
     """
-    client = boto3_session.client('apigateway', region_name=region)
+    client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config())
     apis = []
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)

cartography/intel/aws/config.py

@@ -5,6 +5,7 @@
 import boto3
 import neo4j
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -15,7 +16,7 @@
 @timeit
 @aws_handle_regions
 def get_configuration_recorders(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('config', region_name=region)
+    client = boto3_session.client('config', region_name=region, config=get_botocore_config())
     recorders: List[Dict] = []
     response = client.describe_configuration_recorders()
     for recorder in response.get('ConfigurationRecorders'):
@@ -26,7 +27,7 @@ def get_configuration_recorders(boto3_session: boto3.session.Session, region: st
 @timeit
 @aws_handle_regions
 def get_delivery_channels(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('config', region_name=region)
+    client = boto3_session.client('config', region_name=region, config=get_botocore_config())
     channels: List[Dict] = []
     response = client.describe_delivery_channels()
     for channel in response.get('DeliveryChannels'):
@@ -37,7 +38,7 @@ def get_delivery_channels(boto3_session: boto3.session.Session, region: str) ->
 @timeit
 @aws_handle_regions
 def get_config_rules(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('config', region_name=region)
+    client = boto3_session.client('config', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_config_rules')
     rules: List[Dict] = []
     for page in paginator.paginate():

cartography/intel/aws/dynamodb.py

@@ -8,7 +8,7 @@
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.dynamodb.gsi import DynamoDBGSISchema
 from cartography.models.aws.dynamodb.tables import DynamoDBTableSchema
 from cartography.stats import get_stats_client

cartography/intel/aws/ec2/__init__.py

@@ -3,13 +3,14 @@
 
 import boto3
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def get_ec2_regions(boto3_session: boto3.session.Session) -> List[str]:
-    client = boto3_session.client('ec2')
+    client = boto3_session.client('ec2', config=get_botocore_config())
     result = client.describe_regions()
     return [r['RegionName'] for r in result['Regions']]

cartography/intel/aws/ec2/auto_scaling_groups.py

@@ -5,7 +5,7 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/elastic_ip_addresses.py

@@ -6,7 +6,7 @@
 import neo4j
 from botocore.exceptions import ClientError
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/images.py

@@ -9,7 +9,7 @@
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.images import EC2ImageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit

cartography/intel/aws/ec2/instances.py

@@ -10,7 +10,7 @@
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.instances import EC2InstanceSchema
 from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
 from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema

cartography/intel/aws/ec2/internet_gateways.py

@@ -5,7 +5,7 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/key_pairs.py

@@ -5,8 +5,8 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
 from cartography.graph.job import GraphJob
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit

cartography/intel/aws/ec2/launch_templates.py

@@ -5,9 +5,9 @@
 import neo4j
 from botocore.exceptions import ClientError
 
-from .util import get_botocore_config
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema
 from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema
 from cartography.util import aws_handle_regions

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -6,7 +6,7 @@
 import botocore
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/load_balancers.py

@@ -5,7 +5,7 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/network_interfaces.py

@@ -8,9 +8,9 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema
 from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema
 from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema

cartography/intel/aws/ec2/reserved_instances.py

@@ -6,7 +6,7 @@
 import neo4j
 from botocore.exceptions import ClientError
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/security_groups.py

@@ -6,8 +6,8 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
 from cartography.graph.job import GraphJob
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job

cartography/intel/aws/ec2/snapshots.py

@@ -6,6 +6,7 @@
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -27,7 +28,7 @@ def get_snapshots_in_use(neo4j_session: neo4j.Session, region: str, current_aws_
 @timeit
 @aws_handle_regions
 def get_snapshots(boto3_session: boto3.session.Session, region: str, in_use_snapshot_ids: List[str]) -> List[Dict]:
-    client = boto3_session.client('ec2', region_name=region)
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_snapshots')
     snapshots: List[Dict] = []
     for page in paginator.paginate(OwnerIds=['self']):

cartography/intel/aws/ec2/subnets.py

@@ -5,8 +5,8 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
 from cartography.graph.job import GraphJob
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job

cartography/intel/aws/ec2/tgw.py

@@ -6,7 +6,7 @@
 import botocore.exceptions
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/volumes.py

@@ -9,6 +9,7 @@
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.util.arns import build_arn
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.models.aws.ec2.volumes import EBSVolumeSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
@@ -19,7 +20,7 @@
 @timeit
 @aws_handle_regions
 def get_volumes(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ec2', region_name=region)
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_volumes')
     volumes: List[Dict] = []
     for page in paginator.paginate():

cartography/intel/aws/ec2/vpc.py

@@ -6,7 +6,7 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ec2/vpc_peerings.py

@@ -5,7 +5,7 @@
 import boto3
 import neo4j
 
-from .util import get_botocore_config
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit

cartography/intel/aws/ecr.py

@@ -6,6 +6,7 @@
 import boto3
 import neo4j
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import batch
 from cartography.util import run_cleanup_job
@@ -20,7 +21,7 @@
 @aws_handle_regions
 def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
     logger.info("Getting ECR repositories for region '%s'.", region)
-    client = boto3_session.client('ecr', region_name=region)
+    client = boto3_session.client('ecr', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_repositories')
     ecr_repositories: List[Dict] = []
     for page in paginator.paginate():
@@ -32,7 +33,7 @@ def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> L
 @aws_handle_regions
 def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]:
     logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region)
-    client = boto3_session.client('ecr', region_name=region)
+    client = boto3_session.client('ecr', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('list_images')
     ecr_repository_images: List[Dict] = []
     for page in paginator.paginate(repositoryName=repository_name):

cartography/intel/aws/ecs.py

@@ -6,6 +6,7 @@
 import boto3
 import neo4j
 
+from cartography.intel.aws.util.boto3 import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import camel_to_snake
 from cartography.util import dict_date_to_epoch
@@ -18,7 +19,7 @@
 @timeit
 @aws_handle_regions
 def get_ecs_cluster_arns(boto3_session: boto3.session.Session, region: str) -> List[str]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('list_clusters')
     cluster_arns: List[str] = []
     for page in paginator.paginate():
@@ -33,7 +34,7 @@ def get_ecs_clusters(
     region: str,
     cluster_arns: List[str],
 ) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     # TODO: also include attachment info, and make relationships between the attachements
     # and the cluster.
     includes = ['SETTINGS', 'CONFIGURATIONS']
@@ -52,7 +53,7 @@ def get_ecs_container_instances(
     boto3_session: boto3.session.Session,
     region: str,
 ) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('list_container_instances')
     container_instances: List[Dict[str, Any]] = []
     container_instance_arns: List[str] = []
@@ -73,7 +74,7 @@ def get_ecs_container_instances(
 @timeit
 @aws_handle_regions
 def get_ecs_services(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('list_services')
     services: List[Dict[str, Any]] = []
     service_arns: List[str] = []
@@ -96,7 +97,7 @@ def get_ecs_task_definitions(
     region: str,
     tasks: List[Dict[str, Any]],
 ) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     task_definitions: List[Dict[str, Any]] = []
     for task in tasks:
         task_definition = client.describe_task_definition(
@@ -109,7 +110,7 @@ def get_ecs_task_definitions(
 @timeit
 @aws_handle_regions
 def get_ecs_tasks(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ecs', region_name=region)
+    client = boto3_session.client('ecs', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('list_tasks')
     tasks: List[Dict[str, Any]] = []
     task_arns: List[str] = []