From df5b2a61fb80b6301a5ef3f8f7dc74144b37e886 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 20:15:47 -0700 Subject: [PATCH 1/6] Specify botocore config for all boto clients --- cartography/intel/aws/__init__.py | 3 ++- cartography/intel/aws/apigateway.py | 5 +++-- cartography/intel/aws/config.py | 7 ++++--- cartography/intel/aws/ec2/__init__.py | 3 ++- cartography/intel/aws/ec2/snapshots.py | 3 ++- cartography/intel/aws/ec2/volumes.py | 3 ++- cartography/intel/aws/ecr.py | 5 +++-- cartography/intel/aws/ecs.py | 13 +++++++------ cartography/intel/aws/eks.py | 5 +++-- cartography/intel/aws/elasticache.py | 3 ++- cartography/intel/aws/iam.py | 15 ++++++++------- cartography/intel/aws/inspector.py | 5 +++-- cartography/intel/aws/kms.py | 5 +++-- cartography/intel/aws/lambda_function.py | 5 +++-- cartography/intel/aws/organizations.py | 3 ++- cartography/intel/aws/rds.py | 7 ++++--- cartography/intel/aws/redshift.py | 2 +- cartography/intel/aws/resourcegroupstaggingapi.py | 2 +- cartography/intel/aws/route53.py | 2 +- cartography/intel/aws/s3.py | 5 +++-- cartography/intel/aws/secretsmanager.py | 3 ++- cartography/intel/aws/securityhub.py | 3 ++- cartography/intel/aws/sqs.py | 5 +++-- cartography/intel/aws/ssm.py | 5 +++-- 24 files changed, 69 insertions(+), 48 deletions(-) diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index c378ddd93d..ef1d9cca99 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -12,6 +12,7 @@ from . import ec2 from . import organizations +from .ec2.util import get_botocore_config from .resources import RESOURCE_FUNCTIONS from cartography.config import Config from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs @@ -121,7 +122,7 @@ def _autodiscover_accounts( logger.info("Trying to autodiscover accounts.") try: # Fetch all accounts - client = boto3_session.client('organizations') + client = boto3_session.client('organizations', config=get_botocore_config()) paginator = client.get_paginator('list_accounts') accounts: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 828b5c86c5..0f177e036a 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -12,6 +12,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -22,7 +23,7 @@ @timeit @aws_handle_regions def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('apigateway', region_name=region) + client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('get_rest_apis') apis: List[Any] = [] for page in paginator.paginate(): @@ -38,7 +39,7 @@ def get_rest_api_details( """ Iterates over all API Gateway REST APIs. """ - client = boto3_session.client('apigateway', region_name=region) + client = boto3_session.client('apigateway', region_name=region, config=get_botocore_config()) apis = [] for api in rest_apis: stages = get_rest_api_stages(api, client) diff --git a/cartography/intel/aws/config.py b/cartography/intel/aws/config.py index 0cf746f6c9..bab935dbc0 100644 --- a/cartography/intel/aws/config.py +++ b/cartography/intel/aws/config.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -15,7 +16,7 @@ @timeit @aws_handle_regions def get_configuration_recorders(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) recorders: List[Dict] = [] response = client.describe_configuration_recorders() for recorder in response.get('ConfigurationRecorders'): @@ -26,7 +27,7 @@ def get_configuration_recorders(boto3_session: boto3.session.Session, region: st @timeit @aws_handle_regions def get_delivery_channels(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) channels: List[Dict] = [] response = client.describe_delivery_channels() for channel in response.get('DeliveryChannels'): @@ -37,7 +38,7 @@ def get_delivery_channels(boto3_session: boto3.session.Session, region: str) -> @timeit @aws_handle_regions def get_config_rules(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('config', region_name=region) + client = boto3_session.client('config', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_config_rules') rules: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/ec2/__init__.py b/cartography/intel/aws/ec2/__init__.py index 1c28d7e773..5e0bcd4642 100644 --- a/cartography/intel/aws/ec2/__init__.py +++ b/cartography/intel/aws/ec2/__init__.py @@ -3,6 +3,7 @@ import boto3 +from cartography.intel.aws import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) @@ -10,6 +11,6 @@ @timeit def get_ec2_regions(boto3_session: boto3.session.Session) -> List[str]: - client = boto3_session.client('ec2') + client = boto3_session.client('ec2', config=get_botocore_config()) result = client.describe_regions() return [r['RegionName'] for r in result['Regions']] diff --git a/cartography/intel/aws/ec2/snapshots.py b/cartography/intel/aws/ec2/snapshots.py index a235d69edf..8d88591513 100644 --- a/cartography/intel/aws/ec2/snapshots.py +++ b/cartography/intel/aws/ec2/snapshots.py @@ -6,6 +6,7 @@ import neo4j from botocore.exceptions import ClientError +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -27,7 +28,7 @@ def get_snapshots_in_use(neo4j_session: neo4j.Session, region: str, current_aws_ @timeit @aws_handle_regions def get_snapshots(boto3_session: boto3.session.Session, region: str, in_use_snapshot_ids: List[str]) -> List[Dict]: - client = boto3_session.client('ec2', region_name=region) + client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_snapshots') snapshots: List[Dict] = [] for page in paginator.paginate(OwnerIds=['self']): diff --git a/cartography/intel/aws/ec2/volumes.py b/cartography/intel/aws/ec2/volumes.py index 6b8610d3cc..8e9c7cfa72 100644 --- a/cartography/intel/aws/ec2/volumes.py +++ b/cartography/intel/aws/ec2/volumes.py @@ -8,6 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws import get_botocore_config from cartography.intel.aws.util.arns import build_arn from cartography.models.aws.ec2.volumes import EBSVolumeSchema from cartography.util import aws_handle_regions @@ -19,7 +20,7 @@ @timeit @aws_handle_regions def get_volumes(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ec2', region_name=region) + client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_volumes') volumes: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/ecr.py b/cartography/intel/aws/ecr.py index 4569024d52..0f2e014941 100644 --- a/cartography/intel/aws/ecr.py +++ b/cartography/intel/aws/ecr.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job @@ -20,7 +21,7 @@ @aws_handle_regions def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> List[Dict]: logger.info("Getting ECR repositories for region '%s'.", region) - client = boto3_session.client('ecr', region_name=region) + client = boto3_session.client('ecr', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_repositories') ecr_repositories: List[Dict] = [] for page in paginator.paginate(): @@ -32,7 +33,7 @@ def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> L @aws_handle_regions def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]: logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region) - client = boto3_session.client('ecr', region_name=region) + client = boto3_session.client('ecr', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_images') ecr_repository_images: List[Dict] = [] for page in paginator.paginate(repositoryName=repository_name): diff --git a/cartography/intel/aws/ecs.py b/cartography/intel/aws/ecs.py index 70e8d503cf..b123347266 100644 --- a/cartography/intel/aws/ecs.py +++ b/cartography/intel/aws/ecs.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import camel_to_snake from cartography.util import dict_date_to_epoch @@ -18,7 +19,7 @@ @timeit @aws_handle_regions def get_ecs_cluster_arns(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_clusters') cluster_arns: List[str] = [] for page in paginator.paginate(): @@ -33,7 +34,7 @@ def get_ecs_clusters( region: str, cluster_arns: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) # TODO: also include attachment info, and make relationships between the attachements # and the cluster. includes = ['SETTINGS', 'CONFIGURATIONS'] @@ -52,7 +53,7 @@ def get_ecs_container_instances( boto3_session: boto3.session.Session, region: str, ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_container_instances') container_instances: List[Dict[str, Any]] = [] container_instance_arns: List[str] = [] @@ -73,7 +74,7 @@ def get_ecs_container_instances( @timeit @aws_handle_regions def get_ecs_services(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_services') services: List[Dict[str, Any]] = [] service_arns: List[str] = [] @@ -96,7 +97,7 @@ def get_ecs_task_definitions( region: str, tasks: List[Dict[str, Any]], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) task_definitions: List[Dict[str, Any]] = [] for task in tasks: task_definition = client.describe_task_definition( @@ -109,7 +110,7 @@ def get_ecs_task_definitions( @timeit @aws_handle_regions def get_ecs_tasks(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: - client = boto3_session.client('ecs', region_name=region) + client = boto3_session.client('ecs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_tasks') tasks: List[Dict[str, Any]] = [] task_arns: List[str] = [] diff --git a/cartography/intel/aws/eks.py b/cartography/intel/aws/eks.py index 07173eccbc..6c4caef360 100644 --- a/cartography/intel/aws/eks.py +++ b/cartography/intel/aws/eks.py @@ -8,6 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws import get_botocore_config from cartography.models.aws.eks.clusters import EKSClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit @@ -18,7 +19,7 @@ @timeit @aws_handle_regions def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('eks', region_name=region) + client = boto3_session.client('eks', region_name=region, config=get_botocore_config()) clusters: List[str] = [] paginator = client.get_paginator('list_clusters') for page in paginator.paginate(): @@ -28,7 +29,7 @@ def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[ @timeit def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict: - client = boto3_session.client('eks', region_name=region) + client = boto3_session.client('eks', region_name=region, config=get_botocore_config()) response = client.describe_cluster(name=cluster_name) return response['cluster'] diff --git a/cartography/intel/aws/elasticache.py b/cartography/intel/aws/elasticache.py index f5afb25685..e9ec1b7b9e 100644 --- a/cartography/intel/aws/elasticache.py +++ b/cartography/intel/aws/elasticache.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import merge_module_sync_metadata @@ -39,7 +40,7 @@ def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]: @aws_handle_regions def get_elasticache_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]: logger.debug(f"Getting ElastiCache Clusters in region '{region}'.") - client = boto3_session.client('elasticache', region_name=region) + client = boto3_session.client('elasticache', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_cache_clusters') clusters: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/iam.py b/cartography/intel/aws/iam.py index eec5634a43..65e5d6b8ac 100644 --- a/cartography/intel/aws/iam.py +++ b/cartography/intel/aws/iam.py @@ -9,6 +9,7 @@ import boto3 import neo4j +from cartography.intel.aws import get_botocore_config from cartography.intel.aws.permission_relationships import parse_statement_node from cartography.intel.aws.permission_relationships import principal_allowed_on_resource from cartography.stats import get_stats_client @@ -34,7 +35,7 @@ def get_policy_name_from_arn(arn: str) -> str: @timeit def get_group_policies(boto3_session: boto3.session.Session, group_name: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_group_policies') policy_names: List[Dict] = [] for page in paginator.paginate(GroupName=group_name): @@ -46,13 +47,13 @@ def get_group_policies(boto3_session: boto3.session.Session, group_name: str) -> def get_group_policy_info( boto3_session: boto3.session.Session, group_name: str, policy_name: str, ) -> Any: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) return client.get_group_policy(GroupName=group_name, PolicyName=policy_name) @timeit def get_group_membership_data(boto3_session: boto3.session.Session, group_name: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) try: memberships = client.get_group(GroupName=group_name) return memberships @@ -187,7 +188,7 @@ def get_role_tags(boto3_session: boto3.session.Session) -> List[Dict]: @timeit def get_user_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_users') users: List[Dict] = [] @@ -198,7 +199,7 @@ def get_user_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_group_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_groups') groups: List[Dict] = [] for page in paginator.paginate(): @@ -208,7 +209,7 @@ def get_group_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_role_list_data(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) paginator = client.get_paginator('list_roles') roles: List[Dict] = [] for page in paginator.paginate(): @@ -218,7 +219,7 @@ def get_role_list_data(boto3_session: boto3.session.Session) -> Dict: @timeit def get_account_access_key_data(boto3_session: boto3.session.Session, username: str) -> Dict: - client = boto3_session.client('iam') + client = boto3_session.client('iam', config=get_botocore_config()) # NOTE we can get away without using a paginator here because users are limited to two access keys access_keys: Dict = {} try: diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index c424a42096..a9e89b1c38 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -9,6 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema from cartography.util import aws_handle_regions @@ -25,7 +26,7 @@ def get_inspector_findings( session: boto3.session.Session, region: str, current_aws_account_id: str, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ We must list_findings by filtering the request, otherwise the request could tiemout. First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away, @@ -33,7 +34,7 @@ def get_inspector_findings( list_members will get us all the accounts that have delegated access to the account specified by current_aws_account_id. """ - client = session.client('inspector2', region_name=region) + client = session.client('inspector2', region_name=region, config=get_botocore_config()) members = aws_paginate(client, 'list_members', 'members') # the current host account may not be considered a "member", but we still fetch its findings diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index ec60cb48ba..443f23c4b2 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -13,6 +13,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -23,7 +24,7 @@ @timeit @aws_handle_regions def get_kms_key_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('kms', region_name=region) + client = boto3_session.client('kms', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_keys') key_list: List[Any] = [] for page in paginator.paginate(): @@ -50,7 +51,7 @@ def get_kms_key_details( """ Iterates over all KMS Keys. """ - client = boto3_session.client('kms', region_name=region) + client = boto3_session.client('kms', region_name=region, config=get_botocore_config()) for key in kms_key_data: policy = get_policy(key, client) aliases = get_aliases(key, client) diff --git a/cartography/intel/aws/lambda_function.py b/cartography/intel/aws/lambda_function.py index b9e8ca6237..467bf55f49 100644 --- a/cartography/intel/aws/lambda_function.py +++ b/cartography/intel/aws/lambda_function.py @@ -8,6 +8,7 @@ import botocore import neo4j +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -21,7 +22,7 @@ def get_lambda_data(boto3_session: boto3.session.Session, region: str) -> List[D """ Create an Lambda boto3 client and grab all the lambda functions. """ - client = boto3_session.client('lambda', region_name=region) + client = boto3_session.client('lambda', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_functions') lambda_functions = [] for page in paginator.paginate(): @@ -111,7 +112,7 @@ def get_event_source_mappings(lambda_function: Dict, client: botocore.client.Bas def get_lambda_function_details( boto3_session: boto3.session.Session, data: List[Dict], region: str, ) -> List[Tuple[str, List[Any], List[Any], List[Any]]]: - client = boto3_session.client('lambda', region_name=region) + client = boto3_session.client('lambda', region_name=region, config=get_botocore_config()) details = [] for lambda_function in data: function_aliases = get_function_aliases(lambda_function, client) diff --git a/cartography/intel/aws/organizations.py b/cartography/intel/aws/organizations.py index bfcbc9cb08..2dd5a061d7 100644 --- a/cartography/intel/aws/organizations.py +++ b/cartography/intel/aws/organizations.py @@ -5,6 +5,7 @@ import botocore.exceptions import neo4j +from cartography.intel.aws import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) @@ -16,7 +17,7 @@ def get_account_from_arn(arn: str) -> str: def get_caller_identity(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('sts') + client = boto3_session.client('sts', config=get_botocore_config()) return client.get_caller_identity() diff --git a/cartography/intel/aws/rds.py b/cartography/intel/aws/rds.py index 333766503e..3551742602 100644 --- a/cartography/intel/aws/rds.py +++ b/cartography/intel/aws/rds.py @@ -6,6 +6,7 @@ import boto3 import neo4j +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import aws_paginate @@ -24,7 +25,7 @@ def get_rds_cluster_data(boto3_session: boto3.session.Session, region: str) -> L """ Create an RDS boto3 client and grab all the DBClusters. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_db_clusters') instances: List[Any] = [] for page in paginator.paginate(): @@ -117,7 +118,7 @@ def get_rds_instance_data(boto3_session: boto3.session.Session, region: str) -> """ Create an RDS boto3 client and grab all the DBInstances. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_db_instances') instances: List[Any] = [] for page in paginator.paginate(): @@ -223,7 +224,7 @@ def get_rds_snapshot_data(boto3_session: boto3.session.Session, region: str) -> """ Create an RDS boto3 client and grab all the DBSnapshots. """ - client = boto3_session.client('rds', region_name=region) + client = boto3_session.client('rds', region_name=region, config=get_botocore_config()) return aws_paginate(client, 'describe_db_snapshots', 'DBSnapshots') diff --git a/cartography/intel/aws/redshift.py b/cartography/intel/aws/redshift.py index 290ec0535b..4e18e49ee8 100644 --- a/cartography/intel/aws/redshift.py +++ b/cartography/intel/aws/redshift.py @@ -15,7 +15,7 @@ @timeit @aws_handle_regions def get_redshift_cluster_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('redshift', region_name=region) + client = boto3_session.client('redshift', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_clusters') clusters: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/resourcegroupstaggingapi.py b/cartography/intel/aws/resourcegroupstaggingapi.py index b3444c7d8c..e9cb6f4e89 100644 --- a/cartography/intel/aws/resourcegroupstaggingapi.py +++ b/cartography/intel/aws/resourcegroupstaggingapi.py @@ -128,7 +128,7 @@ def get_tags(boto3_session: boto3.session.Session, resource_type: str, region: s if resource_type == 'iam:role': return get_role_tags(boto3_session) - client = boto3_session.client('resourcegroupstaggingapi', region_name=region) + client = boto3_session.client('resourcegroupstaggingapi', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('get_resources') resources: List[Dict] = [] for page in paginator.paginate( diff --git a/cartography/intel/aws/route53.py b/cartography/intel/aws/route53.py index f7bc2eb6bc..213a77b19e 100644 --- a/cartography/intel/aws/route53.py +++ b/cartography/intel/aws/route53.py @@ -421,7 +421,7 @@ def sync( update_tag: int, common_job_parameters: Dict, ) -> None: logger.info("Syncing Route53 for account '%s'.", current_aws_account_id) - client = boto3_session.client('route53') + client = boto3_session.client('route53', config=get_botocore_config()) zones = get_zones(client) load_dns_details(neo4j_session, zones, current_aws_account_id, update_tag) link_sub_zones(neo4j_session, update_tag) diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index 396c275926..93721fd32e 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -16,6 +16,7 @@ from botocore.exceptions import EndpointConnectionError from policyuniverse.policy import Policy +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_job @@ -30,7 +31,7 @@ @timeit def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]: - client = boto3_session.client('s3') + client = boto3_session.client('s3', config=get_botocore_config()) # NOTE no paginator available for this operation buckets = client.list_buckets() for bucket in buckets['Buckets']: @@ -66,7 +67,7 @@ async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail: # in us-east-1 region client = s3_regional_clients.get(bucket['Region']) if not client: - client = boto3_session.client('s3', bucket['Region']) + client = boto3_session.client('s3', bucket['Region'], config=get_botocore_config()) s3_regional_clients[bucket['Region']] = client ( acl, diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index 2fa5bbd0b1..155c5a30e5 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import dict_date_to_epoch from cartography.util import run_cleanup_job @@ -16,7 +17,7 @@ @timeit @aws_handle_regions def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]: - client = boto3_session.client('secretsmanager', region_name=region) + client = boto3_session.client('secretsmanager', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_secrets') secrets: List[Dict] = [] for page in paginator.paginate(): diff --git a/cartography/intel/aws/securityhub.py b/cartography/intel/aws/securityhub.py index 5dfff53cee..3fd8a4fde7 100644 --- a/cartography/intel/aws/securityhub.py +++ b/cartography/intel/aws/securityhub.py @@ -6,6 +6,7 @@ import neo4j from dateutil import parser +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit @@ -14,7 +15,7 @@ @timeit def get_hub(boto3_session: boto3.session.Session) -> Dict: - client = boto3_session.client('securityhub') + client = boto3_session.client('securityhub', config=get_botocore_config()) try: return client.describe_hub() except client.exceptions.ResourceNotFoundException: diff --git a/cartography/intel/aws/sqs.py b/cartography/intel/aws/sqs.py index 31f59a401e..5df4960fa6 100644 --- a/cartography/intel/aws/sqs.py +++ b/cartography/intel/aws/sqs.py @@ -9,6 +9,7 @@ import neo4j from botocore.exceptions import ClientError +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -19,7 +20,7 @@ @timeit @aws_handle_regions def get_sqs_queue_list(boto3_session: boto3.session.Session, region: str) -> List[str]: - client = boto3_session.client('sqs', region_name=region) + client = boto3_session.client('sqs', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('list_queues') queues: List[Any] = [] for page in paginator.paginate(): @@ -36,7 +37,7 @@ def get_sqs_queue_attributes( """ Iterates over all SQS queues. Returns a dict with url as key, and attributes as value. """ - client = boto3_session.client('sqs') + client = boto3_session.client('sqs', config=get_botocore_config()) queue_attributes = [] for queue_url in queue_urls: diff --git a/cartography/intel/aws/ssm.py b/cartography/intel/aws/ssm.py index 937cba52bf..ef45b36c4a 100644 --- a/cartography/intel/aws/ssm.py +++ b/cartography/intel/aws/ssm.py @@ -8,6 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.ec2.util import get_botocore_config from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema from cartography.util import aws_handle_regions @@ -38,7 +39,7 @@ def get_instance_information( region: str, instance_ids: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ssm', region_name=region) + client = boto3_session.client('ssm', region_name=region, config=get_botocore_config()) instance_information: List[Dict[str, Any]] = [] paginator = client.get_paginator('describe_instance_information') for i in range(0, len(instance_ids), 50): @@ -67,7 +68,7 @@ def get_instance_patches( region: str, instance_ids: List[str], ) -> List[Dict[str, Any]]: - client = boto3_session.client('ssm', region_name=region) + client = boto3_session.client('ssm', region_name=region, config=get_botocore_config()) instance_patches: List[Dict[str, Any]] = [] paginator = client.get_paginator('describe_instance_patches') for instance_id in instance_ids: From 031789351a032dd42386f92233e9b196fcc7af04 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 22:09:36 -0700 Subject: [PATCH 2/6] move get_botocore_config --- cartography/intel/aws/__init__.py | 3 +-- cartography/intel/aws/apigateway.py | 2 +- cartography/intel/aws/config.py | 2 +- cartography/intel/aws/dynamodb.py | 2 +- cartography/intel/aws/ec2/__init__.py | 2 +- cartography/intel/aws/ec2/auto_scaling_groups.py | 2 +- cartography/intel/aws/ec2/elastic_ip_addresses.py | 2 +- cartography/intel/aws/ec2/images.py | 2 +- cartography/intel/aws/ec2/instances.py | 2 +- cartography/intel/aws/ec2/internet_gateways.py | 2 +- cartography/intel/aws/ec2/key_pairs.py | 2 +- cartography/intel/aws/ec2/launch_templates.py | 2 +- cartography/intel/aws/ec2/load_balancer_v2s.py | 2 +- cartography/intel/aws/ec2/load_balancers.py | 2 +- cartography/intel/aws/ec2/network_interfaces.py | 2 +- cartography/intel/aws/ec2/reserved_instances.py | 2 +- cartography/intel/aws/ec2/security_groups.py | 2 +- cartography/intel/aws/ec2/subnets.py | 2 +- cartography/intel/aws/ec2/tgw.py | 2 +- cartography/intel/aws/ec2/util.py | 11 ----------- cartography/intel/aws/ec2/vpc.py | 2 +- cartography/intel/aws/ec2/vpc_peerings.py | 2 +- cartography/intel/aws/ecr.py | 2 +- cartography/intel/aws/elasticache.py | 2 +- cartography/intel/aws/emr.py | 2 +- cartography/intel/aws/inspector.py | 2 +- cartography/intel/aws/kms.py | 2 +- cartography/intel/aws/rds.py | 2 +- cartography/intel/aws/s3.py | 2 +- cartography/intel/aws/secretsmanager.py | 2 +- cartography/intel/aws/securityhub.py | 2 +- cartography/intel/aws/sqs.py | 2 +- cartography/intel/aws/ssm.py | 2 +- cartography/intel/aws/util/common.py | 11 +++++++++++ 34 files changed, 43 insertions(+), 44 deletions(-) delete mode 100644 cartography/intel/aws/ec2/util.py diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index ef1d9cca99..9c374a15d7 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -12,10 +12,9 @@ from . import ec2 from . import organizations -from .ec2.util import get_botocore_config from .resources import RESOURCE_FUNCTIONS from cartography.config import Config -from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs +from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs, get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_and_ensure_deps diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 0f177e036a..96a9ca486b 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -12,7 +12,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/config.py b/cartography/intel/aws/config.py index bab935dbc0..89e34126a9 100644 --- a/cartography/intel/aws/config.py +++ b/cartography/intel/aws/config.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/dynamodb.py b/cartography/intel/aws/dynamodb.py index 05f39caf98..c9596c0ae3 100644 --- a/cartography/intel/aws/dynamodb.py +++ b/cartography/intel/aws/dynamodb.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.dynamodb.gsi import DynamoDBGSISchema from cartography.models.aws.dynamodb.tables import DynamoDBTableSchema from cartography.stats import get_stats_client diff --git a/cartography/intel/aws/ec2/__init__.py b/cartography/intel/aws/ec2/__init__.py index 5e0bcd4642..001e167601 100644 --- a/cartography/intel/aws/ec2/__init__.py +++ b/cartography/intel/aws/ec2/__init__.py @@ -3,7 +3,7 @@ import boto3 -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) diff --git a/cartography/intel/aws/ec2/auto_scaling_groups.py b/cartography/intel/aws/ec2/auto_scaling_groups.py index 3e456be736..e9b5ed0151 100644 --- a/cartography/intel/aws/ec2/auto_scaling_groups.py +++ b/cartography/intel/aws/ec2/auto_scaling_groups.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/elastic_ip_addresses.py b/cartography/intel/aws/ec2/elastic_ip_addresses.py index fb9eeb35df..ddd4c20047 100644 --- a/cartography/intel/aws/ec2/elastic_ip_addresses.py +++ b/cartography/intel/aws/ec2/elastic_ip_addresses.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/images.py b/cartography/intel/aws/ec2/images.py index d5af19d657..db803493ec 100644 --- a/cartography/intel/aws/ec2/images.py +++ b/cartography/intel/aws/ec2/images.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.ec2.images import EC2ImageSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index d288c27e63..b8a234e35b 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema diff --git a/cartography/intel/aws/ec2/internet_gateways.py b/cartography/intel/aws/ec2/internet_gateways.py index 6d1b73a56b..e5caa07bf5 100644 --- a/cartography/intel/aws/ec2/internet_gateways.py +++ b/cartography/intel/aws/ec2/internet_gateways.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/key_pairs.py b/cartography/intel/aws/ec2/key_pairs.py index df6dd46219..67bb3c7095 100644 --- a/cartography/intel/aws/ec2/key_pairs.py +++ b/cartography/intel/aws/ec2/key_pairs.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.graph.job import GraphJob from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/launch_templates.py b/cartography/intel/aws/ec2/launch_templates.py index 01cd412700..17407f78ce 100644 --- a/cartography/intel/aws/ec2/launch_templates.py +++ b/cartography/intel/aws/ec2/launch_templates.py @@ -5,7 +5,7 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config +from .. import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema diff --git a/cartography/intel/aws/ec2/load_balancer_v2s.py b/cartography/intel/aws/ec2/load_balancer_v2s.py index bc4b4d55eb..a652ab7503 100644 --- a/cartography/intel/aws/ec2/load_balancer_v2s.py +++ b/cartography/intel/aws/ec2/load_balancer_v2s.py @@ -6,7 +6,7 @@ import botocore import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/load_balancers.py b/cartography/intel/aws/ec2/load_balancers.py index caf7f61f89..537ebb0be3 100644 --- a/cartography/intel/aws/ec2/load_balancers.py +++ b/cartography/intel/aws/ec2/load_balancers.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 1b6bcf069a..5bc75d3e22 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -8,7 +8,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema diff --git a/cartography/intel/aws/ec2/reserved_instances.py b/cartography/intel/aws/ec2/reserved_instances.py index 3ceac7c5b1..f357e0b709 100644 --- a/cartography/intel/aws/ec2/reserved_instances.py +++ b/cartography/intel/aws/ec2/reserved_instances.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index c0a66c8d22..abff885611 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.graph.job import GraphJob from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index d306049835..10de78af89 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.graph.job import GraphJob from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/tgw.py b/cartography/intel/aws/ec2/tgw.py index 340d879730..7f4c001c2c 100644 --- a/cartography/intel/aws/ec2/tgw.py +++ b/cartography/intel/aws/ec2/tgw.py @@ -6,7 +6,7 @@ import botocore.exceptions import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/util.py b/cartography/intel/aws/ec2/util.py deleted file mode 100644 index 1f85b05124..0000000000 --- a/cartography/intel/aws/ec2/util.py +++ /dev/null @@ -1,11 +0,0 @@ -import botocore.config - - -# TODO memoize this -def get_botocore_config() -> botocore.config.Config: - return botocore.config.Config( - read_timeout=360, - retries={ - 'max_attempts': 10, - }, - ) diff --git a/cartography/intel/aws/ec2/vpc.py b/cartography/intel/aws/ec2/vpc.py index f62194c482..1def6fe375 100644 --- a/cartography/intel/aws/ec2/vpc.py +++ b/cartography/intel/aws/ec2/vpc.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc_peerings.py b/cartography/intel/aws/ec2/vpc_peerings.py index 93e11a57a8..18248ba7eb 100644 --- a/cartography/intel/aws/ec2/vpc_peerings.py +++ b/cartography/intel/aws/ec2/vpc_peerings.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .util import get_botocore_config +from .. import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ecr.py b/cartography/intel/aws/ecr.py index 0f2e014941..2427759a2a 100644 --- a/cartography/intel/aws/ecr.py +++ b/cartography/intel/aws/ecr.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/elasticache.py b/cartography/intel/aws/elasticache.py index e9ec1b7b9e..5d9df71a2d 100644 --- a/cartography/intel/aws/elasticache.py +++ b/cartography/intel/aws/elasticache.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import merge_module_sync_metadata diff --git a/cartography/intel/aws/emr.py b/cartography/intel/aws/emr.py index 3ed964e731..dda4872ae6 100644 --- a/cartography/intel/aws/emr.py +++ b/cartography/intel/aws/emr.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.emr import EMRClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index a9e89b1c38..1cb055962e 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index 443f23c4b2..b96267cd47 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -13,7 +13,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/rds.py b/cartography/intel/aws/rds.py index 3551742602..9119f3ba3e 100644 --- a/cartography/intel/aws/rds.py +++ b/cartography/intel/aws/rds.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import aws_paginate diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index 93721fd32e..84e841607f 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -16,7 +16,7 @@ from botocore.exceptions import EndpointConnectionError from policyuniverse.policy import Policy -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_job diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index 155c5a30e5..f77ed7d126 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import dict_date_to_epoch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/securityhub.py b/cartography/intel/aws/securityhub.py index 3fd8a4fde7..8b2087b6dc 100644 --- a/cartography/intel/aws/securityhub.py +++ b/cartography/intel/aws/securityhub.py @@ -6,7 +6,7 @@ import neo4j from dateutil import parser -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/sqs.py b/cartography/intel/aws/sqs.py index 5df4960fa6..b09fbd63aa 100644 --- a/cartography/intel/aws/sqs.py +++ b/cartography/intel/aws/sqs.py @@ -9,7 +9,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ssm.py b/cartography/intel/aws/ssm.py index ef45b36c4a..1493d81b83 100644 --- a/cartography/intel/aws/ssm.py +++ b/cartography/intel/aws/ssm.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.intel.aws import get_botocore_config from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/util/common.py b/cartography/intel/aws/util/common.py index 27a9fe1b08..16da697e51 100644 --- a/cartography/intel/aws/util/common.py +++ b/cartography/intel/aws/util/common.py @@ -1,5 +1,7 @@ from typing import List +import botocore.config + from cartography.intel.aws.resources import RESOURCE_FUNCTIONS @@ -19,3 +21,12 @@ def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str f'Our full list of valid values is: {valid_syncs}.', ) return validated_resources + + +def get_botocore_config() -> botocore.config.Config: + return botocore.config.Config( + read_timeout=360, + retries={ + 'max_attempts': 10, + }, + ) From af3676634fc7a89b571453245a64b582beb18204 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 22:16:17 -0700 Subject: [PATCH 3/6] linter --- cartography/intel/aws/__init__.py | 3 ++- cartography/intel/aws/apigateway.py | 2 +- cartography/intel/aws/config.py | 2 +- cartography/intel/aws/dynamodb.py | 2 +- cartography/intel/aws/ec2/auto_scaling_groups.py | 2 +- cartography/intel/aws/ec2/elastic_ip_addresses.py | 2 +- cartography/intel/aws/ec2/images.py | 2 +- cartography/intel/aws/ec2/instances.py | 2 +- cartography/intel/aws/ec2/internet_gateways.py | 2 +- cartography/intel/aws/ec2/key_pairs.py | 2 +- cartography/intel/aws/ec2/launch_templates.py | 2 +- cartography/intel/aws/ec2/load_balancer_v2s.py | 2 +- cartography/intel/aws/ec2/load_balancers.py | 2 +- cartography/intel/aws/ec2/network_interfaces.py | 2 +- cartography/intel/aws/ec2/reserved_instances.py | 2 +- cartography/intel/aws/ec2/security_groups.py | 2 +- cartography/intel/aws/ec2/snapshots.py | 2 +- cartography/intel/aws/ec2/subnets.py | 2 +- cartography/intel/aws/ec2/tgw.py | 2 +- cartography/intel/aws/ec2/volumes.py | 2 +- cartography/intel/aws/ec2/vpc.py | 2 +- cartography/intel/aws/ec2/vpc_peerings.py | 2 +- cartography/intel/aws/ecr.py | 2 +- cartography/intel/aws/ecs.py | 2 +- cartography/intel/aws/eks.py | 2 +- cartography/intel/aws/elasticache.py | 2 +- cartography/intel/aws/emr.py | 2 +- cartography/intel/aws/iam.py | 2 +- cartography/intel/aws/inspector.py | 2 +- cartography/intel/aws/kms.py | 2 +- cartography/intel/aws/lambda_function.py | 2 +- cartography/intel/aws/organizations.py | 2 +- cartography/intel/aws/rds.py | 2 +- cartography/intel/aws/redshift.py | 1 + cartography/intel/aws/resourcegroupstaggingapi.py | 1 + cartography/intel/aws/route53.py | 1 + cartography/intel/aws/s3.py | 2 +- cartography/intel/aws/secretsmanager.py | 2 +- cartography/intel/aws/securityhub.py | 2 +- cartography/intel/aws/sqs.py | 2 +- cartography/intel/aws/ssm.py | 2 +- 41 files changed, 42 insertions(+), 38 deletions(-) diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index 9c374a15d7..9b1056002e 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -14,7 +14,8 @@ from . import organizations from .resources import RESOURCE_FUNCTIONS from cartography.config import Config -from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs, get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_and_ensure_deps diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 96a9ca486b..142cb35ee3 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -12,7 +12,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/config.py b/cartography/intel/aws/config.py index 89e34126a9..4dbdfde326 100644 --- a/cartography/intel/aws/config.py +++ b/cartography/intel/aws/config.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/dynamodb.py b/cartography/intel/aws/dynamodb.py index c9596c0ae3..0ee7092b95 100644 --- a/cartography/intel/aws/dynamodb.py +++ b/cartography/intel/aws/dynamodb.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.dynamodb.gsi import DynamoDBGSISchema from cartography.models.aws.dynamodb.tables import DynamoDBTableSchema from cartography.stats import get_stats_client diff --git a/cartography/intel/aws/ec2/auto_scaling_groups.py b/cartography/intel/aws/ec2/auto_scaling_groups.py index e9b5ed0151..fc3c71705d 100644 --- a/cartography/intel/aws/ec2/auto_scaling_groups.py +++ b/cartography/intel/aws/ec2/auto_scaling_groups.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/elastic_ip_addresses.py b/cartography/intel/aws/ec2/elastic_ip_addresses.py index ddd4c20047..63da731d04 100644 --- a/cartography/intel/aws/ec2/elastic_ip_addresses.py +++ b/cartography/intel/aws/ec2/elastic_ip_addresses.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/images.py b/cartography/intel/aws/ec2/images.py index db803493ec..ecf92c63ab 100644 --- a/cartography/intel/aws/ec2/images.py +++ b/cartography/intel/aws/ec2/images.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.images import EC2ImageSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index b8a234e35b..4c5ab9cf13 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema diff --git a/cartography/intel/aws/ec2/internet_gateways.py b/cartography/intel/aws/ec2/internet_gateways.py index e5caa07bf5..87797a303e 100644 --- a/cartography/intel/aws/ec2/internet_gateways.py +++ b/cartography/intel/aws/ec2/internet_gateways.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/key_pairs.py b/cartography/intel/aws/ec2/key_pairs.py index 67bb3c7095..e0f41c3e7a 100644 --- a/cartography/intel/aws/ec2/key_pairs.py +++ b/cartography/intel/aws/ec2/key_pairs.py @@ -5,8 +5,8 @@ import boto3 import neo4j -from .. import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/launch_templates.py b/cartography/intel/aws/ec2/launch_templates.py index 17407f78ce..b593eb9e93 100644 --- a/cartography/intel/aws/ec2/launch_templates.py +++ b/cartography/intel/aws/ec2/launch_templates.py @@ -5,9 +5,9 @@ import neo4j from botocore.exceptions import ClientError -from .. import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/load_balancer_v2s.py b/cartography/intel/aws/ec2/load_balancer_v2s.py index a652ab7503..96add71aa4 100644 --- a/cartography/intel/aws/ec2/load_balancer_v2s.py +++ b/cartography/intel/aws/ec2/load_balancer_v2s.py @@ -6,7 +6,7 @@ import botocore import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/load_balancers.py b/cartography/intel/aws/ec2/load_balancers.py index 537ebb0be3..9505a66bf3 100644 --- a/cartography/intel/aws/ec2/load_balancers.py +++ b/cartography/intel/aws/ec2/load_balancers.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 5bc75d3e22..88cf70c061 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -8,9 +8,9 @@ import boto3 import neo4j -from .. import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema diff --git a/cartography/intel/aws/ec2/reserved_instances.py b/cartography/intel/aws/ec2/reserved_instances.py index f357e0b709..60061e04f3 100644 --- a/cartography/intel/aws/ec2/reserved_instances.py +++ b/cartography/intel/aws/ec2/reserved_instances.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index abff885611..55814fe3a4 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -6,8 +6,8 @@ import boto3 import neo4j -from .. import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/snapshots.py b/cartography/intel/aws/ec2/snapshots.py index 8d88591513..b74f30ea1b 100644 --- a/cartography/intel/aws/ec2/snapshots.py +++ b/cartography/intel/aws/ec2/snapshots.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index 10de78af89..8744665bbe 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -5,8 +5,8 @@ import boto3 import neo4j -from .. import get_botocore_config from cartography.graph.job import GraphJob +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/tgw.py b/cartography/intel/aws/ec2/tgw.py index 7f4c001c2c..a4b4d3845a 100644 --- a/cartography/intel/aws/ec2/tgw.py +++ b/cartography/intel/aws/ec2/tgw.py @@ -6,7 +6,7 @@ import botocore.exceptions import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/volumes.py b/cartography/intel/aws/ec2/volumes.py index 8e9c7cfa72..64ceb043fc 100644 --- a/cartography/intel/aws/ec2/volumes.py +++ b/cartography/intel/aws/ec2/volumes.py @@ -8,8 +8,8 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config from cartography.intel.aws.util.arns import build_arn +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ec2.volumes import EBSVolumeSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc.py b/cartography/intel/aws/ec2/vpc.py index 1def6fe375..fc88f54959 100644 --- a/cartography/intel/aws/ec2/vpc.py +++ b/cartography/intel/aws/ec2/vpc.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc_peerings.py b/cartography/intel/aws/ec2/vpc_peerings.py index 18248ba7eb..c4ed622f0f 100644 --- a/cartography/intel/aws/ec2/vpc_peerings.py +++ b/cartography/intel/aws/ec2/vpc_peerings.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from .. import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ecr.py b/cartography/intel/aws/ecr.py index 2427759a2a..4310e7258e 100644 --- a/cartography/intel/aws/ecr.py +++ b/cartography/intel/aws/ecr.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ecs.py b/cartography/intel/aws/ecs.py index b123347266..97fc124f0d 100644 --- a/cartography/intel/aws/ecs.py +++ b/cartography/intel/aws/ecs.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import camel_to_snake from cartography.util import dict_date_to_epoch diff --git a/cartography/intel/aws/eks.py b/cartography/intel/aws/eks.py index 6c4caef360..6b61afde02 100644 --- a/cartography/intel/aws/eks.py +++ b/cartography/intel/aws/eks.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.eks.clusters import EKSClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/elasticache.py b/cartography/intel/aws/elasticache.py index 5d9df71a2d..8aa6543458 100644 --- a/cartography/intel/aws/elasticache.py +++ b/cartography/intel/aws/elasticache.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import merge_module_sync_metadata diff --git a/cartography/intel/aws/emr.py b/cartography/intel/aws/emr.py index dda4872ae6..afe213d1b1 100644 --- a/cartography/intel/aws/emr.py +++ b/cartography/intel/aws/emr.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.emr import EMRClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/iam.py b/cartography/intel/aws/iam.py index 65e5d6b8ac..1eca954e64 100644 --- a/cartography/intel/aws/iam.py +++ b/cartography/intel/aws/iam.py @@ -9,9 +9,9 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config from cartography.intel.aws.permission_relationships import parse_statement_node from cartography.intel.aws.permission_relationships import principal_allowed_on_resource +from cartography.intel.aws.util.common import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index 1cb055962e..29193efe2a 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index b96267cd47..c3d6967258 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -13,7 +13,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/lambda_function.py b/cartography/intel/aws/lambda_function.py index 467bf55f49..2adc956f1b 100644 --- a/cartography/intel/aws/lambda_function.py +++ b/cartography/intel/aws/lambda_function.py @@ -8,7 +8,7 @@ import botocore import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/organizations.py b/cartography/intel/aws/organizations.py index 2dd5a061d7..32e600b183 100644 --- a/cartography/intel/aws/organizations.py +++ b/cartography/intel/aws/organizations.py @@ -5,7 +5,7 @@ import botocore.exceptions import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) diff --git a/cartography/intel/aws/rds.py b/cartography/intel/aws/rds.py index 9119f3ba3e..153eb72b8f 100644 --- a/cartography/intel/aws/rds.py +++ b/cartography/intel/aws/rds.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import aws_paginate diff --git a/cartography/intel/aws/redshift.py b/cartography/intel/aws/redshift.py index 4e18e49ee8..d468db804a 100644 --- a/cartography/intel/aws/redshift.py +++ b/cartography/intel/aws/redshift.py @@ -5,6 +5,7 @@ import boto3 import neo4j +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/resourcegroupstaggingapi.py b/cartography/intel/aws/resourcegroupstaggingapi.py index e9cb6f4e89..cb4767b3e4 100644 --- a/cartography/intel/aws/resourcegroupstaggingapi.py +++ b/cartography/intel/aws/resourcegroupstaggingapi.py @@ -7,6 +7,7 @@ import neo4j from cartography.intel.aws.iam import get_role_tags +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/route53.py b/cartography/intel/aws/route53.py index 213a77b19e..76ae6d8f6c 100644 --- a/cartography/intel/aws/route53.py +++ b/cartography/intel/aws/route53.py @@ -8,6 +8,7 @@ import botocore import neo4j +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index 84e841607f..bf64aaca15 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -16,7 +16,7 @@ from botocore.exceptions import EndpointConnectionError from policyuniverse.policy import Policy -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_job diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index f77ed7d126..9d6a498279 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import dict_date_to_epoch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/securityhub.py b/cartography/intel/aws/securityhub.py index 8b2087b6dc..426d26bb75 100644 --- a/cartography/intel/aws/securityhub.py +++ b/cartography/intel/aws/securityhub.py @@ -6,7 +6,7 @@ import neo4j from dateutil import parser -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/sqs.py b/cartography/intel/aws/sqs.py index b09fbd63aa..5e38304663 100644 --- a/cartography/intel/aws/sqs.py +++ b/cartography/intel/aws/sqs.py @@ -9,7 +9,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ssm.py b/cartography/intel/aws/ssm.py index 1493d81b83..0a193bfa55 100644 --- a/cartography/intel/aws/ssm.py +++ b/cartography/intel/aws/ssm.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws import get_botocore_config +from cartography.intel.aws.util.common import get_botocore_config from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema from cartography.util import aws_handle_regions From 65bc705b32bdefeab365dff8dbf7594fee6a3928 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 23:48:08 -0700 Subject: [PATCH 4/6] unit tests --- cartography/intel/aws/__init__.py | 8 +- cartography/intel/aws/apigateway.py | 2 +- cartography/intel/aws/config.py | 2 +- cartography/intel/aws/dynamodb.py | 2 +- cartography/intel/aws/ec2/__init__.py | 2 +- .../intel/aws/ec2/auto_scaling_groups.py | 2 +- .../intel/aws/ec2/elastic_ip_addresses.py | 2 +- cartography/intel/aws/ec2/images.py | 2 +- cartography/intel/aws/ec2/instances.py | 2 +- .../intel/aws/ec2/internet_gateways.py | 2 +- cartography/intel/aws/ec2/key_pairs.py | 2 +- cartography/intel/aws/ec2/launch_templates.py | 2 +- .../intel/aws/ec2/load_balancer_v2s.py | 2 +- cartography/intel/aws/ec2/load_balancers.py | 2 +- .../intel/aws/ec2/network_interfaces.py | 2 +- .../intel/aws/ec2/reserved_instances.py | 2 +- cartography/intel/aws/ec2/security_groups.py | 2 +- cartography/intel/aws/ec2/snapshots.py | 2 +- cartography/intel/aws/ec2/subnets.py | 2 +- cartography/intel/aws/ec2/tgw.py | 2 +- cartography/intel/aws/ec2/volumes.py | 2 +- cartography/intel/aws/ec2/vpc.py | 2 +- cartography/intel/aws/ec2/vpc_peerings.py | 2 +- cartography/intel/aws/ecr.py | 2 +- cartography/intel/aws/ecs.py | 2 +- cartography/intel/aws/eks.py | 2 +- cartography/intel/aws/elasticache.py | 2 +- cartography/intel/aws/emr.py | 2 +- cartography/intel/aws/iam.py | 2 +- cartography/intel/aws/inspector.py | 2 +- cartography/intel/aws/kms.py | 2 +- cartography/intel/aws/lambda_function.py | 2 +- cartography/intel/aws/organizations.py | 2 +- cartography/intel/aws/rds.py | 2 +- cartography/intel/aws/redshift.py | 2 +- .../intel/aws/resourcegroupstaggingapi.py | 2 +- cartography/intel/aws/resources.py | 82 +++++++++---------- cartography/intel/aws/route53.py | 2 +- cartography/intel/aws/s3.py | 2 +- cartography/intel/aws/secretsmanager.py | 2 +- cartography/intel/aws/securityhub.py | 2 +- cartography/intel/aws/sqs.py | 2 +- cartography/intel/aws/ssm.py | 2 +- cartography/intel/aws/util/boto3.py | 10 +++ cartography/intel/aws/util/common.py | 11 --- .../cartography/intel/github/test_github.py | 4 +- 46 files changed, 98 insertions(+), 99 deletions(-) create mode 100644 cartography/intel/aws/util/boto3.py diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index 9b1056002e..d21ff92cb5 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -10,11 +10,11 @@ import botocore.exceptions import neo4j -from . import ec2 -from . import organizations -from .resources import RESOURCE_FUNCTIONS from cartography.config import Config -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws import ec2 +from cartography.intel.aws import organizations +from cartography.intel.aws.resources import RESOURCE_FUNCTIONS +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 142cb35ee3..36c17cbbfc 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -12,7 +12,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/config.py b/cartography/intel/aws/config.py index 4dbdfde326..142bdf775c 100644 --- a/cartography/intel/aws/config.py +++ b/cartography/intel/aws/config.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/dynamodb.py b/cartography/intel/aws/dynamodb.py index 0ee7092b95..660f54d503 100644 --- a/cartography/intel/aws/dynamodb.py +++ b/cartography/intel/aws/dynamodb.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.dynamodb.gsi import DynamoDBGSISchema from cartography.models.aws.dynamodb.tables import DynamoDBTableSchema from cartography.stats import get_stats_client diff --git a/cartography/intel/aws/ec2/__init__.py b/cartography/intel/aws/ec2/__init__.py index 001e167601..5def9d24dc 100644 --- a/cartography/intel/aws/ec2/__init__.py +++ b/cartography/intel/aws/ec2/__init__.py @@ -3,7 +3,7 @@ import boto3 -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) diff --git a/cartography/intel/aws/ec2/auto_scaling_groups.py b/cartography/intel/aws/ec2/auto_scaling_groups.py index fc3c71705d..1687fdf68d 100644 --- a/cartography/intel/aws/ec2/auto_scaling_groups.py +++ b/cartography/intel/aws/ec2/auto_scaling_groups.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/elastic_ip_addresses.py b/cartography/intel/aws/ec2/elastic_ip_addresses.py index 63da731d04..76abd59dc0 100644 --- a/cartography/intel/aws/ec2/elastic_ip_addresses.py +++ b/cartography/intel/aws/ec2/elastic_ip_addresses.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/images.py b/cartography/intel/aws/ec2/images.py index ecf92c63ab..a902a94925 100644 --- a/cartography/intel/aws/ec2/images.py +++ b/cartography/intel/aws/ec2/images.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.images import EC2ImageSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index 4c5ab9cf13..94f3e07525 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema diff --git a/cartography/intel/aws/ec2/internet_gateways.py b/cartography/intel/aws/ec2/internet_gateways.py index 87797a303e..8bd8ac7b25 100644 --- a/cartography/intel/aws/ec2/internet_gateways.py +++ b/cartography/intel/aws/ec2/internet_gateways.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/key_pairs.py b/cartography/intel/aws/ec2/key_pairs.py index e0f41c3e7a..6866ccb9a9 100644 --- a/cartography/intel/aws/ec2/key_pairs.py +++ b/cartography/intel/aws/ec2/key_pairs.py @@ -6,7 +6,7 @@ import neo4j from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/launch_templates.py b/cartography/intel/aws/ec2/launch_templates.py index b593eb9e93..7b30a1d210 100644 --- a/cartography/intel/aws/ec2/launch_templates.py +++ b/cartography/intel/aws/ec2/launch_templates.py @@ -7,7 +7,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/ec2/load_balancer_v2s.py b/cartography/intel/aws/ec2/load_balancer_v2s.py index 96add71aa4..4d200dc149 100644 --- a/cartography/intel/aws/ec2/load_balancer_v2s.py +++ b/cartography/intel/aws/ec2/load_balancer_v2s.py @@ -6,7 +6,7 @@ import botocore import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/load_balancers.py b/cartography/intel/aws/ec2/load_balancers.py index 9505a66bf3..4bbfce1cc4 100644 --- a/cartography/intel/aws/ec2/load_balancers.py +++ b/cartography/intel/aws/ec2/load_balancers.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 88cf70c061..b6a731d22c 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema diff --git a/cartography/intel/aws/ec2/reserved_instances.py b/cartography/intel/aws/ec2/reserved_instances.py index 60061e04f3..bb824c2d1c 100644 --- a/cartography/intel/aws/ec2/reserved_instances.py +++ b/cartography/intel/aws/ec2/reserved_instances.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index 55814fe3a4..f32070def0 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -7,7 +7,7 @@ import neo4j from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/snapshots.py b/cartography/intel/aws/ec2/snapshots.py index b74f30ea1b..aea1460dd3 100644 --- a/cartography/intel/aws/ec2/snapshots.py +++ b/cartography/intel/aws/ec2/snapshots.py @@ -6,7 +6,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index 8744665bbe..571ca9f905 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -6,7 +6,7 @@ import neo4j from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ec2/tgw.py b/cartography/intel/aws/ec2/tgw.py index a4b4d3845a..a0904150b9 100644 --- a/cartography/intel/aws/ec2/tgw.py +++ b/cartography/intel/aws/ec2/tgw.py @@ -6,7 +6,7 @@ import botocore.exceptions import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/volumes.py b/cartography/intel/aws/ec2/volumes.py index 64ceb043fc..23df61d33e 100644 --- a/cartography/intel/aws/ec2/volumes.py +++ b/cartography/intel/aws/ec2/volumes.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.intel.aws.util.arns import build_arn -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ec2.volumes import EBSVolumeSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc.py b/cartography/intel/aws/ec2/vpc.py index fc88f54959..a8b8665725 100644 --- a/cartography/intel/aws/ec2/vpc.py +++ b/cartography/intel/aws/ec2/vpc.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ec2/vpc_peerings.py b/cartography/intel/aws/ec2/vpc_peerings.py index c4ed622f0f..867ac48565 100644 --- a/cartography/intel/aws/ec2/vpc_peerings.py +++ b/cartography/intel/aws/ec2/vpc_peerings.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ecr.py b/cartography/intel/aws/ecr.py index 4310e7258e..461b691b46 100644 --- a/cartography/intel/aws/ecr.py +++ b/cartography/intel/aws/ecr.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/ecs.py b/cartography/intel/aws/ecs.py index 97fc124f0d..2f95f2c73d 100644 --- a/cartography/intel/aws/ecs.py +++ b/cartography/intel/aws/ecs.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import camel_to_snake from cartography.util import dict_date_to_epoch diff --git a/cartography/intel/aws/eks.py b/cartography/intel/aws/eks.py index 6b61afde02..9a4f116b6c 100644 --- a/cartography/intel/aws/eks.py +++ b/cartography/intel/aws/eks.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.eks.clusters import EKSClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/elasticache.py b/cartography/intel/aws/elasticache.py index 8aa6543458..a721f80df7 100644 --- a/cartography/intel/aws/elasticache.py +++ b/cartography/intel/aws/elasticache.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import merge_module_sync_metadata diff --git a/cartography/intel/aws/emr.py b/cartography/intel/aws/emr.py index afe213d1b1..049ad91178 100644 --- a/cartography/intel/aws/emr.py +++ b/cartography/intel/aws/emr.py @@ -10,7 +10,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.emr import EMRClusterSchema from cartography.util import aws_handle_regions from cartography.util import timeit diff --git a/cartography/intel/aws/iam.py b/cartography/intel/aws/iam.py index 1eca954e64..0759f14ee6 100644 --- a/cartography/intel/aws/iam.py +++ b/cartography/intel/aws/iam.py @@ -11,7 +11,7 @@ from cartography.intel.aws.permission_relationships import parse_statement_node from cartography.intel.aws.permission_relationships import principal_allowed_on_resource -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index 29193efe2a..8407e26355 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -9,7 +9,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index c3d6967258..fcb3c34fa8 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -13,7 +13,7 @@ from botocore.exceptions import ClientError from policyuniverse.policy import Policy -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/lambda_function.py b/cartography/intel/aws/lambda_function.py index 2adc956f1b..ee8a239208 100644 --- a/cartography/intel/aws/lambda_function.py +++ b/cartography/intel/aws/lambda_function.py @@ -8,7 +8,7 @@ import botocore import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/organizations.py b/cartography/intel/aws/organizations.py index 32e600b183..1fb11ed652 100644 --- a/cartography/intel/aws/organizations.py +++ b/cartography/intel/aws/organizations.py @@ -5,7 +5,7 @@ import botocore.exceptions import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import timeit logger = logging.getLogger(__name__) diff --git a/cartography/intel/aws/rds.py b/cartography/intel/aws/rds.py index 153eb72b8f..b2bc3cd437 100644 --- a/cartography/intel/aws/rds.py +++ b/cartography/intel/aws/rds.py @@ -6,7 +6,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import aws_handle_regions from cartography.util import aws_paginate diff --git a/cartography/intel/aws/redshift.py b/cartography/intel/aws/redshift.py index d468db804a..bb25932e59 100644 --- a/cartography/intel/aws/redshift.py +++ b/cartography/intel/aws/redshift.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/resourcegroupstaggingapi.py b/cartography/intel/aws/resourcegroupstaggingapi.py index cb4767b3e4..e5011fe83c 100644 --- a/cartography/intel/aws/resourcegroupstaggingapi.py +++ b/cartography/intel/aws/resourcegroupstaggingapi.py @@ -7,7 +7,7 @@ import neo4j from cartography.intel.aws.iam import get_role_tags -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import batch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/resources.py b/cartography/intel/aws/resources.py index 8c7879049d..52ba1aab0e 100644 --- a/cartography/intel/aws/resources.py +++ b/cartography/intel/aws/resources.py @@ -1,46 +1,46 @@ from typing import Dict -from . import apigateway -from . import config -from . import dynamodb -from . import ecr -from . import ecs -from . import eks -from . import elasticache -from . import elasticsearch -from . import emr -from . import iam -from . import inspector -from . import kms -from . import lambda_function -from . import permission_relationships -from . import rds -from . import redshift -from . import resourcegroupstaggingapi -from . import route53 -from . import s3 -from . import secretsmanager -from . import securityhub -from . import sqs -from . import ssm -from .ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups -from .ec2.elastic_ip_addresses import sync_elastic_ip_addresses -from .ec2.images import sync_ec2_images -from .ec2.instances import sync_ec2_instances -from .ec2.internet_gateways import sync_internet_gateways -from .ec2.key_pairs import sync_ec2_key_pairs -from .ec2.launch_templates import sync_ec2_launch_templates -from .ec2.load_balancer_v2s import sync_load_balancer_v2s -from .ec2.load_balancers import sync_load_balancers -from .ec2.network_interfaces import sync_network_interfaces -from .ec2.reserved_instances import sync_ec2_reserved_instances -from .ec2.security_groups import sync_ec2_security_groupinfo -from .ec2.snapshots import sync_ebs_snapshots -from .ec2.subnets import sync_subnets -from .ec2.tgw import sync_transit_gateways -from .ec2.volumes import sync_ebs_volumes -from .ec2.vpc import sync_vpc -from .ec2.vpc_peerings import sync_vpc_peerings +from cartography.intel.aws import apigateway +from cartography.intel.aws import config +from cartography.intel.aws import dynamodb +from cartography.intel.aws import ecr +from cartography.intel.aws import ecs +from cartography.intel.aws import eks +from cartography.intel.aws import elasticache +from cartography.intel.aws import elasticsearch +from cartography.intel.aws import emr +from cartography.intel.aws import iam +from cartography.intel.aws import inspector +from cartography.intel.aws import kms +from cartography.intel.aws import lambda_function +from cartography.intel.aws import permission_relationships +from cartography.intel.aws import rds +from cartography.intel.aws import redshift +from cartography.intel.aws import resourcegroupstaggingapi +from cartography.intel.aws import route53 +from cartography.intel.aws import s3 +from cartography.intel.aws import secretsmanager +from cartography.intel.aws import securityhub +from cartography.intel.aws import sqs +from cartography.intel.aws import ssm +from cartography.intel.aws.ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups +from cartography.intel.aws.ec2.elastic_ip_addresses import sync_elastic_ip_addresses +from cartography.intel.aws.ec2.images import sync_ec2_images +from cartography.intel.aws.ec2.instances import sync_ec2_instances +from cartography.intel.aws.ec2.internet_gateways import sync_internet_gateways +from cartography.intel.aws.ec2.key_pairs import sync_ec2_key_pairs +from cartography.intel.aws.ec2.launch_templates import sync_ec2_launch_templates +from cartography.intel.aws.ec2.load_balancer_v2s import sync_load_balancer_v2s +from cartography.intel.aws.ec2.load_balancers import sync_load_balancers +from cartography.intel.aws.ec2.network_interfaces import sync_network_interfaces +from cartography.intel.aws.ec2.reserved_instances import sync_ec2_reserved_instances +from cartography.intel.aws.ec2.security_groups import sync_ec2_security_groupinfo +from cartography.intel.aws.ec2.snapshots import sync_ebs_snapshots +from cartography.intel.aws.ec2.subnets import sync_subnets +from cartography.intel.aws.ec2.tgw import sync_transit_gateways +from cartography.intel.aws.ec2.volumes import sync_ebs_volumes +from cartography.intel.aws.ec2.vpc import sync_vpc +from cartography.intel.aws.ec2.vpc_peerings import sync_vpc_peerings RESOURCE_FUNCTIONS: Dict = { 'iam': iam.sync, diff --git a/cartography/intel/aws/route53.py b/cartography/intel/aws/route53.py index 76ae6d8f6c..f2c5289023 100644 --- a/cartography/intel/aws/route53.py +++ b/cartography/intel/aws/route53.py @@ -8,7 +8,7 @@ import botocore import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index bf64aaca15..e199892296 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -16,7 +16,7 @@ from botocore.exceptions import EndpointConnectionError from policyuniverse.policy import Policy -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.stats import get_stats_client from cartography.util import merge_module_sync_metadata from cartography.util import run_analysis_job diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index 9d6a498279..f1b3a1117f 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -5,7 +5,7 @@ import boto3 import neo4j -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import dict_date_to_epoch from cartography.util import run_cleanup_job diff --git a/cartography/intel/aws/securityhub.py b/cartography/intel/aws/securityhub.py index 426d26bb75..d0b1fb1c30 100644 --- a/cartography/intel/aws/securityhub.py +++ b/cartography/intel/aws/securityhub.py @@ -6,7 +6,7 @@ import neo4j from dateutil import parser -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/sqs.py b/cartography/intel/aws/sqs.py index 5e38304663..0716ca7bc3 100644 --- a/cartography/intel/aws/sqs.py +++ b/cartography/intel/aws/sqs.py @@ -9,7 +9,7 @@ import neo4j from botocore.exceptions import ClientError -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit diff --git a/cartography/intel/aws/ssm.py b/cartography/intel/aws/ssm.py index 0a193bfa55..fb5b471e9e 100644 --- a/cartography/intel/aws/ssm.py +++ b/cartography/intel/aws/ssm.py @@ -8,7 +8,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.util.common import get_botocore_config +from cartography.intel.aws.util.boto3 import get_botocore_config from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema from cartography.util import aws_handle_regions diff --git a/cartography/intel/aws/util/boto3.py b/cartography/intel/aws/util/boto3.py new file mode 100644 index 0000000000..ecebb5afd3 --- /dev/null +++ b/cartography/intel/aws/util/boto3.py @@ -0,0 +1,10 @@ +import botocore.config + + +def get_botocore_config() -> botocore.config.Config: + return botocore.config.Config( + read_timeout=360, + retries={ + 'max_attempts': 10, + }, + ) diff --git a/cartography/intel/aws/util/common.py b/cartography/intel/aws/util/common.py index 16da697e51..27a9fe1b08 100644 --- a/cartography/intel/aws/util/common.py +++ b/cartography/intel/aws/util/common.py @@ -1,7 +1,5 @@ from typing import List -import botocore.config - from cartography.intel.aws.resources import RESOURCE_FUNCTIONS @@ -21,12 +19,3 @@ def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str f'Our full list of valid values is: {valid_syncs}.', ) return validated_resources - - -def get_botocore_config() -> botocore.config.Config: - return botocore.config.Config( - read_timeout=360, - retries={ - 'max_attempts': 10, - }, - ) diff --git a/tests/unit/cartography/intel/github/test_github.py b/tests/unit/cartography/intel/github/test_github.py index 0a2ffd5c05..a90587d355 100644 --- a/tests/unit/cartography/intel/github/test_github.py +++ b/tests/unit/cartography/intel/github/test_github.py @@ -16,7 +16,7 @@ from tests.data.github.rate_limit import RATE_LIMIT_RESPONSE_JSON -@patch('cartography.intel.github.util.handle_rate_limit_sleep') +@patch('time.sleep') @patch('cartography.intel.github.util.fetch_page') def test_fetch_all_handles_retries( mock_fetch_page: Mock, @@ -35,7 +35,7 @@ def test_fetch_all_handles_retries( with pytest.raises(exception) as excinfo: fetch_all('my-token', 'my-api_url', 'my-org', 'my-query', 'my-resource', retries=retries) # Assert - assert mock_handle_rate_limit_sleep.call_count == retries + assert mock_handle_rate_limit_sleep.call_count == retries - 1 # Sleep is called one less than retries assert mock_fetch_page.call_count == retries assert 'my-error' in str(excinfo.value) From ab5caad1da86da47a2dfa777a516f4a525fabbe5 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 23:48:23 -0700 Subject: [PATCH 5/6] unrelated change: correctly patch out time.sleep in github test --- .../cartography/intel/github/test_github.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/cartography/intel/github/test_github.py b/tests/unit/cartography/intel/github/test_github.py index a90587d355..78ede421e3 100644 --- a/tests/unit/cartography/intel/github/test_github.py +++ b/tests/unit/cartography/intel/github/test_github.py @@ -10,6 +10,8 @@ from requests import Response from requests.exceptions import HTTPError +import cartography.intel.github.util + from cartography.intel.github.util import _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD from cartography.intel.github.util import fetch_all from cartography.intel.github.util import handle_rate_limit_sleep @@ -17,25 +19,23 @@ @patch('time.sleep') -@patch('cartography.intel.github.util.fetch_page') +@patch('cartography.intel.github.util.handle_rate_limit_sleep') +@patch('cartography.intel.github.util.fetch_page', side_effect=HTTPError('my-error', response=Response())) def test_fetch_all_handles_retries( - mock_fetch_page: Mock, - mock_handle_rate_limit_sleep: Mock, + mock_fetch_page: Mock, + mock_handle_rate_limit_sleep: Mock, + mock_sleep: Mock, ) -> None: ''' Ensures that fetch_all re-reaises the same exceptions when exceeding retry limit ''' # Arrange - exception = HTTPError - response = Response() - response.status_code = 500 - mock_fetch_page.side_effect = exception('my-error', response=response) retries = 3 # Act - with pytest.raises(exception) as excinfo: + with pytest.raises(Exception) as excinfo: fetch_all('my-token', 'my-api_url', 'my-org', 'my-query', 'my-resource', retries=retries) # Assert - assert mock_handle_rate_limit_sleep.call_count == retries - 1 # Sleep is called one less than retries + assert mock_handle_rate_limit_sleep.call_count == retries assert mock_fetch_page.call_count == retries assert 'my-error' in str(excinfo.value) From aefccca0de7b3b754728da06968a8f920d18ffaa Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 22 Jul 2024 23:48:52 -0700 Subject: [PATCH 6/6] linter --- tests/unit/cartography/intel/github/test_github.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/cartography/intel/github/test_github.py b/tests/unit/cartography/intel/github/test_github.py index 78ede421e3..500f842aea 100644 --- a/tests/unit/cartography/intel/github/test_github.py +++ b/tests/unit/cartography/intel/github/test_github.py @@ -10,8 +10,6 @@ from requests import Response from requests.exceptions import HTTPError -import cartography.intel.github.util - from cartography.intel.github.util import _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD from cartography.intel.github.util import fetch_all from cartography.intel.github.util import handle_rate_limit_sleep