From e9d67aff4fe710bbeaf3c15ef0f21fea6f09d6a5 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 29 Oct 2022 19:38:01 +0000 Subject: [PATCH 01/18] pre-commit: add codespell, detect-secrets, semgrep, bandit, (commented out black) --- .pre-commit-config.yaml | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c96838d8ee..1602d7a28d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,8 @@ repos: - id: debug-statements - id: end-of-file-fixer - id: trailing-whitespace + - id: detect-private-key + - id: check-added-large-files - repo: https://gitlab.com/pycqa/flake8 rev: 3.9.2 hooks: @@ -46,3 +48,43 @@ repos: - types-requests - types-PyYAML - types-python-dateutil +#- repo: https://github.com/psf/black +# rev: 20.8b1 +# hooks: +# - id: black +#- repo: https://github.com/asottile/blacken-docs +# rev: v1.8.0 +# hooks: +# - id: blacken-docs +# additional_dependencies: [black==20.8b1] +- repo: https://github.com/codespell-project/codespell + rev: v2.2.1 + hooks: + - id: codespell + args: [-I, .codespellignore] +- repo: git@github.com:Yelp/detect-secrets + rev: v1.4.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + # exclude: .*/tests/.* +- repo: https://github.com/returntocorp/semgrep + rev: v0.118.0 + hooks: + - id: semgrep + name: Semgrep Python + types: [python] + exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$" + args: ["--config", "https://semgrep.dev/p/python", "--error"] + - id: semgrep + name: Semgrep Bandit + types: [python] + exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$" +- repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + exclude: ^tests/ + args: + - -s + - B101 From 3245e2faee709ba63a8b896593e06184986a5855 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 29 Oct 2022 19:42:00 +0000 Subject: [PATCH 02/18] add few nosemgrep --- cartography/cli.py | 10 +++++----- cartography/driftdetect/cli.py | 4 ++-- cartography/intel/aws/secretsmanager.py | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cartography/cli.py b/cartography/cli.py index 81fa2d7041..b06108929a 100644 --- a/cartography/cli.py +++ b/cartography/cli.py @@ -398,7 +398,7 @@ def _build_parser(self): default=False, action='store_true', help=( - 'enable the experimental suppor for neo4j 4.x. Can also be enabled by environment variable. ' + 'enable the experimental support for neo4j 4.x. Can also be enabled by environment variable. ' 'See cartography.__init__.py' ), ) @@ -425,10 +425,10 @@ def main(self, argv: str) -> int: if config.neo4j_user: config.neo4j_password = None if config.neo4j_password_prompt: - logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user) + logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user) # nosemgrep config.neo4j_password = getpass.getpass() elif config.neo4j_password_env_var: - logger.debug( + logger.debug( # nosemgrep "Reading password for Neo4j user '%s' from environment variable '%s'.", config.neo4j_user, config.neo4j_password_env_var, @@ -446,7 +446,7 @@ def main(self, argv: str) -> int: # Azure config if config.azure_sp_auth and config.azure_client_secret_env_var: - logger.debug( + logger.debug( # nosemgrep "Reading Client Secret for Azure Service Principal Authentication from environment variable %s", config.azure_client_secret_env_var, ) @@ -487,7 +487,7 @@ def main(self, argv: str) -> int: if config.jamf_user: config.jamf_password = None if config.jamf_password_env_var: - logger.debug( + logger.debug( # nosemgrep "Reading password for Jamf user '%s' from environment variable '%s'.", config.jamf_user, config.jamf_password_env_var, diff --git a/cartography/driftdetect/cli.py b/cartography/driftdetect/cli.py index 16c79600ce..f701353780 100644 --- a/cartography/driftdetect/cli.py +++ b/cartography/driftdetect/cli.py @@ -215,10 +215,10 @@ def configure_get_state_neo4j(config): if config.neo4j_user: config.neo4j_password = None if config.neo4j_password_prompt: - logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user) + logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user) # nosemgrep config.neo4j_password = getpass.getpass() elif config.neo4j_password_env_var: - logger.debug( + logger.debug( # nosemgrep "Reading password for Neo4j user '%s' from environment variable '%s'.", config.neo4j_user, config.neo4j_password_env_var, diff --git a/cartography/intel/aws/secretsmanager.py b/cartography/intel/aws/secretsmanager.py index a8cffdd190..75ed6e5564 100644 --- a/cartography/intel/aws/secretsmanager.py +++ b/cartography/intel/aws/secretsmanager.py @@ -77,7 +77,10 @@ def sync( update_tag: int, common_job_parameters: Dict, ) -> None: for region in regions: - logger.info("Syncing Secrets Manager for region '%s' in account '%s'.", region, current_aws_account_id) + logger.info( # nosemgrep + "Syncing Secrets Manager for region '%s' in account '%s'.", + region, current_aws_account_id, + ) secrets = get_secret_list(boto3_session, region) load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag) cleanup_secrets(neo4j_session, common_job_parameters) From 971d602da21e67825f1767ecbd92ffcf5bf92932 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 16:22:22 +0000 Subject: [PATCH 03/18] add codespellignore --- .codespellignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .codespellignore diff --git a/.codespellignore b/.codespellignore new file mode 100644 index 0000000000..e69de29bb2 From f70c4fd3294106ec27795481c498b571291e1356 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 16:22:45 +0000 Subject: [PATCH 04/18] add .secrets-baseline for detect-secrets --- .secrets.baseline | 204 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 .secrets.baseline diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000000..404948c8cb --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,204 @@ +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": { + "docs/root/modules/gcp/schema.md": [ + { + "type": "Basic Auth Credentials", + "filename": "docs/root/modules/gcp/schema.md", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "is_verified": false, + "line_number": 555 + } + ], + "tests/data/aws/apigateway.py": [ + { + "type": "Secret Keyword", + "filename": "tests/data/aws/apigateway.py", + "hashed_secret": "d3053d5db9cc8cb93b26db3c26c76bdfdff06ace", + "is_verified": false, + "line_number": 20 + } + ], + "tests/data/aws/ec2/load_balancers.py": [ + { + "type": "Secret Keyword", + "filename": "tests/data/aws/ec2/load_balancers.py", + "hashed_secret": "ecb252044b5ea0f679ee78ec1a12904739e2904d", + "is_verified": false, + "line_number": 48 + } + ], + "tests/data/aws/iam/role_policies.py": [ + { + "type": "Base64 High Entropy String", + "filename": "tests/data/aws/iam/role_policies.py", + "hashed_secret": "43ecf08db7dbbf0212e3596e6ddd216f745cb68e", + "is_verified": false, + "line_number": 42 + } + ], + "tests/data/crxcavator/crxcavator.py": [ + { + "type": "Hex High Entropy String", + "filename": "tests/data/crxcavator/crxcavator.py", + "hashed_secret": "040b9fac2febf877a3b4e1b406f639ff5e5ff509", + "is_verified": false, + "line_number": 85 + } + ], + "tests/data/okta/groups.py": [ + { + "type": "Secret Keyword", + "filename": "tests/data/okta/groups.py", + "hashed_secret": "df4b7a1b3f8b7d7559cf710ab999f28a9bfeb6fc", + "is_verified": false, + "line_number": 14 + }, + { + "type": "Secret Keyword", + "filename": "tests/data/okta/groups.py", + "hashed_secret": "069cf67d25b36dd1210e4afd8daf21cbd9ecc2c1", + "is_verified": false, + "line_number": 43 + }, + { + "type": "Secret Keyword", + "filename": "tests/data/okta/groups.py", + "hashed_secret": "db1a4d8672d5b5c1a7d82fcb4764529b53db5129", + "is_verified": false, + "line_number": 95 + }, + { + "type": "Secret Keyword", + "filename": "tests/data/okta/groups.py", + "hashed_secret": "e52a05b42953592d8d2a32414cbad7d1d34a981e", + "is_verified": false, + "line_number": 135 + }, + { + "type": "Secret Keyword", + "filename": "tests/data/okta/groups.py", + "hashed_secret": "8ed33b56d80640b82aa34c1c4c1ec500b30f3471", + "is_verified": false, + "line_number": 174 + } + ], + "tests/integration/cartography/intel/crxcavator/test_crxcavator.py": [ + { + "type": "Hex High Entropy String", + "filename": "tests/integration/cartography/intel/crxcavator/test_crxcavator.py", + "hashed_secret": "040b9fac2febf877a3b4e1b406f639ff5e5ff509", + "is_verified": false, + "line_number": 111 + } + ] + }, + "generated_at": "2023-01-07T16:20:21Z" +} From f241f0f3bf888ad060b5e95d6beb48fb32a1b06f Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 16:39:53 +0000 Subject: [PATCH 05/18] fix pre-commit detect-secrets repo --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f3c64036a..389a196be2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: hooks: - id: codespell args: [-I, .codespellignore] -- repo: git@github.com:Yelp/detect-secrets +- repo: https://github.com/Yelp/detect-secrets rev: v1.4.0 hooks: - id: detect-secrets From 6e10ac899884c4874b0ae2da99f8fdd65013bba0 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 17:22:07 +0000 Subject: [PATCH 06/18] update codespellignore --- .codespellignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codespellignore b/.codespellignore index e69de29bb2..e479c66089 100644 --- a/.codespellignore +++ b/.codespellignore @@ -0,0 +1,2 @@ +# Business Unit +bu From 946bbebef65cedfb8a0d92ed73bf9945501dfe40 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 17:23:59 +0000 Subject: [PATCH 07/18] update codespellignore - rapid7 --- .codespellignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codespellignore b/.codespellignore index e479c66089..c10614ea8e 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,2 +1,4 @@ # Business Unit bu +# Rapid7 SQL query +ot From 9584db83b727626610335920d48c5f9fce43d679 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sat, 7 Jan 2023 18:45:30 +0000 Subject: [PATCH 08/18] use yaml.SafeLoader - fix bandit --- cartography/intel/aws/permission_relationships.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartography/intel/aws/permission_relationships.py b/cartography/intel/aws/permission_relationships.py index 27253debb7..8a7af04f70 100644 --- a/cartography/intel/aws/permission_relationships.py +++ b/cartography/intel/aws/permission_relationships.py @@ -333,7 +333,7 @@ def parse_permission_relationships_file(file_path: str) -> List[Any]: if not os.path.isabs(file_path): file_path = os.path.join(os.getcwd(), file_path) with open(file_path) as f: - relationship_mapping = yaml.load(f, Loader=yaml.FullLoader) + relationship_mapping = yaml.load(f, Loader=yaml.SafeLoader) return relationship_mapping except FileNotFoundError: logger.warning( From 68b2c38c2d52f7793c6fcb957a12614877848d7e Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 16:46:23 -0400 Subject: [PATCH 09/18] refactor: fix cosmetic formatting --- .pre-commit-config.yaml | 90 ++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ddcbe56c5..c140435f48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,74 +1,74 @@ default_language_version: - # force all unspecified python hooks to run python3 - python: python3 + # force all unspecified python hooks to run python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-merge-conflict - - id: check-vcs-permalinks - - id: check-yaml - - id: debug-statements - - id: end-of-file-fixer - - id: trailing-whitespace - - id: detect-private-key - - id: check-added-large-files -- repo: https://github.com/pycqa/flake8 + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-private-key + - id: check-added-large-files + - repo: https://github.com/pycqa/flake8 rev: 6.0.0 hooks: - - id: flake8 -- repo: https://github.com/pre-commit/mirrors-autopep8 + - id: flake8 + - repo: https://github.com/pre-commit/mirrors-autopep8 rev: v2.0.1 hooks: - - id: autopep8 + - id: autopep8 # disable a few rewrites which will cause autopep8 to reflow - args: [--in-place, '--ignore=E265,E501,W504'] -- repo: https://github.com/asottile/pyupgrade + args: [--in-place, "--ignore=E265,E501,W504"] + - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 hooks: - - id: pyupgrade + - id: pyupgrade args: [--py36-plus] -- repo: https://github.com/asottile/add-trailing-comma + - repo: https://github.com/asottile/add-trailing-comma rev: v2.4.0 hooks: - - id: add-trailing-comma + - id: add-trailing-comma args: [--py36-plus] -- repo: https://github.com/asottile/reorder_python_imports + - repo: https://github.com/asottile/reorder_python_imports rev: v3.9.0 hooks: - - id: reorder-python-imports + - id: reorder-python-imports args: [--py3-plus] -- repo: https://github.com/pre-commit/mirrors-mypy + - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.0.0 hooks: - - id: mypy + - id: mypy additional_dependencies: - - types-requests - - types-PyYAML - - types-python-dateutil -#- repo: https://github.com/psf/black -# rev: 20.8b1 -# hooks: -# - id: black -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.8.0 -# hooks: -# - id: blacken-docs -# additional_dependencies: [black==20.8b1] -- repo: https://github.com/codespell-project/codespell + - types-requests + - types-PyYAML + - types-python-dateutil + #- repo: https://github.com/psf/black + # rev: 20.8b1 + # hooks: + # - id: black + #- repo: https://github.com/asottile/blacken-docs + # rev: v1.8.0 + # hooks: + # - id: blacken-docs + # additional_dependencies: [black==20.8b1] + - repo: https://github.com/codespell-project/codespell rev: v2.2.1 hooks: - id: codespell args: [-I, .codespellignore] -- repo: https://github.com/Yelp/detect-secrets + - repo: https://github.com/Yelp/detect-secrets rev: v1.4.0 hooks: - id: detect-secrets - args: ['--baseline', '.secrets.baseline'] + args: ["--baseline", ".secrets.baseline"] # exclude: .*/tests/.* -- repo: https://github.com/returntocorp/semgrep + - repo: https://github.com/returntocorp/semgrep rev: v0.118.0 hooks: - id: semgrep @@ -81,11 +81,11 @@ repos: types: [python] exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$" args: ["--config", "https://semgrep.dev/p/bandit", "--error"] -- repo: https://github.com/PyCQA/bandit + - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: - id: bandit exclude: ^tests/ args: - - -s - - B101 + - -s + - B101 From 84d5e6853a55f5b09d1e004877a5169ec95f35b7 Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 16:52:25 -0400 Subject: [PATCH 10/18] build: update to latest available versions --- .pre-commit-config.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c140435f48..bd57ed3a32 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-docstring-first - id: check-executables-have-shebangs @@ -16,32 +16,32 @@ repos: - id: detect-private-key - id: check-added-large-files - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.1.0 hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.1 + rev: v2.0.4 hooks: - id: autopep8 # disable a few rewrites which will cause autopep8 to reflow args: [--in-place, "--ignore=E265,E501,W504"] - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.16.0 hooks: - id: pyupgrade args: [--py36-plus] - repo: https://github.com/asottile/add-trailing-comma - rev: v2.4.0 + rev: v3.1.0 hooks: - id: add-trailing-comma args: [--py36-plus] - repo: https://github.com/asottile/reorder_python_imports - rev: v3.9.0 + rev: v3.13.0 hooks: - id: reorder-python-imports args: [--py3-plus] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.0.0 + rev: v1.10.1 hooks: - id: mypy additional_dependencies: @@ -49,27 +49,27 @@ repos: - types-PyYAML - types-python-dateutil #- repo: https://github.com/psf/black - # rev: 20.8b1 + # rev: 24.4.2 # hooks: # - id: black #- repo: https://github.com/asottile/blacken-docs - # rev: v1.8.0 + # rev: v1.18.0 # hooks: # - id: blacken-docs # additional_dependencies: [black==20.8b1] - repo: https://github.com/codespell-project/codespell - rev: v2.2.1 + rev: v2.3.0 hooks: - id: codespell args: [-I, .codespellignore] - repo: https://github.com/Yelp/detect-secrets - rev: v1.4.0 + rev: v1.5.0 hooks: - id: detect-secrets args: ["--baseline", ".secrets.baseline"] # exclude: .*/tests/.* - - repo: https://github.com/returntocorp/semgrep - rev: v0.118.0 + - repo: https://github.com/semgrep/semgrep + rev: v1.78.0 hooks: - id: semgrep name: Semgrep Python @@ -82,7 +82,7 @@ repos: exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$" args: ["--config", "https://semgrep.dev/p/bandit", "--error"] - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.9 hooks: - id: bandit exclude: ^tests/ From f78dafede32cfded1664949b3203724b2b6f299b Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:01:32 -0400 Subject: [PATCH 11/18] build: baseline with v1.5.0 --- .secrets.baseline | 64 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 404948c8cb..c9c6ce052d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,5 +1,5 @@ { - "version": "1.4.0", + "version": "1.5.0", "plugins_used": [ { "name": "ArtifactoryDetector" @@ -26,6 +26,9 @@ { "name": "GitHubTokenDetector" }, + { + "name": "GitLabTokenDetector" + }, { "name": "HexHighEntropyString", "limit": 3.0 @@ -36,6 +39,9 @@ { "name": "IbmCosHmacDetector" }, + { + "name": "IPPublicDetector" + }, { "name": "JwtTokenDetector" }, @@ -49,9 +55,15 @@ { "name": "NpmDetector" }, + { + "name": "OpenAIDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "PypiTokenDetector" + }, { "name": "SendGridDetector" }, @@ -67,6 +79,9 @@ { "name": "StripeDetector" }, + { + "name": "TelegramBotTokenDetector" + }, { "name": "TwilioKeyDetector" } @@ -117,6 +132,15 @@ "line_number": 555 } ], + "docs/root/modules/gsuite/config.md": [ + { + "type": "Secret Keyword", + "filename": "docs/root/modules/gsuite/config.md", + "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114", + "is_verified": false, + "line_number": 65 + } + ], "tests/data/aws/apigateway.py": [ { "type": "Secret Keyword", @@ -153,6 +177,15 @@ "line_number": 85 } ], + "tests/data/duo/endpoints.py": [ + { + "type": "Secret Keyword", + "filename": "tests/data/duo/endpoints.py", + "hashed_secret": "bc7819b34ff87570745fbe461e36a16f80e562ce", + "is_verified": false, + "line_number": 27 + } + ], "tests/data/okta/groups.py": [ { "type": "Secret Keyword", @@ -198,7 +231,34 @@ "is_verified": false, "line_number": 111 } + ], + "tests/integration/cartography/intel/cve/test_feed.py": [ + { + "type": "Secret Keyword", + "filename": "tests/integration/cartography/intel/cve/test_feed.py", + "hashed_secret": "aeaf4b967f1b127a4b590ce5e40132c47a90b414", + "is_verified": false, + "line_number": 8 + } + ], + "tests/integration/cartography/intel/github/test_teams.py": [ + { + "type": "Secret Keyword", + "filename": "tests/integration/cartography/intel/github/test_teams.py", + "hashed_secret": "3da541559918a808c2402bba5012f6c60b27661c", + "is_verified": false, + "line_number": 14 + } + ], + "tests/unit/cartography/intel/cve/test_feed.py": [ + { + "type": "Secret Keyword", + "filename": "tests/unit/cartography/intel/cve/test_feed.py", + "hashed_secret": "aeaf4b967f1b127a4b590ce5e40132c47a90b414", + "is_verified": false, + "line_number": 18 + } ] }, - "generated_at": "2023-01-07T16:20:21Z" + "generated_at": "2024-06-30T21:00:49Z" } From e58452eb0d9304e23e75a995fee4eb84cde356bf Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:13:36 -0400 Subject: [PATCH 12/18] fix: Requests call without timeout --- cartography/intel/azure/util/credentials.py | 9 ++++++++- cartography/intel/github/util.py | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cartography/intel/azure/util/credentials.py b/cartography/intel/azure/util/credentials.py index 9d5e5bf431..acb94b9617 100644 --- a/cartography/intel/azure/util/credentials.py +++ b/cartography/intel/azure/util/credentials.py @@ -15,6 +15,9 @@ logger = logging.getLogger(__name__) AUTHORITY_HOST_URI = 'https://login.microsoftonline.com' +# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts +_TIMEOUT = (60, 60) + class Credentials: @@ -46,7 +49,11 @@ def get_tenant_id(self) -> Any: # This is a last resort, e.g. for MSI authentication try: h = {'Authorization': 'Bearer {}'.format(self.arm_credentials.token['access_token'])} - r = requests.get('https://management.azure.com/tenants?api-version=2020-01-01', headers=h) + r = requests.get( + 'https://management.azure.com/tenants?api-version=2020-01-01', + headers=h, + timeout=_TIMEOUT + ) r2 = r.json() return r2.get('value')[0].get('tenantId') except requests.ConnectionError as e: diff --git a/cartography/intel/github/util.py b/cartography/intel/github/util.py index bd73ef2f4f..2511e09001 100644 --- a/cartography/intel/github/util.py +++ b/cartography/intel/github/util.py @@ -30,7 +30,11 @@ def handle_rate_limit_sleep(token: str) -> None: Check the remaining rate limit and sleep if remaining is below threshold :param token: The Github API token as string. ''' - response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': f"token {token}"}) + response = requests.get( + 'https://api.github.com/rate_limit', + headers={'Authorization': f"token {token}"}, + timeout=_TIMEOUT, + ) response.raise_for_status() response_json = response.json() rate_limit_obj = response_json['resources']['graphql'] From 381491bb8b322f5e9c7311f49f7a4d4f98117c1e Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:22:34 -0400 Subject: [PATCH 13/18] fix: verify TLS cert --- cartography/intel/bigfix/computers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cartography/intel/bigfix/computers.py b/cartography/intel/bigfix/computers.py index 4eece053ec..c0a9191537 100644 --- a/cartography/intel/bigfix/computers.py +++ b/cartography/intel/bigfix/computers.py @@ -133,14 +133,14 @@ def get_computer_details( def _get_computer_list_raw_xml(bigfix_api_url: str, headers: Dict[str, str]) -> str: list_endpoint = f"{bigfix_api_url}/api/computers" - resp = requests.get(list_endpoint, headers=headers, verify=False, timeout=_TIMEOUT) + resp = requests.get(list_endpoint, headers=headers, timeout=_TIMEOUT) resp.raise_for_status() return resp.text def _get_computer_details_raw_xml(bigfix_api_url: str, headers: Dict[str, str], computer_id: str) -> str: details_endpoint = f"{bigfix_api_url}/api/computer/{computer_id}" - resp = requests.get(details_endpoint, headers=headers, verify=False, timeout=_TIMEOUT) + resp = requests.get(details_endpoint, headers=headers, timeout=_TIMEOUT) resp.raise_for_status() return resp.text From a48812be0eb20deff4edf32e40f6f97967a53d7e Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:22:59 -0400 Subject: [PATCH 14/18] lint: add trailing comma --- cartography/intel/azure/util/credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartography/intel/azure/util/credentials.py b/cartography/intel/azure/util/credentials.py index acb94b9617..4f08cf818a 100644 --- a/cartography/intel/azure/util/credentials.py +++ b/cartography/intel/azure/util/credentials.py @@ -52,7 +52,7 @@ def get_tenant_id(self) -> Any: r = requests.get( 'https://management.azure.com/tenants?api-version=2020-01-01', headers=h, - timeout=_TIMEOUT + timeout=_TIMEOUT, ) r2 = r.json() return r2.get('value')[0].get('tenantId') From fb5682ec2e02623056f0884aabff37aa974d8baf Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:24:17 -0400 Subject: [PATCH 15/18] lint: use is for type comparison --- tests/unit/cartography/intel/aws/iam/test_iam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cartography/intel/aws/iam/test_iam.py b/tests/unit/cartography/intel/aws/iam/test_iam.py index 7486a5f545..c06c0fd4ae 100644 --- a/tests/unit/cartography/intel/aws/iam/test_iam.py +++ b/tests/unit/cartography/intel/aws/iam/test_iam.py @@ -115,4 +115,4 @@ def test_transform_policy_data_correctly_creates_lists_of_statements(): transform_policy_data(pol_statement_map, PolicyType.inline.value) # Assert that we correctly converted the statement to a list - assert type(pol_statement_map['some-arn']['pol-name']) == list + assert type(pol_statement_map['some-arn']['pol-name']) is list From 4aaabeac17514523a473c0c2a8437b5b7cb4e7b5 Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:43:46 -0400 Subject: [PATCH 16/18] ci: ignore words from codespell --- .codespellignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.codespellignore b/.codespellignore index c10614ea8e..a29719e21a 100644 --- a/.codespellignore +++ b/.codespellignore @@ -2,3 +2,7 @@ bu # Rapid7 SQL query ot +# Pylint dependency package +astroid +# Marge Simpson +Marge From 0b34e234d5a6a4272acbfbdbe883fbac1caac4e7 Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 17:48:21 -0400 Subject: [PATCH 17/18] lint: fix spelling errors --- README.md | 2 +- .../data/jobs/analysis/aws_ec2_iaminstance.json | 2 +- .../jobs/analysis/aws_ec2_iaminstanceprofile.json | 2 +- .../data/jobs/analysis/aws_ec2_keypair_analysis.json | 2 +- cartography/data/jobs/analysis/aws_lambda_ecr.json | 2 +- .../jobs/cleanup/crowdstrike_import_cleanup.json | 2 +- .../scoped_analysis/semgrep_sca_risk_analysis.json | 2 +- cartography/intel/aws/apigateway.py | 2 +- cartography/intel/aws/ecs.py | 2 +- cartography/intel/aws/iam.py | 2 +- cartography/intel/aws/inspector.py | 2 +- cartography/intel/aws/permission_relationships.py | 2 +- cartography/intel/aws/s3.py | 4 ++-- cartography/intel/crxcavator/crxcavator.py | 2 +- cartography/intel/dns.py | 6 +++--- cartography/models/core/nodes.py | 4 ++-- docs/root/dev/testing-with-docker.md | 2 +- docs/root/modules/aws/config.md | 2 +- docs/root/modules/aws/schema.md | 12 ++++++------ docs/root/modules/duo/schema.md | 2 +- docs/root/modules/gcp/schema.md | 2 +- docs/root/modules/gsuite/config.md | 2 +- docs/root/modules/gsuite/schema.md | 2 +- docs/root/modules/kubernetes/config.md | 4 ++-- docs/root/modules/lastpass/schema.md | 4 ++-- tests/data/aws/iam/__init__.py | 2 +- tests/data/cve/feed.py | 8 ++++---- .../cartography/intel/aws/ec2/test_ec2_instances.py | 2 +- tests/integration/cartography/intel/aws/test_ecr.py | 2 +- 29 files changed, 43 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index ad5242bb80..cfca36a20f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Start [here](https://lyft.github.io/cartography/install.html). - [Microsoft Azure](https://lyft.github.io/cartography/modules/azure/index.html) - CosmosDB, SQL, Storage, Virtual Machine - [Kubernetes](https://lyft.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container - [PagerDuty](https://lyft.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors -- [Crowdstrike Falcon](https://lyft.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilites, CVEs +- [Crowdstrike Falcon](https://lyft.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs - [NIST CVE](https://lyft.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers diff --git a/cartography/data/jobs/analysis/aws_ec2_iaminstance.json b/cartography/data/jobs/analysis/aws_ec2_iaminstance.json index bb69fe17f0..6127687b92 100644 --- a/cartography/data/jobs/analysis/aws_ec2_iaminstance.json +++ b/cartography/data/jobs/analysis/aws_ec2_iaminstance.json @@ -2,7 +2,7 @@ "name": "EC2 Instances assume IAM roles", "statements": [ { - "__comment": "Create STS_ASSUMEROLE_ALLOW realtionship from ec2 instance to the associated iaminstance iam role", + "__comment": "Create STS_ASSUMEROLE_ALLOW relationship from ec2 instance to the associated iaminstance iam role", "query":"MATCH (aa:AWSAccount)-[:RESOURCE]->(i:EC2Instance)\nWITH SPLIT(i.iaminstanceprofile, '/')[-1] AS role_name, aa, i\nMATCH (aa)-[:RESOURCE]->(r:AWSRole)\nWHERE r.arn ENDS WITH role_name\nMERGE (i)-[:STS_ASSUMEROLE_ALLOW]->(r)", "iterative": false } diff --git a/cartography/data/jobs/analysis/aws_ec2_iaminstanceprofile.json b/cartography/data/jobs/analysis/aws_ec2_iaminstanceprofile.json index 510c2798ac..e3a7e65c82 100644 --- a/cartography/data/jobs/analysis/aws_ec2_iaminstanceprofile.json +++ b/cartography/data/jobs/analysis/aws_ec2_iaminstanceprofile.json @@ -2,7 +2,7 @@ "name": "EC2 Instances assume IAM roles", "statements": [ { - "__comment": "Create STS_ASSUMEROLE_ALLOW realtionships from EC2 instances to the IAM roles they can assume via their iaminstanceprofiles", + "__comment": "Create STS_ASSUMEROLE_ALLOW relationships from EC2 instances to the IAM roles they can assume via their iaminstanceprofiles", "query":"MATCH (aa:AWSAccount)-[:RESOURCE]->(i:EC2Instance)\nWITH SPLIT(i.iaminstanceprofile, '/')[-1] AS role_name, aa, i\nMATCH (aa)-[:RESOURCE]->(r:AWSRole)\nWHERE r.arn ENDS WITH role_name\nMERGE (i)-[:STS_ASSUMEROLE_ALLOW]->(r)", "iterative": false } diff --git a/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json b/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json index dc8952ddbb..3a73c8ab05 100644 --- a/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json +++ b/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json @@ -22,7 +22,7 @@ "iterative": false }, { - "__comment__": "Attach EC2KeyPairs with matching fingerprints to eachother and set duplicate_keyfingerprint = True", + "__comment__": "Attach EC2KeyPairs with matching fingerprints to each other and set duplicate_keyfingerprint = True", "query": "MATCH (k1:EC2KeyPair), (k2:EC2KeyPair) WHERE k1.id <> k2.id AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG return COUNT(*) as TotalCompleted", "iterative": false } diff --git a/cartography/data/jobs/analysis/aws_lambda_ecr.json b/cartography/data/jobs/analysis/aws_lambda_ecr.json index 48d7e50caa..8946705c5c 100644 --- a/cartography/data/jobs/analysis/aws_lambda_ecr.json +++ b/cartography/data/jobs/analysis/aws_lambda_ecr.json @@ -2,7 +2,7 @@ "name": "Lambda functions with ECR images", "statements": [ { - "__comment": "Create HAS_IMAGE realtionship from lambda functions to the associated ECR image", + "__comment": "Create HAS_IMAGE relationship from lambda functions to the associated ECR image", "query":"MATCH (l:AWSLambda) \n WITH COLLECT(l) as lmbda_list \n UNWIND lmbda_list as lmbda \n MATCH (e:ECRImage) \n WHERE e.digest = 'sha256:' + lmbda.codesha256 \n MERGE (lmbda)-[r:HAS]->(e) \n SET r.lastupdated = $UPDATE_TAG", "iterative": false }, diff --git a/cartography/data/jobs/cleanup/crowdstrike_import_cleanup.json b/cartography/data/jobs/cleanup/crowdstrike_import_cleanup.json index 6293a562c0..9213aace22 100644 --- a/cartography/data/jobs/cleanup/crowdstrike_import_cleanup.json +++ b/cartography/data/jobs/cleanup/crowdstrike_import_cleanup.json @@ -26,7 +26,7 @@ "query": "MATCH (:SpotlightVulnerability)<-[hv:HAS_VULNERABILITY]-(:CrowdstrikeHost) WHERE hv.lastupdated <> $UPDATE_TAG WITH hv LIMIT $LIMIT_SIZE DELETE (hv)", "iterative": true, "iterationsize": 100, - "__comment__": "Delete relationship between vulnerabilty and host if host exists and vulnerabilty does not" + "__comment__": "Delete relationship between vulnerability and host if host exists and vulnerability does not" } ], "name": "cleanup crowdstrike" diff --git a/cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json b/cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json index 4ec2ef5374..952a813473 100644 --- a/cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json +++ b/cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json @@ -52,7 +52,7 @@ "iterative": false }, { - "__comment__": "if reachability analysis confirmed that is rechable (likelihood = certain) -> Risk = Severity", + "__comment__": "if reachability analysis confirmed that is reachable (likelihood = certain) -> Risk = Severity", "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'REACHABILITY', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) SET s.reachability_risk = s.severity return COUNT(*) as TotalCompleted", "iterative": false }, diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 828b5c86c5..9e168e95ed 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -76,7 +76,7 @@ def get_rest_api_client_certificate(stages: Dict, client: botocore.client.BaseCl response = client.get_client_certificate(clientCertificateId=stage['clientCertificateId']) response['stageName'] = stage['stageName'] except ClientError as e: - logger.warning(f"Failed to retrive Client Certificate for Stage {stage['stageName']} - {e}") + logger.warning(f"Failed to retrieve Client Certificate for Stage {stage['stageName']} - {e}") raise else: return [] diff --git a/cartography/intel/aws/ecs.py b/cartography/intel/aws/ecs.py index 70e8d503cf..31af66b635 100644 --- a/cartography/intel/aws/ecs.py +++ b/cartography/intel/aws/ecs.py @@ -34,7 +34,7 @@ def get_ecs_clusters( cluster_arns: List[str], ) -> List[Dict[str, Any]]: client = boto3_session.client('ecs', region_name=region) - # TODO: also include attachment info, and make relationships between the attachements + # TODO: also include attachment info, and make relationships between the attachments # and the cluster. includes = ['SETTINGS', 'CONFIGURATIONS'] clusters: List[Dict[str, Any]] = [] diff --git a/cartography/intel/aws/iam.py b/cartography/intel/aws/iam.py index eec5634a43..faad1d0b92 100644 --- a/cartography/intel/aws/iam.py +++ b/cartography/intel/aws/iam.py @@ -354,7 +354,7 @@ def load_roles( # but not sync yet. # - The inscope attribute - set when the account is being sync. # - The foreign attribute - the attribute assignment logic is in aws_foreign_accounts.json analysis job - # - Why seperate statement is needed - the arn may point to service level principals ex - ec2.amazonaws.com + # - Why separate statement is needed - the arn may point to service level principals ex - ec2.amazonaws.com ingest_spnmap_statement = """ MERGE (aa:AWSAccount{id: $SpnAccountId}) ON CREATE SET aa.firstseen = timestamp() diff --git a/cartography/intel/aws/inspector.py b/cartography/intel/aws/inspector.py index c424a42096..6a120100d0 100644 --- a/cartography/intel/aws/inspector.py +++ b/cartography/intel/aws/inspector.py @@ -27,7 +27,7 @@ def get_inspector_findings( current_aws_account_id: str, ) -> List[Dict[str, Any]]: """ - We must list_findings by filtering the request, otherwise the request could tiemout. + We must list_findings by filtering the request, otherwise the request could timeout. First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away, we will only fetch those in ACTIVE or SUPPRESSED statuses. list_members will get us all the accounts that diff --git a/cartography/intel/aws/permission_relationships.py b/cartography/intel/aws/permission_relationships.py index 8a7af04f70..4481e5582e 100644 --- a/cartography/intel/aws/permission_relationships.py +++ b/cartography/intel/aws/permission_relationships.py @@ -134,7 +134,7 @@ def principal_allowed_on_resource(policies: Dict, resource_arn: str, permissions Arguments: - policies {[dict]} -- The policys to evaluate + policies {[dict]} -- The policies to evaluate resource_arn {str} -- The resource to test the permission against permissions {[str]} -- The permissions to evaluate diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index 396c275926..cf4de2e452 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -55,7 +55,7 @@ def get_s3_bucket_details( Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON), default encryption policy (JSON), Versioning (JSON), and Public Access Block (JSON) """ - # a local store for s3 clients so that we may re-use clients for an AWS region + # a local store for s3 clients so that we may reuse clients for an AWS region s3_regional_clients: Dict[Any, Any] = {} BucketDetail = Tuple[str, Dict[str, Any], Dict[str, Any], Dict[str, Any], Dict[str, Any], Dict[str, Any]] @@ -709,7 +709,7 @@ def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict, current_aws_accoun """ # The owner data returned by the API maps to the aws account nickname and not the IAM user - # there doesn't seem to be a way to retreive the mapping but we can get the current context account + # there doesn't seem to be a way to retrieve the mapping but we can get the current context account # so we map to that directly for bucket in data["Buckets"]: diff --git a/cartography/intel/crxcavator/crxcavator.py b/cartography/intel/crxcavator/crxcavator.py index 19b500a230..859952215c 100644 --- a/cartography/intel/crxcavator/crxcavator.py +++ b/cartography/intel/crxcavator/crxcavator.py @@ -79,7 +79,7 @@ def get_extensions(crxcavator_api_key: str, crxcavator_base_url: str, extensions Retrieves the detailed information for all the extension_id and version pairs :param crxcavator_api_key: The API key to access the CRXcavator service :param crxcavator_base_url: The URL for the CRXcavator API - :param extensions_list: list of dictonary items containing the extension_id and version pairs + :param extensions_list: list of dictionary items containing the extension_id and version pairs :return: list containing all metadata for extensions """ extensions_details: List[Dict] = [] diff --git a/cartography/intel/dns.py b/cartography/intel/dns.py index d0e077d445..704ad707ef 100644 --- a/cartography/intel/dns.py +++ b/cartography/intel/dns.py @@ -33,7 +33,7 @@ def ingest_dns_record_by_fqdn( dns_node_additional_label="ESDomain") :param neo4j_session: Neo4j session object - :param update_tag: Update tag to set the node with and childs + :param update_tag: Update tag to set the node with and children :param fqdn: the fqdn record to add :param points_to_record: parent record to set DNS_POINTS_TO relationship to. Can be None :param record_label: the label of the node to attach to a DNS record, e.g. "ESDomain" @@ -72,7 +72,7 @@ def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list: Link A record to to its IP :param neo4j_session: Neo4j session object - :param update_tag: Update tag to set the node with and childs + :param update_tag: Update tag to set the node with and children :param ip_list: List of IP to link :param parent_record: parent record to set DNS_POINTS_TO relationship to """ @@ -109,7 +109,7 @@ def ingest_dns_record( :param name: record name :param value: record value :param type: record type - :param update_tag: Update tag to set the node with and childs + :param update_tag: Update tag to set the node with and children :param points_to_record: parent record to set DNS_POINTS_TO relationship to. Can be None :param record_label: the label of the node to attach to a DNS record :param dns_node_additional_label: The specific label of the DNSRecord, e.g. AWSDNSRecord. diff --git a/cartography/models/core/nodes.py b/cartography/models/core/nodes.py index 22769e960e..9933590092 100644 --- a/cartography/models/core/nodes.py +++ b/cartography/models/core/nodes.py @@ -88,7 +88,7 @@ def other_relationships(self) -> Optional[OtherRelationships]: """ Optional. Allows subclasses to specify additional cartography relationships on the node. - :return: None if not overriden. Else return the node's OtherRelationships. + :return: None if not overridden. Else return the node's OtherRelationships. """ return None @@ -97,6 +97,6 @@ def extra_node_labels(self) -> Optional[ExtraNodeLabels]: """ Optional. Allows specifying extra labels on the node. - :return: None if not overriden. Else return the ExtraNodeLabels specified on the node. + :return: None if not overridden. Else return the ExtraNodeLabels specified on the node. """ return None diff --git a/docs/root/dev/testing-with-docker.md b/docs/root/dev/testing-with-docker.md index 82a64ebd3e..6c40b412a0 100644 --- a/docs/root/dev/testing-with-docker.md +++ b/docs/root/dev/testing-with-docker.md @@ -62,7 +62,7 @@ export GITHUB_KEY=BASE64ENCODEDKEY # You need to set this after starting neo4j once, and resetting # the default neo4j password, which is neo4j export NEO4j_PASSWORD=... -# Reenable bash command history +# Re-enable bash command history set -o history # Start cartography dependencies docker-compose up -d diff --git a/docs/root/modules/aws/config.md b/docs/root/modules/aws/config.md index 7a02c5e59b..912b740e2b 100644 --- a/docs/root/modules/aws/config.md +++ b/docs/root/modules/aws/config.md @@ -6,7 +6,7 @@ Follow these steps to analyze AWS assets with Cartography. ### Single AWS Account Setup -1. Set up an AWS identity (user, group, or role) for Cartography to use. Ensure that this identity has the built-in AWS [SecurityAudit policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_job-functions.html#jf_security-auditor) (arn:aws:iam::aws:policy/SecurityAudit) attached. This policy grants access to read security config metadata. The SecurityAudit policy does not yet containe permissions for `inspector2`, so you will also need the [AmazonInspector2ReadOnlyAccess policy](https://docs.aws.amazon.com/inspector/latest/user/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonInspector2ReadOnlyAccess). +1. Set up an AWS identity (user, group, or role) for Cartography to use. Ensure that this identity has the built-in AWS [SecurityAudit policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_job-functions.html#jf_security-auditor) (arn:aws:iam::aws:policy/SecurityAudit) attached. This policy grants access to read security config metadata. The SecurityAudit policy does not yet contain permissions for `inspector2`, so you will also need the [AmazonInspector2ReadOnlyAccess policy](https://docs.aws.amazon.com/inspector/latest/user/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonInspector2ReadOnlyAccess). 1. Set up AWS credentials to this identity on your server, using a `config` and `credential` file. For details, see AWS' [official guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). 1. [Optional] Configure AWS Retry settings using `AWS_MAX_ATTEMPTS` and `AWS_RETRY_MODE` environment variables. This helps in API Rate Limit throttling and TooManyRequestException related errors. For details, see AWS' [official guide](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-environment-variables). diff --git a/docs/root/modules/aws/schema.md b/docs/root/modules/aws/schema.md index 2d9b122ef3..d62aab1de6 100644 --- a/docs/root/modules/aws/schema.md +++ b/docs/root/modules/aws/schema.md @@ -163,7 +163,7 @@ Representation of an AWS [Inspector Finding](https://docs.aws.amazon.com/inspect |portrange|Port range affected for network findings| |portrangebegin|Beginning of the port range affected for network findings| |portrangeend|End of the port range affected for network findings| -|vulnerabilityid|Vulnerability ID associdated with the finding for package findings| +|vulnerabilityid|Vulnerability ID associated with the finding for package findings| |referenceurls|Reference URLs for the found vulnerabilities| |relatedvulnerabilities|A list of any related vulnerabilities| |source|Source for the vulnerability| @@ -393,7 +393,7 @@ Representation of an [AWS Policy](https://docs.aws.amazon.com/IAM/latest/APIRefe | createdate | ISO 8601 date-time when the policy was created| | type | "inline" or "managed" - the type of policy it is| | arn | The arn for this object | -| **id** | The unique identifer for a policy. If the policy is managed this will be the Arn. If the policy is inline this will calculated as _AWSPrincipal_/inline_policy/_PolicyName_| +| **id** | The unique identifier for a policy. If the policy is managed this will be the Arn. If the policy is inline this will calculated as _AWSPrincipal_/inline_policy/_PolicyName_| #### Relationships @@ -1027,7 +1027,7 @@ Representation of an AWS [EC2 Key Pair](https://docs.aws.amazon.com/AWSEC2/lates (AWSAccount)-[RESOURCE]->(EC2KeyPair) ``` -- EC2 key pairs can be used to log in to AWS EC2 isntances. +- EC2 key pairs can be used to log in to AWS EC2 instances. ``` (EC2KeyPair)-[SSH_LOGIN_TO]->(EC2Instance) @@ -1231,7 +1231,7 @@ Representation of an AWS EC2 [Subnet](https://docs.aws.amazon.com/AWSEC2/latest/ ### AWSInternetGateway - Representation of an AWS [Interent Gateway](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InternetGateway.html). + Representation of an AWS [Internet Gateway](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InternetGateway.html). | Field | Description | |--------|-----------| @@ -2223,7 +2223,7 @@ Representation of an AWS [KMS Key](https://docs.aws.amazon.com/kms/latest/APIRef (AWSAccount)-[RESOURCE]->(KMSKey) ``` -- AWS KMS Key may also be refered as KMSAlias via aliases. +- AWS KMS Key may also be referred as KMSAlias via aliases. ``` (KMSKey)-[KNOWN_AS]->(KMSAlias) @@ -2249,7 +2249,7 @@ Representation of an AWS [KMS Key Alias](https://docs.aws.amazon.com/kms/latest/ #### Relationships -- AWS KMS Key may also be refered as KMSAlias via aliases. +- AWS KMS Key may also be referred as KMSAlias via aliases. ``` (KMSKey)-[KNOWN_AS]->(KMSAlias) diff --git a/docs/root/modules/duo/schema.md b/docs/root/modules/duo/schema.md index 0e648de3da..9e193d9eff 100644 --- a/docs/root/modules/duo/schema.md +++ b/docs/root/modules/duo/schema.md @@ -4,7 +4,7 @@ ### DuoApiHost -Represents a Duo API Host to conain Duo resources. +Represents a Duo API Host to contain Duo resources. | Field | Description | |-------|--------------| diff --git a/docs/root/modules/gcp/schema.md b/docs/root/modules/gcp/schema.md index daa5999ddf..232cdbf6e6 100644 --- a/docs/root/modules/gcp/schema.md +++ b/docs/root/modules/gcp/schema.md @@ -191,7 +191,7 @@ Representation of a GCP [Instance](https://cloud.google.com/compute/docs/referen | instancename | The name of the instance, e.g. "my-instance" | | zone_name | The zone that the instance is installed on | | hostname | If present, the hostname of the instance | -| exposed_internet | Set to True with `exposed_internet_type = 'direct'` if there is an 'allow' IPRule attached to one of the instance's ingress firewalls with the following conditions: The 'allow' IpRule allows traffic from one or more TCP ports, and the 'allow' IpRule is not superceded by a 'deny' IPRule (in GCP, a firewall rule of priority 1 gets applied ahead of a firewall rule of priority 100, and 'deny' rules of the same priority are applied ahead of 'allow' rules) | +| exposed_internet | Set to True with `exposed_internet_type = 'direct'` if there is an 'allow' IPRule attached to one of the instance's ingress firewalls with the following conditions: The 'allow' IpRule allows traffic from one or more TCP ports, and the 'allow' IpRule is not superseded by a 'deny' IPRule (in GCP, a firewall rule of priority 1 gets applied ahead of a firewall rule of priority 100, and 'deny' rules of the same priority are applied ahead of 'allow' rules) | | status | The [GCP Instance Lifecycle](https://cloud.google.com/compute/docs/instances/instance-life-cycle) state of the instance | #### Relationships diff --git a/docs/root/modules/gsuite/config.md b/docs/root/modules/gsuite/config.md index d726405ba7..87a0beb331 100644 --- a/docs/root/modules/gsuite/config.md +++ b/docs/root/modules/gsuite/config.md @@ -74,7 +74,7 @@ print(f'Please go to this URL: {auth_url}') code = input('Enter the authorization code: ') flow.fetch_token(code=code) creds = flow.credentials -print('Testing your credentials by gettings first 10 users in the domain ...') +print('Testing your credentials by getting first 10 users in the domain ...') service = build('admin', 'directory_v1', credentials=creds) print('Getting the first 10 users in the domain') results = service.users().list(customer='my_customer', maxResults=10, diff --git a/docs/root/modules/gsuite/schema.md b/docs/root/modules/gsuite/schema.md index 75e77c723b..6a65ba5059 100644 --- a/docs/root/modules/gsuite/schema.md +++ b/docs/root/modules/gsuite/schema.md @@ -18,7 +18,7 @@ https://developers.google.com/admin-sdk/directory/v1/reference/users#resource | etag | ETag of the resource | include_in_global_address_list | Indicates if the user's profile is visible in the G Suite global address list when the contact sharing feature is enabled for the domain. For more information about excluding user profiles, see the administration help center. | ip_whitelisted | If true, the user's IP address is white listed. -| is_admin | Indicates a user with super admininistrator privileges. The isAdmin property can only be edited in the Make a user an administrator operation (makeAdmin method). If edited in the user insert or update methods, the edit is ignored by the API service. +| is_admin | Indicates a user with super administrator privileges. The isAdmin property can only be edited in the Make a user an administrator operation (makeAdmin method). If edited in the user insert or update methods, the edit is ignored by the API service. | is_delegated_admin | Indicates if the user is a delegated administrator. Delegated administrators are supported by the API but cannot create or undelete users, or make users administrators. These requests are ignored by the API service. Roles and privileges for administrators are assigned using the Admin console. | is_enforced_in_2_sv | Is 2-step verification enforced (Read-only) | is_enrolled_in_2_sv | Is enrolled in 2-step verification (Read-only) diff --git a/docs/root/modules/kubernetes/config.md b/docs/root/modules/kubernetes/config.md index f6b2b5047d..985c8a01a2 100644 --- a/docs/root/modules/kubernetes/config.md +++ b/docs/root/modules/kubernetes/config.md @@ -4,6 +4,6 @@ Follow these steps to analyze Kubernetes objects in Cartography. -1. Configure a [kubeconfig file](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) specifying access to one or mulitple clusters. - - Access to mutliple K8 clusters can be organized in a single kubeconfig file. Intel module of Kubernetes will automatically detect that and attempt to sync each cluster. +1. Configure a [kubeconfig file](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) specifying access to one or multiple clusters. + - Access to multiple K8 clusters can be organized in a single kubeconfig file. Intel module of Kubernetes will automatically detect that and attempt to sync each cluster. 2. Note down the path of configured kubeconfig file and pass it to cartography CLI with `--k8s-kubeconfig` parameter. diff --git a/docs/root/modules/lastpass/schema.md b/docs/root/modules/lastpass/schema.md index cbbff9a5bf..ef1a8804c7 100644 --- a/docs/root/modules/lastpass/schema.md +++ b/docs/root/modules/lastpass/schema.md @@ -32,10 +32,10 @@ Representation of a single User in Lastpass | last_pw_change | Timestamp of the last master password change | | last_login | Timestamp of the last login | | neverloggedin | Flag indicating the user never logged in | -| disabled | Flag indicating accout is disabled | +| disabled | Flag indicating account is disabled | | admin | Flag for admin account | | totalscore | Lastpass security score (max 100) | -| mpstrength | Master password strenght (max 100) | +| mpstrength | Master password strength (max 100) | | sites | Number of site credentials stored | | notes | Number of secured notes stored | | formfills | Number of forms stored | diff --git a/tests/data/aws/iam/__init__.py b/tests/data/aws/iam/__init__.py index 3f942b1531..19fae9e2b4 100644 --- a/tests/data/aws/iam/__init__.py +++ b/tests/data/aws/iam/__init__.py @@ -143,7 +143,7 @@ ], } -INSTACE = { +INSTANCE = { 'Roles': [ { "AssumeRolePolicyDocument": { diff --git a/tests/data/cve/feed.py b/tests/data/cve/feed.py index 7bdf74b952..1ca82457ca 100644 --- a/tests/data/cve/feed.py +++ b/tests/data/cve/feed.py @@ -21,7 +21,7 @@ }, { "lang": "es", - "value": "Existe una vulnerabilidad de secuestro de DLL en ZTE ZXCLOUD iRAI. Un atacante podría colocar un archivo DLL falso en un directorio específico y explotar con éxito esta vulnerabilidad para ejecutar código malicioso.", + "value": "Existe una vulnerabilidad de secuestro de DLL en ZTE ZXCLOUD iRAI. Un atacante podría colocar un archivo DLL false en un directorio específico y explotar con éxito esta vulnerabilidad para ejecutar código malicioso.", }, ], "metrics": { @@ -82,7 +82,7 @@ }, { "lang": "es", - "value": "El complemento The Depicter Slider – Responsive Image Slider, Video Slider & Post Slider para WordPress es vulnerable a Cross-Site Request Forgery en todas las versiones hasta la 2.0.6 incluida. Esto se debe a una validación nonce faltante o incorrecta en la función \"save\". Esto hace posible que atacantes no autenticados modifiquen la configuración del complemento mediante una solicitud falsificada, siempre que puedan engañar al administrador del sitio para que realice una acción como hacer clic en un enlace. CVE-2023-51491 parece ser un duplicado de este problema.", + "value": "El complemento The Depicter Slider – Responsive Image Slider, Video Slider & Post Slider para WordPress es vulnerable a Cross-Site Request Forgery en todas las versiones hasta la 2.0.6 incluida. Esto se debe a una validación nonce faltante o incorrecta en la función \"save\". Esto have possible que atacantes no autenticados modifiquen la configuración del complemento mediante una solicitud falsificada, siempre que puedan engañar al administrador del sitio para que realice una acción como hacer clic en un enlace. CVE-2023-51491 parece set un duplicado de este problema.", }, ], "metrics": { @@ -279,11 +279,11 @@ "descriptions": [ { "lang": "en", - "value": "PyCryptodome and pycryptodomex before 3.19.1 allow side-channel leakage for OAEP decryption, exploitable for a Manger attack.", + "value": "PyCryptodome and pycryptodomex before 3.19.1 allow side-channel leakage for OAEP decryption, exploitable for a Manager attack.", }, { "lang": "es", - "value": "PyCryptodome y pycryptodomex anteriores a 3.19.1 permiten la fuga de canal lateral para el descifrado OAEP, explotable para un ataque Manger.", + "value": "PyCryptodome y pycryptodomex anteriores a 3.19.1 permiten la fuga de canal lateral para el descifrado OAEP, explotable para un ataque Manager.", }, ], "metrics": {}, diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_instances.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_instances.py index 1a4da9663b..7829e27e95 100644 --- a/tests/integration/cartography/intel/aws/ec2/test_ec2_instances.py +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_instances.py @@ -266,7 +266,7 @@ def test_ec2_iaminstanceprofiles(mock_get_instances, neo4j_session): # Arrange boto3_session = MagicMock() create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) - data_iam = tests.data.aws.iam.INSTACE['Roles'] + data_iam = tests.data.aws.iam.INSTANCE['Roles'] sync_ec2_instances( neo4j_session, boto3_session, diff --git a/tests/integration/cartography/intel/aws/test_ecr.py b/tests/integration/cartography/intel/aws/test_ecr.py index 2376367add..3a588f7bb9 100644 --- a/tests/integration/cartography/intel/aws/test_ecr.py +++ b/tests/integration/cartography/intel/aws/test_ecr.py @@ -50,7 +50,7 @@ def test_cleanup_repositories(neo4j_session): # Arrange create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) repo_data = {**tests.data.aws.ecr.DESCRIBE_REPOSITORIES} - # add additional repository noes, for a total of 103, since + # add additional repository notes, for a total of 103, since cleanup_jobs = json.load(open('./cartography/data/jobs/cleanup/aws_import_ecr_cleanup.json')) iter_size = cleanup_jobs['statements'][-1]['iterationsize'] repo_data['repositories'].extend([ From d0a075c97caf258da5270a99d447851a63c7c719 Mon Sep 17 00:00:00 2001 From: chandanchowdhury Date: Sun, 30 Jun 2024 18:00:08 -0400 Subject: [PATCH 18/18] ci: ignore false semgrep find --- cartography/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cartography/cli.py b/cartography/cli.py index 5c5326b96e..3bce89639e 100644 --- a/cartography/cli.py +++ b/cartography/cli.py @@ -647,6 +647,7 @@ def main(self, argv: str) -> int: # Kandji config if config.kandji_base_uri: if config.kandji_token_env_var: + # nosemgrep-next-line:python-logger-credential-disclosure logger.debug( "Reading Kandji API token from environment variable '%s'.", config.kandji_token_env_var,