Add more controls in pre-commit, partly security

.codespellignore

@@ -0,0 +1,8 @@
+# Business Unit
+bu
+# Rapid7 SQL query
+ot
+# Pylint dependency package
+astroid
+# Marge Simpson
+Marge

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 default_language_version:
-    # force all unspecified python hooks to run python3
-    python: python3
+  # force all unspecified python hooks to run python3
+  python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
@@ -26,23 +26,64 @@ repos:
 -   repo: https://github.com/asottile/pyupgrade
     rev: v3.17.0
     hooks:
-    -   id: pyupgrade
+      - id: pyupgrade
         args: [--py36-plus]
 -   repo: https://github.com/asottile/add-trailing-comma
     rev: v3.1.0
     hooks:
-    -   id: add-trailing-comma
+      - id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/reorder_python_imports
     rev: v3.13.0
     hooks:
-    -   id: reorder-python-imports
+      - id: reorder-python-imports
         args: [--py3-plus]
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.11.2
     hooks:
-    -   id: mypy
+      - id: mypy
         additional_dependencies:
-        -   types-requests
-        -   types-PyYAML
-        -   types-python-dateutil
+          - types-requests
+          - types-PyYAML
+          - types-python-dateutil
+  #-   repo: https://github.com/psf/black
+  #    rev: 24.4.2
+  #    hooks:
+  #    -   id: black
+  #-   repo: https://github.com/asottile/blacken-docs
+  #    rev: v1.18.0
+  #    hooks:
+  #    -   id: blacken-docs
+  #        additional_dependencies: [black==20.8b1]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+        args: [-I, .codespellignore]
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
+    hooks:
+      - id: detect-secrets
+        args: ["--baseline", ".secrets.baseline"]
+        # exclude: .*/tests/.*
+  - repo: https://github.com/semgrep/semgrep
+    rev: v1.78.0
+    hooks:
+      - id: semgrep
+        name: Semgrep Python
+        types: [python]
+        exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$"
+        args: ["--config", "https://semgrep.dev/p/python", "--error"]
+      - id: semgrep
+        name: Semgrep Bandit
+        types: [python]
+        exclude: "^cli/tests/.+$|^scripts/.+$|^cli/setup.py$"
+        args: ["--config", "https://semgrep.dev/p/bandit", "--error"]
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.9
+    hooks:
+      - id: bandit
+        exclude: ^tests/
+        args:
+          - -s
+          - B101

.secrets.baseline

@@ -0,0 +1,264 @@
+{
+  "version": "1.5.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "OpenAIDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {
+    "docs/root/modules/gcp/schema.md": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "docs/root/modules/gcp/schema.md",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 555
+      }
+    ],
+    "docs/root/modules/gsuite/config.md": [
+      {
+        "type": "Secret Keyword",
+        "filename": "docs/root/modules/gsuite/config.md",
+        "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
+        "is_verified": false,
+        "line_number": 65
+      }
+    ],
+    "tests/data/aws/apigateway.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/aws/apigateway.py",
+        "hashed_secret": "d3053d5db9cc8cb93b26db3c26c76bdfdff06ace",
+        "is_verified": false,
+        "line_number": 20
+      }
+    ],
+    "tests/data/aws/ec2/load_balancers.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/aws/ec2/load_balancers.py",
+        "hashed_secret": "ecb252044b5ea0f679ee78ec1a12904739e2904d",
+        "is_verified": false,
+        "line_number": 48
+      }
+    ],
+    "tests/data/aws/iam/role_policies.py": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tests/data/aws/iam/role_policies.py",
+        "hashed_secret": "43ecf08db7dbbf0212e3596e6ddd216f745cb68e",
+        "is_verified": false,
+        "line_number": 42
+      }
+    ],
+    "tests/data/crxcavator/crxcavator.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/data/crxcavator/crxcavator.py",
+        "hashed_secret": "040b9fac2febf877a3b4e1b406f639ff5e5ff509",
+        "is_verified": false,
+        "line_number": 85
+      }
+    ],
+    "tests/data/duo/endpoints.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/duo/endpoints.py",
+        "hashed_secret": "bc7819b34ff87570745fbe461e36a16f80e562ce",
+        "is_verified": false,
+        "line_number": 27
+      }
+    ],
+    "tests/data/okta/groups.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/okta/groups.py",
+        "hashed_secret": "df4b7a1b3f8b7d7559cf710ab999f28a9bfeb6fc",
+        "is_verified": false,
+        "line_number": 14
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/okta/groups.py",
+        "hashed_secret": "069cf67d25b36dd1210e4afd8daf21cbd9ecc2c1",
+        "is_verified": false,
+        "line_number": 43
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/okta/groups.py",
+        "hashed_secret": "db1a4d8672d5b5c1a7d82fcb4764529b53db5129",
+        "is_verified": false,
+        "line_number": 95
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/okta/groups.py",
+        "hashed_secret": "e52a05b42953592d8d2a32414cbad7d1d34a981e",
+        "is_verified": false,
+        "line_number": 135
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/data/okta/groups.py",
+        "hashed_secret": "8ed33b56d80640b82aa34c1c4c1ec500b30f3471",
+        "is_verified": false,
+        "line_number": 174
+      }
+    ],
+    "tests/integration/cartography/intel/crxcavator/test_crxcavator.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/integration/cartography/intel/crxcavator/test_crxcavator.py",
+        "hashed_secret": "040b9fac2febf877a3b4e1b406f639ff5e5ff509",
+        "is_verified": false,
+        "line_number": 111
+      }
+    ],
+    "tests/integration/cartography/intel/cve/test_feed.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/integration/cartography/intel/cve/test_feed.py",
+        "hashed_secret": "aeaf4b967f1b127a4b590ce5e40132c47a90b414",
+        "is_verified": false,
+        "line_number": 8
+      }
+    ],
+    "tests/integration/cartography/intel/github/test_teams.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/integration/cartography/intel/github/test_teams.py",
+        "hashed_secret": "3da541559918a808c2402bba5012f6c60b27661c",
+        "is_verified": false,
+        "line_number": 14
+      }
+    ],
+    "tests/unit/cartography/intel/cve/test_feed.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/unit/cartography/intel/cve/test_feed.py",
+        "hashed_secret": "aeaf4b967f1b127a4b590ce5e40132c47a90b414",
+        "is_verified": false,
+        "line_number": 18
+      }
+    ]
+  },
+  "generated_at": "2024-06-30T21:00:49Z"
+}

cartography/cli.py

@@ -586,10 +586,10 @@ def main(self, argv: str) -> int:
         if config.neo4j_user:
             config.neo4j_password = None
             if config.neo4j_password_prompt:
-                logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user)
+                logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user)  # nosemgrep
                 config.neo4j_password = getpass.getpass()
             elif config.neo4j_password_env_var:
-                logger.debug(
+                logger.debug(  # nosemgrep
                     "Reading password for Neo4j user '%s' from environment variable '%s'.",
                     config.neo4j_user,
                     config.neo4j_password_env_var,
@@ -611,7 +611,7 @@ def main(self, argv: str) -> int:
 
         # Azure config
         if config.azure_sp_auth and config.azure_client_secret_env_var:
-            logger.debug(
+            logger.debug(  # nosemgrep
                 "Reading Client Secret for Azure Service Principal Authentication from environment variable %s",
                 config.azure_client_secret_env_var,
             )
@@ -652,7 +652,7 @@ def main(self, argv: str) -> int:
             if config.jamf_user:
                 config.jamf_password = None
                 if config.jamf_password_env_var:
-                    logger.debug(
+                    logger.debug(  # nosemgrep
                         "Reading password for Jamf user '%s' from environment variable '%s'.",
                         config.jamf_user,
                         config.jamf_password_env_var,
@@ -669,6 +669,7 @@ def main(self, argv: str) -> int:
         # Kandji config
         if config.kandji_base_uri:
             if config.kandji_token_env_var:
+                # nosemgrep-next-line:python-logger-credential-disclosure
                 logger.debug(
                     "Reading Kandji API token from environment variable '%s'.",
                     config.kandji_token_env_var,