diff --git a/.github/workflows/secrets-detection.yaml b/.github/workflows/secrets-detection.yaml new file mode 100644 index 000000000..b3651645d --- /dev/null +++ b/.github/workflows/secrets-detection.yaml @@ -0,0 +1,70 @@ +name: Secret Detection Workflow +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + secret-detection: + runs-on: ubuntu-latest + steps: + - + name: Checkout code + uses: actions/checkout@v4 + - + name: Install necessary packages + run: | + pip install git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp + pip install jq + + - + name: Create an initial .secrets.baseline if .secrets.baseline does not exist + run: | + if [ ! -f .secrets.baseline ]; then + # This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files. + # Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results. + echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file." + mkdir empty-dir + detect-secrets scan empty-dir > .secrets.baseline + echo "✅ Blank .secrets.baseline file created successfully." + rm -r empty-dir + else + echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created." + fi + + - + name: Scan repository for secrets + run: | + # scripts to scan repository for new secrets + + # backup the list of known secrets + cp .secrets.baseline .secrets.new + + # find the secrets in the repository + detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \ + --exclude-files '\.secrets..*' \ + --exclude-files '\.pre-commit-config\.yaml' \ + --exclude-files '\.git.*' \ + --exclude-files 'target' + + # if there is any difference between the known and newly detected secrets, break the build + # Function to compare secrets without listing them + compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$1" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$2" | sort) >/dev/null; } + + # Check if there's any difference between the known and newly detected secrets + if ! compare_secrets .secrets.baseline .secrets.new; then + echo "⚠️ Attention Required! ⚠️" >&2 + echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2 + echo "" >&2 + echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2 + echo "" >&2 + echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2 + echo "" >&2 + echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2 + echo "" >&2 + echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2 + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6846cad8c..8c598c9f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,4 +4,16 @@ repos: hooks: - id: pretty-format-java args: [--autofix] - files: .*\.java$ \ No newline at end of file + files: .*\.java$ +- repo: https://github.com/NASA-AMMOS/slim-detect-secrets + # using commit id for now, will change to tag when official version is released + rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe + hooks: + - id: detect-secrets + args: + - '--baseline' + - '.secrets.baseline' + - --exclude-files '\.secrets..*' + - --exclude-files '\.git.*' + - --exclude-files '\.pre-commit-config\.yaml' + - --exclude-files 'target' diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 000000000..8e1709cf7 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,376 @@ +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AWSSensitiveInfoDetectorExperimental" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "EmailAddressDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + "\\.secrets..*", + "\\.git.*", + "\\.pre-commit-config\\.yaml", + "target" + ] + } + ], + "results": { + "README.md": [ + { + "type": "Email Address", + "filename": "README.md", + "hashed_secret": "fac2dea9e49a83a2d6ee38c580d1e5358b45efa5", + "is_verified": false, + "line_number": 79 + } + ], + "pom.xml": [ + { + "type": "Email Address", + "filename": "pom.xml", + "hashed_secret": "fac2dea9e49a83a2d6ee38c580d1e5358b45efa5", + "is_verified": false, + "line_number": 68 + } + ], + "src/changes/changes.xml": [ + { + "type": "Email Address", + "filename": "src/changes/changes.xml", + "hashed_secret": "bf1bfc3f1f5b1a63361b4c29a798ea62be348864", + "is_verified": false, + "line_number": 38 + } + ], + "src/main/java/gov/nasa/pds/validate/report/Report.java": [ + { + "type": "Email Address", + "filename": "src/main/java/gov/nasa/pds/validate/report/Report.java", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 68 + } + ], + "src/main/java/gov/nasa/pds/web/ui/constants/ApplicationConstants.java": [ + { + "type": "Secret Keyword", + "filename": "src/main/java/gov/nasa/pds/web/ui/constants/ApplicationConstants.java", + "hashed_secret": "f603f62fb9107f11d7cba9003091b186fcdfa66a", + "is_verified": false, + "line_number": 27 + }, + { + "type": "Secret Keyword", + "filename": "src/main/java/gov/nasa/pds/web/ui/constants/ApplicationConstants.java", + "hashed_secret": "d1c3f644ae603e7d0ee632db02916aea59d36ace", + "is_verified": false, + "line_number": 35 + } + ], + "src/main/resources/gov/nasa/pds/tools/validate/rule/pds3/masterdd.full": [ + { + "type": "Hex High Entropy String", + "filename": "src/main/resources/gov/nasa/pds/tools/validate/rule/pds3/masterdd.full", + "hashed_secret": "a5b4ee58fbd830f6a52d58dc786bc92aebb0c091", + "is_verified": false, + "line_number": 27304 + } + ], + "src/main/resources/gov/nasa/pds/tools/validate/rule/pds3/resources.properties": [ + { + "type": "Email Address", + "filename": "src/main/resources/gov/nasa/pds/tools/validate/rule/pds3/resources.properties", + "hashed_secret": "19f0a21d2a85d6100bc9ec1e75b3247eadfe95cb", + "is_verified": false, + "line_number": 30 + } + ], + "src/site/markdown/operate/index.html": [ + { + "type": "Email Address", + "filename": "src/site/markdown/operate/index.html", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 1976 + } + ], + "src/site/markdown/operate/index.md": [ + { + "type": "Email Address", + "filename": "src/site/markdown/operate/index.md", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 968 + } + ], + "src/site/xdoc/index.xml": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/index.xml", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 123 + } + ], + "src/site/xdoc/install/index-win.xml.vm": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/install/index-win.xml.vm", + "hashed_secret": "bf1bfc3f1f5b1a63361b4c29a798ea62be348864", + "is_verified": false, + "line_number": 38 + }, + { + "type": "Email Address", + "filename": "src/site/xdoc/install/index-win.xml.vm", + "hashed_secret": "1ac7d6deddaec3bd29b1f559a573231d20d764fd", + "is_verified": false, + "line_number": 39 + } + ], + "src/site/xdoc/operate/errors.xml.vm": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/errors.xml.vm", + "hashed_secret": "187c3d164496d1b05c067a8c82e5a664687c1687", + "is_verified": false, + "line_number": 38 + }, + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/errors.xml.vm", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 49 + }, + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/errors.xml.vm", + "hashed_secret": "4fb813c304003b3813b35a85f05b7cb0c3994cc1", + "is_verified": false, + "line_number": 203 + } + ], + "src/site/xdoc/operate/reports/index-full.xml": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/reports/index-full.xml", + "hashed_secret": "bf1bfc3f1f5b1a63361b4c29a798ea62be348864", + "is_verified": false, + "line_number": 38 + } + ], + "src/site/xdoc/operate/reports/index-json.xml": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/reports/index-json.xml", + "hashed_secret": "bf1bfc3f1f5b1a63361b4c29a798ea62be348864", + "is_verified": false, + "line_number": 38 + } + ], + "src/site/xdoc/operate/reports/index-xml.xml": [ + { + "type": "Email Address", + "filename": "src/site/xdoc/operate/reports/index-xml.xml", + "hashed_secret": "bf1bfc3f1f5b1a63361b4c29a798ea62be348864", + "is_verified": false, + "line_number": 38 + } + ], + "src/test/resources/github344/data/u36_maunakea_380cm_2200nm_ring_beta_ingress_sqw.txt": [ + { + "type": "Email Address", + "filename": "src/test/resources/github344/data/u36_maunakea_380cm_2200nm_ring_beta_ingress_sqw.txt", + "hashed_secret": "67e998390f99d9e6ed0ec027b2af603147178670", + "is_verified": false, + "line_number": 1 + } + ], + "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_00007.tsc": [ + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_00007.tsc", + "hashed_secret": "34af3d6f31b0b9c0d5bbbeb8d2e27a7623f8f481", + "is_verified": false, + "line_number": 67 + }, + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_00007.tsc", + "hashed_secret": "dbfbf317dd33075578e4165b8741dc04cb833d78", + "is_verified": false, + "line_number": 71 + } + ], + "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v01.tsc": [ + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v01.tsc", + "hashed_secret": "34af3d6f31b0b9c0d5bbbeb8d2e27a7623f8f481", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v01.tsc", + "hashed_secret": "dbfbf317dd33075578e4165b8741dc04cb833d78", + "is_verified": false, + "line_number": 76 + } + ], + "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v02.tsc": [ + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v02.tsc", + "hashed_secret": "34af3d6f31b0b9c0d5bbbeb8d2e27a7623f8f481", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v02.tsc", + "hashed_secret": "dbfbf317dd33075578e4165b8741dc04cb833d78", + "is_verified": false, + "line_number": 76 + } + ], + "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v03.tsc": [ + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v03.tsc", + "hashed_secret": "34af3d6f31b0b9c0d5bbbeb8d2e27a7623f8f481", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Email Address", + "filename": "src/test/resources/github597/spice_kernels/m2020_168_sclkscet_refit_v03.tsc", + "hashed_secret": "dbfbf317dd33075578e4165b8741dc04cb833d78", + "is_verified": false, + "line_number": 76 + } + ], + "src/test/resources/riut/auth.txt": [ + { + "type": "Secret Keyword", + "filename": "src/test/resources/riut/auth.txt", + "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", + "is_verified": false, + "line_number": 3 + } + ] + }, + "generated_at": "2023-11-22T19:20:04Z" +}