Skip to content

Commit

Permalink
wip(github): scan remote dangling commits
Browse files Browse the repository at this point in the history
  • Loading branch information
rgmz authored and Richard Gomez committed Jun 21, 2024
1 parent fb0d0c7 commit 4e3df24
Show file tree
Hide file tree
Showing 13 changed files with 709 additions and 420 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,8 @@ require (
github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 // indirect
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/sorairolake/lzip-go v0.3.1 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 h1:RCQBSFx5JrsbHltqTtJ+kN3U0Y3a/N/GlVdmRSoxzyE=
github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8=
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
Expand Down
26 changes: 0 additions & 26 deletions hack/snifftest/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,6 @@ func main() {
SkipBinaries: true,
SkipArchives: false,
Concurrency: runtime.NumCPU(),
<<<<<<< HEAD
<<<<<<< HEAD
SourceMetadataFunc: func(repository, commit, commitSource, email, timestamp, file string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Expand All @@ -209,30 +207,6 @@ func main() {
Email: email,
Timestamp: timestamp,
File: file,
=======
SourceMetadataFunc: func(repository, commit, ref, email, timestamp, file string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
Repository: repository,
Commit: commit,
CommitRef: ref,
Email: email,
Timestamp: timestamp,
File: file,
>>>>>>> 3c9809c6 (feat(gitparse): track ref sources)
=======
SourceMetadataFunc: func(repository, commit, commitSource, email, timestamp, file string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
Repository: repository,
Commit: commit,
CommitSource: commitSource,
Email: email,
Timestamp: timestamp,
File: file,
>>>>>>> 2260e4eb (wip: use --mirror)
},
},
}
Expand Down
14 changes: 8 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,13 @@ var (
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
githubIncludeWikis = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool()

githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
githubScanDanglingCommits = githubScan.Flag("dangling-commits", "Include dangling (i.e., unreachable) commits in scan.").Bool()

gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
// TODO: Add more GitLab options
Expand Down Expand Up @@ -580,6 +581,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludeIssueComments: *githubScanIssueComments,
IncludePullRequestComments: *githubScanPRComments,
IncludeGistComments: *githubScanGistComments,
IncludeDanglingCommits: *githubScanDanglingCommits,
Filter: filter,
}
if err := eng.ScanGitHub(ctx, cfg); err != nil {
Expand Down
1 change: 1 addition & 0 deletions pkg/engine/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
IncludeGistComments: c.IncludeGistComments,
IncludeWikis: c.IncludeWikis,
SkipBinaries: c.SkipBinaries,
IncludeDanglingCommits: c.IncludeDanglingCommits,
}
if len(c.Token) > 0 {
connection.Credential = &sourcespb.GitHub_Token{
Expand Down
40 changes: 0 additions & 40 deletions pkg/gitparse/gitparse.go
Original file line number Diff line number Diff line change
Expand Up @@ -637,66 +637,26 @@ func parseCommitLine(line []byte) (hash []byte, ref []byte) {
// ParseCommitSource s
// https://git-scm.com/docs/git-log#Documentation/git-log.txt---source
func parseSourceRef(ref []byte) string {
<<<<<<< HEAD
<<<<<<< HEAD
// We don't care about 'normal' refs.
if bytes.HasPrefix(ref, []byte("refs/heads/")) || bytes.HasPrefix(ref, []byte("refs/tags/")) {
=======
// Remove the `refs/heads/thog` prefix.
// (We don't care about refs without this prefix.)
ref, ok := bytes.CutPrefix(ref, []byte("refs/heads/thog/"))
if !ok {
>>>>>>> 3c9809c6 (feat(gitparse): track ref sources)
=======
// We don't care about 'normal' refs.
if bytes.HasPrefix(ref, []byte("refs/heads/")) || bytes.HasPrefix(ref, []byte("refs/tags/")) {
>>>>>>> 2260e4eb (wip: use --mirror)
return ""
}

// Handle GitHub pull requests.
<<<<<<< HEAD
<<<<<<< HEAD
// e.g., `refs/pull/238/head` or `refs/pull/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("refs/pull/")); ok {
=======
// e.g., `pr/238/head` or `pr/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("pr/")); ok {
>>>>>>> 3c9809c6 (feat(gitparse): track ref sources)
=======
// e.g., `refs/pull/238/head` or `refs/pull/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("refs/pull/")); ok {
>>>>>>> 2260e4eb (wip: use --mirror)
prNumber := after[:bytes.Index(after, []byte("/"))]
return "Pull request #" + string(prNumber)
}

// Handle GitLab merge requests
<<<<<<< HEAD
<<<<<<< HEAD
// e.g., `refs/merge-requests/238/head` or `refs/merge-requests/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("refs/merge-requests/")); ok {
=======
// e.g., `mr/238/head` or `mr/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("mr/")); ok {
>>>>>>> 3c9809c6 (feat(gitparse): track ref sources)
=======
// e.g., `refs/merge-requests/238/head` or `refs/merge-requests/1234/merge`
if after, ok := bytes.CutPrefix(ref, []byte("refs/merge-requests/")); ok {
>>>>>>> 2260e4eb (wip: use --mirror)
mrNumber := after[:bytes.Index(after, []byte("/"))]
return "Merge request #" + string(mrNumber)
}

<<<<<<< HEAD
<<<<<<< HEAD
return fmt.Sprintf("%s (hidden ref)", string(ref))
=======
return ""
>>>>>>> 3c9809c6 (feat(gitparse): track ref sources)
=======
return fmt.Sprintf("%s (hidden ref)", string(ref))
>>>>>>> 2260e4eb (wip: use --mirror)
}

// Author: Bill Rich <[email protected]>
Expand Down
Loading

0 comments on commit 4e3df24

Please sign in to comment.