wip(github): scan remote dangling commits

go.mod

@@ -278,6 +278,8 @@ require (
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 // indirect
+	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/sorairolake/lzip-go v0.3.1 // indirect

go.sum

@@ -744,6 +744,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
 github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
+github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 h1:RCQBSFx5JrsbHltqTtJ+kN3U0Y3a/N/GlVdmRSoxzyE=
+github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

main.go

@@ -106,6 +106,7 @@ var (
 	githubScanPRComments        = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
 	githubScanGistComments      = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
 	githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32()
+	githubScanDanglingCommits   = githubScan.Flag("dangling-commits", "Include dangling (i.e., unreachable) commits in scan.").Bool()
 
 	// GitHub Cross Fork Object Reference Experimental Feature
 	githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.")
@@ -653,6 +654,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			IncludePullRequestComments: *githubScanPRComments,
 			IncludeGistComments:        *githubScanGistComments,
 			CommentsTimeframeDays:      *githubCommentsTimeframeDays,
+			IncludeDanglingCommits:     *githubScanDanglingCommits,
 			Filter:                     filter,
 		}
 		if err := eng.ScanGitHub(ctx, cfg); err != nil {

pkg/engine/github.go

@@ -28,6 +28,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		IncludeWikis:               c.IncludeWikis,
 		SkipBinaries:               c.SkipBinaries,
 		CommentsTimeframeDays:      c.CommentsTimeframeDays,
+		IncludeDanglingCommits:     c.IncludeDanglingCommits,
 	}
 	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{

pkg/sources/git/git.go

@@ -420,13 +420,12 @@ func executeClone(ctx context.Context, params cloneParams) (*git.Repository, err
 	gitArgs := []string{
 		"clone", cloneURL.String(),
 		params.clonePath,
-		"-c", "remote.origin.fetch=+refs/*:refs/remotes/origin/*",
 		"--quiet", // https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--quietcode
 	}
 	if !feature.SkipAdditionalRefs.Load() {
 		gitArgs = append(gitArgs,
 			"-c",
-			"remote.origin.fetch=+refs/*:refs/remotes/origin/*")
+			"remote.origin.fetch=+refs/*:refs/*")
 	}
 	gitArgs = append(gitArgs, params.args...)
 	cloneCmd := exec.Command("git", gitArgs...)
@@ -473,6 +472,48 @@ func executeClone(ctx context.Context, params cloneParams) (*git.Repository, err
 	return repo, nil
 }
 
+var ErrRefNotFound = errors.New("ref not found")
+
+func FetchReference(ctx context.Context, repoPath string, remote string, reference string) error {
+	cmd := exec.Command(
+		"git",
+		"-c", "fetch.parallel=0",
+		"fetch",
+		"--quiet",                 // https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---quiet
+		"--no-auto-gc",            // https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---no-auto-gc
+		"--no-recurse-submodules", // https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---no-recurse-submodules
+		remote,
+		reference)
+	//"--recurse-submodules", "no")
+	absPath, err := filepath.Abs(repoPath)
+	cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git"))
+
+	logger := ctx.Logger().WithValues(
+		"path", repoPath,
+		"command", cmd.String(),
+	)
+
+	// Execute command and wait for the stdout / stderr.
+	outputBytes, err := cmd.CombinedOutput()
+	output := string(outputBytes)
+	if err != nil {
+		if strings.HasPrefix(output, "fatal: remote error: upload-pack: not our ref") {
+			return ErrRefNotFound
+		}
+		err = fmt.Errorf("error executing git fetch: %w, %s", err, output)
+	}
+	logger.V(1).Info("git subcommand finished", "output", output)
+
+	if cmd.ProcessState == nil {
+		return fmt.Errorf("fetch command exited with no output")
+	} else if cmd.ProcessState.ExitCode() != 0 {
+		return err
+	}
+
+	logger.V(1).Info("successfully fetched reference", "repo", repoPath, "ref", reference)
+	return nil
+}
+
 // PingRepoUsingToken executes git ls-remote on a repo and returns any error that occurs. It can be used to validate
 // that a repo actually exists and is reachable.
 //
@@ -1224,7 +1265,7 @@ func (s *Git) handleBinary(
 	path string,
 ) (err error) {
 	fileCtx := context.WithValues(ctx, "commit", commitHash.String()[:7], "path", path)
-	fileCtx.Logger().V(5).Info("handling binary file")
+	fileCtx.Logger().V(5).Info("handling binary file", "gitDir", gitDir)
 
 	if common.SkipFile(path) {
 		fileCtx.Logger().V(5).Info("file contains ignored extension")

pkg/sources/github/connector.go

@@ -2,9 +2,9 @@ package github
 
 import (
 	"fmt"
-
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v66/github"
+	"github.com/shurcooL/githubv4"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -16,6 +16,7 @@ const cloudEndpoint = "https://api.github.com"
 type connector interface {
 	// APIClient returns a configured GitHub client that can be used for GitHub API operations.
 	APIClient() *github.Client
+	GraphQLClient() *githubv4.Client
 	// Clone clones a repository using the configured authentication information.
 	Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error)
 }

pkg/sources/github/connector_app.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"github.com/shurcooL/githubv4"
 	"strconv"
 
 	"github.com/bradleyfalzon/ghinstallation/v2"
@@ -15,6 +16,7 @@ import (
 
 type appConnector struct {
 	apiClient          *github.Client
+	graphQlClient      *githubv4.Client
 	installationClient *github.Client
 	installationID     int64
 }
@@ -69,15 +71,24 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn
 
 	return &appConnector{
 		apiClient:          apiClient,
+		graphQlClient:      githubv4.NewClient(httpClient),
 		installationClient: installationClient,
 		installationID:     installationID,
 	}, nil
 }
 
+func (c *appConnector) InstallationClient() *github.Client {
+	return c.installationClient
+}
+
 func (c *appConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *appConnector) GraphQLClient() *githubv4.Client {
+	return c.graphQlClient
+}
+
 func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	// TODO: Check rate limit for this call.
 	token, _, err := c.installationClient.Apps.CreateInstallationToken(
@@ -90,7 +101,3 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogi
 
 	return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token")
 }
-
-func (c *appConnector) InstallationClient() *github.Client {
-	return c.installationClient
-}

pkg/sources/github/connector_basicauth.go

@@ -2,19 +2,20 @@ package github
 
 import (
 	"fmt"
-
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v66/github"
+	"github.com/shurcooL/githubv4"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
 type basicAuthConnector struct {
-	apiClient *github.Client
-	username  string
-	password  string
+	apiClient     *github.Client
+	graphQlClient *githubv4.Client
+	username      string
+	password      string
 }
 
 var _ connector = (*basicAuthConnector)(nil)
@@ -33,16 +34,21 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*
 	}
 
 	return &basicAuthConnector{
-		apiClient: apiClient,
-		username:  cred.Username,
-		password:  cred.Password,
+		apiClient:     apiClient,
+		graphQlClient: githubv4.NewClient(httpClient),
+		username:      cred.Username,
+		password:      cred.Password,
 	}, nil
 }
 
 func (c *basicAuthConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *basicAuthConnector) GraphQLClient() *githubv4.Client {
+	return c.graphQlClient
+}
+
 func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username)
 }

pkg/sources/github/connector_token.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"github.com/shurcooL/githubv4"
 	"strings"
 	"sync"
 
@@ -15,6 +16,7 @@ import (
 
 type tokenConnector struct {
 	apiClient          *github.Client
+	graphQlClient      *githubv4.Client
 	token              string
 	isGitHubEnterprise bool
 	handleRateLimit    func(context.Context, error) bool
@@ -40,6 +42,7 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(co
 
 	return &tokenConnector{
 		apiClient:          apiClient,
+		graphQlClient:      githubv4.NewClient(httpClient),
 		token:              token,
 		isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint),
 		handleRateLimit:    handleRateLimit,
@@ -50,6 +53,10 @@ func (c *tokenConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *tokenConnector) GraphQLClient() *githubv4.Client {
+	return c.graphQlClient
+}
+
 func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	if err := c.setUserIfUnset(ctx); err != nil {
 		return "", nil, err

pkg/sources/github/connector_unauthenticated.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"github.com/shurcooL/githubv4"
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v66/github"
@@ -12,7 +13,8 @@ import (
 )
 
 type unauthenticatedConnector struct {
-	apiClient *github.Client
+	apiClient     *github.Client
+	graphQlClient *githubv4.Client
 }
 
 var _ connector = (*unauthenticatedConnector)(nil)
@@ -25,14 +27,19 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector,
 		return nil, fmt.Errorf("could not create API client: %w", err)
 	}
 	return &unauthenticatedConnector{
-		apiClient: apiClient,
+		apiClient:     apiClient,
+		graphQlClient: githubv4.NewClient(httpClient),
 	}, nil
 }
 
 func (c *unauthenticatedConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client {
+	return c.graphQlClient
+}
+
 func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	return git.CloneRepoUsingUnauthenticated(ctx, repoURL)
 }