add separate result bucket for execution log

Signed-off-by: Max Fisher <[email protected]>

cmd/analyze/main.go

@@ -24,42 +24,45 @@ import (
 )
 
 var (
-	pkgName             = flag.String("package", "", "package name")
-	localPkg            = flag.String("local", "", "local package path")
-	ecosystem           pkgecosystem.Ecosystem
-	version             = flag.String("version", "", "version")
-	noPull              = flag.Bool("nopull", false, "disables pulling down sandbox images")
-	imageTag            = flag.String("image-tag", "", "set image tag for analysis sandboxes")
-	dynamicUpload       = flag.String("upload", "", "bucket path for uploading dynamic analysis results")
-	staticUpload        = flag.String("upload-static", "", "bucket path for uploading static analysis results")
-	uploadFileWriteInfo = flag.String("upload-file-write-info", "", "bucket path for uploading information from file writes")
-	uploadAnalyzedPkg   = flag.String("upload-analyzed-pkg", "", "bucket path for uploading analyzed packages")
-	offline             = flag.Bool("offline", false, "disables sandbox network access")
-	customSandbox       = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
-	customAnalysisCmd   = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
-	listModes           = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
-	features            = flag.String("features", "", "override features that are enabled/disabled by default")
-	listFeatures        = flag.Bool("list-features", false, "list available features that can be toggled")
-	help                = flag.Bool("help", false, "print help on available options")
-	analysisMode        = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
+	pkgName            = flag.String("package", "", "package name")
+	localPkg           = flag.String("local", "", "local package path")
+	ecosystem          pkgecosystem.Ecosystem
+	version            = flag.String("version", "", "version")
+	noPull             = flag.Bool("nopull", false, "disables pulling down sandbox images")
+	imageTag           = flag.String("image-tag", "", "set image tag for analysis sandboxes")
+	dynamicBucket      = flag.String("dynamic-bucket", "", "bucket path for uploading dynamic analysis results")
+	staticBucket       = flag.String("static-bucket", "", "bucket path for uploading static analysis results")
+	executionLogBucket = flag.String("execution-log-bucket", "", "bucket path for uploading execution log (dynamic analysis)")
+	fileWritesBucket   = flag.String("file-writes-bucket", "", "bucket path for uploading file writes data (dynamic analysis)")
+	analyzedPkgBucket  = flag.String("analyzed-pkg-bucket", "", "bucket path for uploading analyzed packages")
+	offline            = flag.Bool("offline", false, "disables sandbox network access")
+	customSandbox      = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
+	customAnalysisCmd  = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
+	listModes          = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
+	features           = flag.String("features", "", "override features that are enabled/disabled by default")
+	listFeatures       = flag.Bool("list-features", false, "list available features that can be toggled")
+	help               = flag.Bool("help", false, "print help on available options")
+	analysisMode       = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
 		"list of analysis modes to run, separated by commas. Use -list-modes to see available options")
 )
 
 func makeResultStores() worker.ResultStores {
 	rs := worker.ResultStores{}
 
-	if *dynamicUpload != "" {
-		rs.DynamicAnalysis = resultstore.New(*dynamicUpload)
+	if *analyzedPkgBucket != "" {
+		rs.AnalyzedPackage = resultstore.New(*analyzedPkgBucket)
 	}
-	if *staticUpload != "" {
-		rs.StaticAnalysis = resultstore.New(*staticUpload)
+	if *dynamicBucket != "" {
+		rs.DynamicAnalysis = resultstore.New(*dynamicBucket)
 	}
-	if *uploadFileWriteInfo != "" {
-		rs.FileWrites = resultstore.New(*uploadFileWriteInfo)
+	if *executionLogBucket != "" {
+		rs.ExecutionLog = resultstore.New(*executionLogBucket)
 	}
-
-	if *uploadAnalyzedPkg != "" {
-		rs.AnalyzedPackage = resultstore.New(*uploadAnalyzedPkg)
+	if *fileWritesBucket != "" {
+		rs.FileWrites = resultstore.New(*fileWritesBucket)
+	}
+	if *staticBucket != "" {
+		rs.StaticAnalysis = resultstore.New(*staticBucket)
 	}
 
 	return rs

cmd/worker/main.go

@@ -37,10 +37,11 @@ const (
 
 // resultBucketPaths holds bucket paths for the different types of results.
 type resultBucketPaths struct {
+	analyzedPkg     string
 	dynamicAnalysis string
-	staticAnalysis  string
+	executionLog    string
 	fileWrites      string
-	analyzedPkg     string
+	staticAnalysis  string
 }
 
 type sandboxImageSpec struct {
@@ -79,17 +80,20 @@ func copyPackageToLocalFile(ctx context.Context, packagesBucket *blob.Bucket, bu
 func makeResultStores(dest resultBucketPaths) worker.ResultStores {
 	resultStores := worker.ResultStores{}
 
+	if dest.analyzedPkg != "" {
+		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	}
 	if dest.dynamicAnalysis != "" {
 		resultStores.DynamicAnalysis = resultstore.New(dest.dynamicAnalysis, resultstore.ConstructPath())
 	}
-	if dest.staticAnalysis != "" {
-		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
+	if dest.executionLog != "" {
+		resultStores.ExecutionLog = resultstore.New(dest.executionLog, resultstore.ConstructPath())
 	}
 	if dest.fileWrites != "" {
 		resultStores.FileWrites = resultstore.New(dest.fileWrites, resultstore.ConstructPath())
 	}
-	if dest.analyzedPkg != "" {
-		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	if dest.staticAnalysis != "" {
+		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
 	}
 
 	return resultStores
@@ -274,10 +278,11 @@ func main() {
 	}
 
 	resultsBuckets := resultBucketPaths{
+		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
 		dynamicAnalysis: os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS"),
-		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
+		executionLog:    os.Getenv("OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS"),
 		fileWrites:      os.Getenv("OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS"),
-		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
+		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
 	}
 	resultStores := makeResultStores(resultsBuckets)
 
@@ -305,6 +310,7 @@ func main() {
 		"static_results_bucket", resultsBuckets.staticAnalysis,
 		"file_write_results_bucket", resultsBuckets.fileWrites,
 		"analyzed_packages_bucket", resultsBuckets.analyzedPkg,
+		"execution_log_bucket", resultsBuckets.executionLog,
 		"image_tag", imageSpec.tag,
 		"image_nopull", imageSpec.noPull,
 		"topic_notification", notificationTopicURL,

internal/worker/save_data.go

@@ -18,10 +18,11 @@ import (
 // ResultStores holds ResultStore instances for saving each kind of analysis data.
 // They can be nil, in which case calling the associated Upload function here is a no-op
 type ResultStores struct {
+	AnalyzedPackage      *resultstore.ResultStore
 	DynamicAnalysis      *resultstore.ResultStore
-	StaticAnalysis       *resultstore.ResultStore
+	ExecutionLog         *resultstore.ResultStore
 	FileWrites           *resultstore.ResultStore
-	AnalyzedPackage      *resultstore.ResultStore
+	StaticAnalysis       *resultstore.ResultStore
 	AnalyzedPackageSaved bool
 }
 
@@ -64,7 +65,7 @@ func SaveDynamicAnalysisData(ctx context.Context, pkg *pkgmanager.Pkg, dest *Res
 
 // saveExecutionLog saves the execution log to the dynamic analysis resultstore, only if it is nonempty
 func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStores, data analysisrun.DynamicAnalysisResults) error {
-	if dest.DynamicAnalysis == nil || len(data.ExecutionLog) == 0 {
+	if dest.ExecutionLog == nil || len(data.ExecutionLog) == 0 {
 		// nothing to do
 		return nil
 	}
@@ -74,7 +75,7 @@ func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStor
 		execLogFilename = fmt.Sprintf("execution-log-%s.json", pkg.Version())
 	}
 
-	if err := dest.DynamicAnalysis.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil {
+	if err := dest.ExecutionLog.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil {
 		return fmt.Errorf("failed to save execution log to %s: %w", dest.DynamicAnalysis, err)
 	}
 

scripts/run_analysis.sh

@@ -157,7 +157,7 @@ DOCKER_MOUNTS=("-v" "$CONTAINER_MOUNT_DIR:/var/lib/containers" "-v" "$RESULTS_DI
 
 ANALYSIS_IMAGE=gcr.io/ossf-malware-analysis/analysis
 
-ANALYSIS_ARGS=("analyze" "-upload" "file:///results/" "-upload-file-write-info" "file:///writeResults/" "-upload-static" "file:///staticResults/" "-upload-analyzed-pkg" "file:///analyzedPackages/")
+ANALYSIS_ARGS=("analyze" "-dynamic-bucket" "file:///results/" "-file-writes-bucket" "file:///writeResults/" "-static-bucket" "file:///staticResults/" "-analyzed-pkg-bucket" "file:///analyzedPackages/" "-execution-log-bucket" "file:///results")
 
 # Add the remaining command line arguments
 ANALYSIS_ARGS=("${ANALYSIS_ARGS[@]}" "${args[@]}")