Wrap tempfile usage to work around Windows/Golang bug

Makefile

@@ -122,3 +122,6 @@ config_check:
 
 deadcode:
 	go run make.go -v deadcode
+
+api_check:
+	python ./scripts/api_checker.py .

accessors/file/accessor_linux_test.go

@@ -1,26 +1,27 @@
+//go:build linux
 // +build linux
 
 package file
 
 import (
 	"context"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"sort"
 	"testing"
 
 	"github.com/Velocidex/ordereddict"
-	"github.com/alecthomas/assert"
 	"github.com/sebdah/goldie"
 	"github.com/stretchr/testify/suite"
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/config"
 	"www.velocidex.com/golang/velociraptor/glob"
 	"www.velocidex.com/golang/velociraptor/json"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 )
 
 type AccessorLinuxTestSuite struct {
@@ -29,7 +30,7 @@ type AccessorLinuxTestSuite struct {
 }
 
 func (self *AccessorLinuxTestSuite) TestLinuxSymlinks() {
-	tmpdir, err := ioutil.TempDir("", "accessor_test")
+	tmpdir, err := tempfile.TempDir("accessor_test")
 	assert.NoError(self.T(), err)
 
 	// Create two symlinks.

accessors/file/accessor_test.go

@@ -3,7 +3,6 @@ package file_test
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -12,13 +11,14 @@ import (
 	"testing"
 
 	"github.com/Velocidex/ordereddict"
-	"github.com/alecthomas/assert"
 	"github.com/stretchr/testify/suite"
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/config"
 	"www.velocidex.com/golang/velociraptor/glob"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 
 	_ "www.velocidex.com/golang/velociraptor/accessors/ntfs"
 )
@@ -29,7 +29,7 @@ type AccessorWindowsTestSuite struct {
 }
 
 func (self *AccessorWindowsTestSuite) SetupTest() {
-	tmpdir, err := ioutil.TempDir("", "accessor_test")
+	tmpdir, err := tempfile.TempDir("accessor_test")
 	assert.NoError(self.T(), err)
 
 	self.tmpdir = strings.ReplaceAll(tmpdir, "\\", "/")

accessors/raw_registry/raw_registry_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/Velocidex/ordereddict"
-	"github.com/alecthomas/assert"
 	"github.com/sebdah/goldie"
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/config"
@@ -17,6 +16,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/logging"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 	"www.velocidex.com/golang/vfilter"
 
 	_ "www.velocidex.com/golang/velociraptor/accessors/file"

accessors/zip/zip.go

@@ -27,7 +27,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"sync"
@@ -42,6 +41,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/debug"
 	"www.velocidex.com/golang/velociraptor/third_party/zip"
 	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 	utils_tempfile "www.velocidex.com/golang/velociraptor/utils/tempfile"
 	"www.velocidex.com/golang/vfilter"
 	"www.velocidex.com/golang/vfilter/types"
@@ -617,7 +617,7 @@ func (self *SeekableZip) createTmpBackup() (err error) {
 	defer reader.Close()
 
 	// Create a tmp file to unpack the zip member into
-	self.tmp_file_backing, err = ioutil.TempFile("", "zip*.tmp")
+	self.tmp_file_backing, err = tempfile.TempFile("zip*.tmp")
 	if err != nil {
 		return err
 	}

accessors/zip/zip_test.go

@@ -7,13 +7,13 @@ import (
 	"time"
 
 	"github.com/Velocidex/ordereddict"
-	"github.com/alecthomas/assert"
 	"github.com/sebdah/goldie"
 	"github.com/stretchr/testify/suite"
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/file_store/test_utils"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/vtesting"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 
 	_ "www.velocidex.com/golang/velociraptor/accessors/file"
 	_ "www.velocidex.com/golang/velociraptor/accessors/ntfs"

actions/events_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/alecthomas/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"www.velocidex.com/golang/velociraptor/actions"
@@ -24,7 +23,9 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/labels"
 	"www.velocidex.com/golang/velociraptor/services/writeback"
 	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 	"www.velocidex.com/golang/velociraptor/vtesting"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 
 	_ "www.velocidex.com/golang/velociraptor/result_sets/timed"
 )
@@ -60,7 +61,7 @@ func (self *EventsTestSuite) SetupTest() {
 
 	// Set a tempfile for the writeback we need to check that the
 	// new event query is written there.
-	tmpfile, err := ioutil.TempFile("", "")
+	tmpfile, err := tempfile.TempFile("")
 	assert.NoError(self.T(), err)
 	tmpfile.Close()
 

actions/vql_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/alecthomas/assert"
 	"github.com/stretchr/testify/suite"
 	"www.velocidex.com/golang/velociraptor/actions"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
@@ -14,6 +13,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/file_store/test_utils"
 	"www.velocidex.com/golang/velociraptor/responder"
 	"www.velocidex.com/golang/velociraptor/vtesting"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
 )
 
 type ClientVQLTestSuite struct {

api/api.go

@@ -167,6 +167,7 @@ func (self *ApiServer) CollectArtifact(
 	request := &flows_proto.ArtifactCollectorArgs{
 		ClientId:       in.ClientId,
 		Artifacts:      in.Artifacts,
+		Specs:          in.Specs,
 		Creator:        user_record.Name,
 		OpsPerSecond:   in.OpsPerSecond,
 		CpuLimit:       in.CpuLimit,
@@ -175,6 +176,7 @@ func (self *ApiServer) CollectArtifact(
 		MaxRows:        in.MaxRows,
 		MaxUploadBytes: in.MaxUploadBytes,
 		Urgent:         in.Urgent,
+		TraceFreqSec:   in.TraceFreqSec,
 	}
 
 	acl_manager := acl_managers.NewServerACLManager(

artifacts/definitions/Generic/Collectors/File.yaml

@@ -57,7 +57,8 @@ sources:
         FROM glob(globs=specs.Glob, accessor=Accessor)
         WHERE NOT IsDir AND log(message="Found " + SourceFile)
 
-      -- Pass all the results to the next query.
+      -- Pass all the results to the next query. This will serialize
+      -- to disk if there are too many results.
       LET all_results <=
          SELECT Created, Changed, LastAccessed, Modified, Size, SourceFile
          FROM hits

bin/binary_test.go

@@ -5,7 +5,6 @@ package main_test
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -21,6 +20,7 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 )
 
 var (
@@ -94,14 +94,14 @@ func TestAutoexec(t *testing.T) {
 	binary, extension := SetupTest(t)
 
 	// Create a tempfile for the repacked binary.
-	exe, err := ioutil.TempFile("", "exe*"+extension)
+	exe, err := tempfile.TempFile("exe*" + extension)
 	assert.NoError(t, err)
 
 	defer os.Remove(exe.Name())
 	exe.Close()
 
 	// A temp file for the config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(config_file.Name())
@@ -149,7 +149,7 @@ func TestTimeout(t *testing.T) {
 	binary, _ := SetupTest(t)
 
 	// A temp file for the config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(config_file.Name())
@@ -172,7 +172,7 @@ func TestProgressTimeout(t *testing.T) {
 	binary, _ := SetupTest(t)
 
 	// A temp file for the config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(config_file.Name())
@@ -216,7 +216,7 @@ func TestCPULimit(t *testing.T) {
 	binary, _ := SetupTest(t)
 
 	// A temp file for the config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(config_file.Name())
@@ -242,7 +242,7 @@ func TestBuildDeb(t *testing.T) {
 	binary, _ := SetupTest(t)
 
 	// A temp file for the generated config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 	defer os.Remove(config_file.Name())
 
@@ -258,7 +258,7 @@ func TestBuildDeb(t *testing.T) {
 
 	binary_file, _ := filepath.Abs("../artifacts/testdata/files/test.elf")
 
-	output_file, err := ioutil.TempFile("", "output*.deb")
+	output_file, err := tempfile.TempFile("output*.deb")
 	assert.NoError(t, err)
 	output_file.Close()
 	defer os.Remove(output_file.Name())
@@ -280,7 +280,7 @@ func TestBuildDeb(t *testing.T) {
 	assert.Greater(t, stat.Size(), int64(0))
 
 	// Now the server deb
-	output_file, err = ioutil.TempFile("", "output*.deb")
+	output_file, err = tempfile.TempFile("output*.deb")
 	assert.NoError(t, err)
 	output_file.Close()
 	defer os.Remove(output_file.Name())
@@ -306,7 +306,7 @@ func TestGenerateConfigWithMerge(t *testing.T) {
 	binary, extension := SetupTest(t)
 
 	// A temp file for the generated config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 	defer os.Remove(config_file.Name())
 
@@ -366,7 +366,7 @@ func TestGenerateConfigWithMerge(t *testing.T) {
 	require.Error(t, err)
 
 	// Create a tempfile for the repacked binary.
-	exe, err := ioutil.TempFile("", "exe*"+extension)
+	exe, err := tempfile.TempFile("exe*" + extension)
 	assert.NoError(t, err)
 
 	defer os.Remove(exe.Name())
@@ -390,7 +390,7 @@ func TestGenerateConfigWithMerge(t *testing.T) {
 	require.Contains(t, string(out), "Foo")
 
 	// Make second copy of config file and store modified version
-	second_config_file, err := ioutil.TempFile("", "config")
+	second_config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(second_config_file.Name())
@@ -426,7 +426,7 @@ func TestShowConfigWithMergePatch(t *testing.T) {
 	binary, _ := SetupTest(t)
 
 	// A temp file for the generated config.
-	config_file, err := ioutil.TempFile("", "config")
+	config_file, err := tempfile.TempFile("config")
 	assert.NoError(t, err)
 
 	defer os.Remove(config_file.Name())

bin/client.go

@@ -30,6 +30,7 @@ import (
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services/writeback"
 	"www.velocidex.com/golang/velociraptor/startup"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 	"www.velocidex.com/golang/velociraptor/vql/tools"
 )
 
@@ -121,7 +122,7 @@ func runClientOnce(
 		return fmt.Errorf("Invalid config: %w", err)
 	}
 
-	executor.SetTempfile(config_obj)
+	tempfile.SetTempfile(config_obj)
 
 	writeback_service := writeback.GetWritebackService()
 	writeback, err := writeback_service.GetWriteback(config_obj)

bin/collector_test.go

@@ -23,6 +23,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/third_party/zip"
 	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 )
 
 var (
@@ -73,7 +74,7 @@ func (self *CollectorTestSuite) findAndPrepareBinary() {
 	self.binary, _ = filepath.Abs(binaries[0])
 	fmt.Printf("Found binary %v\n", self.binary)
 
-	self.tmpdir, err = ioutil.TempDir("", "tmp")
+	self.tmpdir, err = tempfile.TempDir("tmp")
 	assert.NoError(t, err)
 
 	// Copy the binary into the tmpdir

bin/config_interactive.go

@@ -18,6 +18,7 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services/users"
+	"www.velocidex.com/golang/velociraptor/utils/tempfile"
 )
 
 const (
@@ -90,7 +91,7 @@ portable than plain HTTP. Be sure to test this in you environment.
 
 	log_question = &survey.Input{
 		Message: "Path to the logs directory.",
-		Default: os.TempDir(),
+		Default: tempfile.GetTempDir(),
 	}
 
 	output_question = &survey.Input{