Sifter scanning commands [WIP]

.github/workflows/tests.yml

@@ -33,11 +33,11 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.21
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.50.1
-          args:  --timeout 2m --disable-all -E gofmt 
+          version: v1.54
+          # args:  --timeout 2m 

.golangci.yml

@@ -23,6 +23,5 @@ linters:
     - goimports
     - misspell
     - typecheck
-    - golint
     - gosimple
     - govet

cmd/graphplan/main.go

@@ -0,0 +1,59 @@
+package graphplan
+
+import (
+	"log"
+	"path/filepath"
+
+	"github.com/bmeg/sifter/graphplan"
+	"github.com/bmeg/sifter/playbook"
+	"github.com/spf13/cobra"
+)
+
+var outScriptDir = ""
+var outDataDir = "./"
+
+// Cmd is the declaration of the command line
+var Cmd = &cobra.Command{
+	Use:   "graph-plan",
+	Short: "Scan directory to plan operations",
+	Args:  cobra.MinimumNArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		scriptPath, _ := filepath.Abs(args[0])
+
+		/*
+			if outScriptDir != "" {
+				baseDir, _ = filepath.Abs(outScriptDir)
+			} else if len(args) > 1 {
+				return fmt.Errorf("for multiple input directories, based dir must be defined")
+			}
+
+			_ = baseDir
+		*/
+		outScriptDir, _ = filepath.Abs(outScriptDir)
+		outDataDir, _ = filepath.Abs(outDataDir)
+
+		outDataDir, _ = filepath.Rel(outScriptDir, outDataDir)
+
+		pb := playbook.Playbook{}
+
+		if sifterErr := playbook.ParseFile(scriptPath, &pb); sifterErr == nil {
+			if len(pb.Pipelines) > 0 || len(pb.Inputs) > 0 {
+				err := graphplan.NewGraphBuild(
+					&pb, outScriptDir, outDataDir,
+				)
+				if err != nil {
+					log.Printf("Error: %s\n", err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	flags := Cmd.Flags()
+	flags.StringVarP(&outScriptDir, "dir", "C", outScriptDir, "Change Directory for script base")
+	flags.StringVarP(&outDataDir, "out", "o", outDataDir, "Change output Directory")
+}

cmd/root.go

@@ -3,8 +3,10 @@ package cmd
 import (
 	"os"
 
+	"github.com/bmeg/sifter/cmd/graphplan"
 	"github.com/bmeg/sifter/cmd/inspect"
 	"github.com/bmeg/sifter/cmd/run"
+	"github.com/bmeg/sifter/cmd/scan"
 	"github.com/spf13/cobra"
 )
 
@@ -18,6 +20,8 @@ var RootCmd = &cobra.Command{
 func init() {
 	RootCmd.AddCommand(run.Cmd)
 	RootCmd.AddCommand(inspect.Cmd)
+	RootCmd.AddCommand(graphplan.Cmd)
+	RootCmd.AddCommand(scan.Cmd)
 }
 
 var genBashCompletionCmd = &cobra.Command{

cmd/run/run.go

@@ -21,7 +21,7 @@ func ExecuteFile(playFile string, workDir string, outDir string, inputs map[stri
 	a, _ := filepath.Abs(playFile)
 	baseDir := filepath.Dir(a)
 	log.Printf("basedir: %s", baseDir)
-	log.Printf("playbook: %s", pb)
+	log.Printf("playbook: %#v", pb)
 	return Execute(pb, baseDir, workDir, outDir, inputs)
 }
 

cmd/scan/main.go

@@ -0,0 +1,217 @@
+package scan
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/bmeg/sifter/playbook"
+	"github.com/bmeg/sifter/task"
+	"github.com/spf13/cobra"
+)
+
+var jsonOut = false
+var objectsOnly = false
+var baseDir = ""
+
+type Entry struct {
+	ObjectType string `json:"objectType"`
+	SifterFile string `json:"sifterFile"`
+	Outfile    string `json:"outFile"`
+}
+
+var ObjectCommand = &cobra.Command{
+	Use:   "objects",
+	Short: "Scan for outputs",
+	Args:  cobra.MinimumNArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		scanDir := args[0]
+
+		outputs := []Entry{}
+
+		PathWalker(scanDir, func(pb *playbook.Playbook) {
+			for pname, p := range pb.Pipelines {
+				emitName := ""
+				for _, s := range p {
+					if s.Emit != nil {
+						emitName = s.Emit.Name
+					}
+				}
+				if emitName != "" {
+					for _, s := range p {
+						outdir := pb.GetDefaultOutDir()
+						outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName)
+						outpath := filepath.Join(outdir, outname)
+						o := Entry{SifterFile: pb.GetPath(), Outfile: outpath}
+						if s.ObjectValidate != nil {
+							//outpath, _ = filepath.Rel(baseDir, outpath)
+							//fmt.Printf("%s\t%s\n", s.ObjectValidate.Title, outpath)
+							o.ObjectType = s.ObjectValidate.Title
+						}
+						if objectsOnly {
+							if o.ObjectType != "" {
+								outputs = append(outputs, o)
+							}
+						} else {
+							outputs = append(outputs, o)
+						}
+					}
+				}
+			}
+		})
+
+		if jsonOut {
+			j := json.NewEncoder(os.Stdout)
+			j.SetIndent("", "  ")
+			j.Encode(outputs)
+		} else {
+			for _, i := range outputs {
+				fmt.Printf("%s\t%s\n", i.ObjectType, i.Outfile)
+			}
+		}
+
+		return nil
+
+	},
+}
+
+type ScriptEntry struct {
+	Name    string   `json:"name"`
+	Path    string   `json:"path"`
+	Inputs  []string `json:"inputs"`
+	Outputs []string `json:"outputs"`
+}
+
+func removeDuplicates(s []string) []string {
+	t := map[string]bool{}
+
+	for _, i := range s {
+		t[i] = true
+	}
+	out := []string{}
+	for k := range t {
+		out = append(out, k)
+	}
+	return out
+}
+
+func relPathArray(basedir string, paths []string) []string {
+	out := []string{}
+	for _, i := range paths {
+		if o, err := filepath.Rel(baseDir, i); err == nil {
+			out = append(out, o)
+		}
+	}
+	return out
+}
+
+var ScriptCommand = &cobra.Command{
+	Use:   "scripts",
+	Short: "Scan for scripts",
+	Args:  cobra.MinimumNArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		scanDir := args[0]
+
+		scripts := []ScriptEntry{}
+
+		if baseDir == "" {
+			baseDir, _ = os.Getwd()
+		}
+		baseDir, _ = filepath.Abs(baseDir)
+		//fmt.Printf("basedir: %s\n", baseDir)
+
+		userInputs := map[string]string{}
+
+		PathWalker(scanDir, func(pb *playbook.Playbook) {
+			path := pb.GetPath()
+			scriptDir := filepath.Dir(path)
+
+			config, _ := pb.PrepConfig(userInputs, baseDir)
+
+			task := task.NewTask(pb.Name, scriptDir, baseDir, pb.GetDefaultOutDir(), config)
+			sourcePath, _ := filepath.Abs(path)
+
+			cmdPath, _ := filepath.Rel(baseDir, sourcePath)
+
+			inputs := []string{}
+			outputs := []string{}
+			for _, p := range pb.GetConfigFields() {
+				if p.IsDir() || p.IsFile() {
+					inputs = append(inputs, config[p.Name])
+				}
+			}
+			//inputs = append(inputs, sourcePath)
+
+			sinks, _ := pb.GetOutputs(task)
+			for _, v := range sinks {
+				outputs = append(outputs, v...)
+			}
+
+			emitters, _ := pb.GetEmitters(task)
+			for _, v := range emitters {
+				outputs = append(outputs, v)
+			}
+
+			//for _, e := range pb.Inputs {
+			//}
+
+			s := ScriptEntry{
+				Path:    cmdPath,
+				Name:    pb.Name,
+				Outputs: relPathArray(baseDir, removeDuplicates(outputs)),
+				Inputs:  relPathArray(baseDir, removeDuplicates(inputs)),
+			}
+			scripts = append(scripts, s)
+		})
+
+		if jsonOut {
+			e := json.NewEncoder(os.Stdout)
+			e.SetIndent("", "  ")
+			e.Encode(scripts)
+		} else {
+			for _, i := range scripts {
+				fmt.Printf("%s\n", i)
+			}
+		}
+
+		return nil
+	},
+}
+
+// Cmd is the declaration of the command line
+var Cmd = &cobra.Command{
+	Use:   "scan",
+	Short: "Scan for scripts or objects",
+}
+
+func init() {
+	Cmd.AddCommand(ObjectCommand)
+	Cmd.AddCommand(ScriptCommand)
+
+	objFlags := ObjectCommand.Flags()
+	objFlags.BoolVarP(&objectsOnly, "objects", "s", objectsOnly, "Objects Only")
+	objFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON")
+
+	scriptFlags := ScriptCommand.Flags()
+	scriptFlags.StringVarP(&baseDir, "base", "b", baseDir, "Base Dir")
+	scriptFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON")
+
+}
+
+func PathWalker(baseDir string, userFunc func(*playbook.Playbook)) {
+	filepath.Walk(baseDir,
+		func(path string, info fs.FileInfo, err error) error {
+			if strings.HasSuffix(path, ".yaml") {
+				pb := playbook.Playbook{}
+				if parseErr := playbook.ParseFile(path, &pb); parseErr == nil {
+					userFunc(&pb)
+				}
+			}
+			return nil
+		})
+}

docs/categories/index.xml

@@ -5,6 +5,7 @@
     <link>https://bmeg.github.io/sifter/categories/</link>
     <description>Recent content in Categories on Sifter</description>
     <generator>Hugo -- gohugo.io</generator>
-    <language>en-us</language><atom:link href="https://bmeg.github.io/sifter/categories/index.xml" rel="self" type="application/rss+xml" />
+    <language>en-us</language>
+    <atom:link href="https://bmeg.github.io/sifter/categories/index.xml" rel="self" type="application/rss+xml" />
   </channel>
 </rss>