Skip to content

Commit

Permalink
✨ Implement exporters!
Browse files Browse the repository at this point in the history
  • Loading branch information
ewen-lbh committed Apr 14, 2024
1 parent 72e4c22 commit e09f29a
Show file tree
Hide file tree
Showing 14 changed files with 522 additions and 25 deletions.
12 changes: 8 additions & 4 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- exporters: run custom shell commands before and after the build, and/or after each work is built.
- SQL exporter: a rudimentary SQL exporter, written in the Go code directly
- SSH exporter: a rudimentary SSH exporter that uploads the built database somewhere via ssh. written as a normal YAML exporter, see exporters/ssh.yaml

## [1.0.0] - 2024-04-13

### Added
Expand Down Expand Up @@ -46,13 +52,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

- Initial release

[Unreleased]: https://github.com/ortfo/db/compare/v0.3.2...HEAD
[1.0.0]: https://github.com/ortfo/db/-/releases/tag/v1.0.0
[Unreleased]: https://github.com/ortfo/db/compare/v1.0.0...HEAD
[1.0.0]: https://github.com/ortfo/db/compare/v0.3.2...v1.0.0
[0.3.2]: https://github.com/ortfo/db/compare/v0.3.1...v0.3.2
[0.3.1]: https://github.com/ortfo/db/compare/v0.3.0...v0.3.1
[0.3.0]: https://github.com/ortfo/db/compare/v0.2.0...v0.3.0
[0.2.0]: https://github.com/ortfo/db/releases/tag/v0.2.0

[//]: # (C3-2-DKAC:GGH:Rortfo/db:Tv{t})

[unreleased]: https://github.com/ortfo/db/-/compare/v1.0.0...main
52 changes: 51 additions & 1 deletion build.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ type RunContext struct {
Flags Flags
BuildMetadata BuildMetadata
ProgressInfoFile string
Exporters []Exporter

TagsRepository []Tag
TechnologiesRepository []Technology
Expand All @@ -157,6 +158,7 @@ type Flags struct {
NoCache bool
WorkersCount int
ProgressInfoFile string
ExportersToUse []string
}

// Project represents a project.
Expand Down Expand Up @@ -209,6 +211,20 @@ func PrepareBuild(databaseDirectory string, outputFilename string, flags Flags,
}
}

exportersToUse := flags.ExportersToUse
if len(exportersToUse) == 0 {
exportersToUse = mapKeys(config.Exporters)
}

for _, exporterName := range exportersToUse {
exporter, err := ctx.FindExporter(exporterName)
if err != nil {
return &ctx, fmt.Errorf("while finding exporter %s: %w", exporterName, err)
}

ctx.Exporters = append(ctx.Exporters, exporter)
}

ctx.LogDebug("Running with configuration %#v", &config)

previousBuiltDatabaseRaw, err := os.ReadFile(outputFilename)
Expand Down Expand Up @@ -246,6 +262,18 @@ func PrepareBuild(databaseDirectory string, outputFilename string, flags Flags,
return &ctx, fmt.Errorf("another ortfo build is in progress (could not acquire build lock): %w", err)
}

for _, exporter := range ctx.Exporters {
options, err := ctx.ExporterOptions(exporter)
if err != nil {
return &ctx, err
}

err = exporter.Before(&ctx, options)
if err != nil {
return &ctx, fmt.Errorf("while running exporter %s before hook: %w", exporter.Name(), err)
}
}

return &ctx, nil
}

Expand All @@ -272,6 +300,20 @@ func directoriesLeftToBuild(all []string, built []string) []string {
return remaining
}

func (ctx *RunContext) RunExporters(work *AnalyzedWork) error {
for _, exporter := range ctx.Exporters {
if os.Getenv("DEBUG") == "1" {
LogCustom("Exporting", "magenta", "%s to %s", work.ID, exporter.Name())
}
options, _ := ctx.Config.Exporters[exporter.Name()]
err := exporter.Export(ctx, options, work)
if err != nil {
return fmt.Errorf("while exporting %s: %w", exporter.Name(), err)
}
}
return nil
}

func (ctx *RunContext) BuildSome(include string, databaseDirectory string, outputFilename string, flags Flags, config Configuration) (Database, error) {
defer ctx.ReleaseBuildLock(outputFilename)

Expand Down Expand Up @@ -343,6 +385,7 @@ func (ctx *RunContext) BuildSome(include string, databaseDirectory string, outpu
// Skip it!
// ctx.LogInfo("%s: Build skipped: description file unmodified", workID)
ctx.Status(workID, PhaseUnchanged)
ctx.RunExporters(&oldWork)
} else {
ctx.Status(workID, PhaseBuilding)
// Build it
Expand All @@ -352,6 +395,7 @@ func (ctx *RunContext) BuildSome(include string, databaseDirectory string, outpu
builtChannel <- builtItem{err: fmt.Errorf("while building %s (%s): %w", workID, ctx.DescriptionFilename(databaseDirectory, workID), err)}
continue
}
ctx.RunExporters(&newWork)
ctx.Status(workID, PhaseBuilt)

// Set meta-info
Expand Down Expand Up @@ -398,8 +442,14 @@ func (ctx *RunContext) BuildSome(include string, databaseDirectory string, outpu
ctx.LogDebug("main: built dirs: %d out of %d", len(builtDirectories), len(workDirectories))
ctx.LogDebug("main: left to build: %v", directoriesLeftToBuild(workDirectoriesNames, builtDirectories))
}
return works, nil

for _, exporter := range ctx.Exporters {
ctx.LogDebug("Running exporter %s's after hook", exporter.Name())
options := ctx.Config.Exporters[exporter.Name()]
exporter.After(ctx, options, &works)
}

return works, nil
}

func (ctx *RunContext) WriteDatabase(works Database, flags Flags, outputFilename string, partial bool) {
Expand Down
9 changes: 9 additions & 0 deletions cmd/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@ func init() {
buildCmd.PersistentFlags().StringVar(&flags.ProgressInfoFile, "write-progress", "", "Write progress information to a file. See https://pkg.go.dev/github.com/ortfo/db#ProgressInfoEvent for more information.")
buildCmd.PersistentFlags().BoolVar(&flags.NoCache, "no-cache", false, "Disable usage of previous database build as cache for this build (used for media analysis among other things).")
buildCmd.PersistentFlags().IntVar(&flags.WorkersCount, "workers", runtime.NumCPU(), "Use <count> workers to build the database. Defaults to the number of CPU cores.")
buildCmd.PersistentFlags().StringArrayVarP(&flags.ExportersToUse, "exporters", "e", []string{}, "Exporters to enable. If not provided, all the exporters configured in the configuration file will be enabled.")
buildCmd.RegisterFlagCompletionFunc("exporters", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
config, err := ortfodb.NewConfiguration(flags.Config)
if err != nil {
handleError(err)
}
//TODO omit already enabled exporters
return keys(config.Exporters), cobra.ShellCompDirectiveNoFileComp
})
rootCmd.AddCommand(buildCmd)
}

Expand Down
8 changes: 8 additions & 0 deletions cmd/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,11 @@ func handleControlC(outputFilepath string, context *ortfodb.RunContext) {
}
}()
}

func keys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}
15 changes: 4 additions & 11 deletions configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,7 @@ type Configuration struct {
ScatteredModeFolder string `yaml:"scattered mode folder"`
// Signals whether the configuration was instanciated by DefaultConfiguration.
IsDefault bool `yaml:"-"`
// Markdown struct {
// Abbreviations bool `yaml:"abbreviations"`
// DefinitionLists bool `yaml:"definition lists"`
// Admonitions bool `yaml:"admonitions"`
// Footnotes bool `yaml:"footnotes"`
// MarkdownInHTML bool `yaml:"markdown in html"`
// NewLineToLineBreak bool `yaml:"new-line-to-line-break"`
// SmartyPants bool `yaml:"smarty pants"`
// AnchoredHeadings configurationMarkdownAnchoredHeadings `yaml:"anchored headings"`
// CustomSyntaxes []configurationMarkdownCustomSyntax `yaml:"custom syntaxes"`
// }

Tags struct {
// Path to file describing all tags.
Repository string `yaml:"repository"`
Expand All @@ -78,6 +68,9 @@ type Configuration struct {
// Path to the directory containing all projects. Must be absolute.
ProjectsDirectory string `yaml:"projects at"`

// Exporter-specific configuration. Maps exporter names to their configuration.
Exporters map[string]map[string]interface{} `yaml:"exporters,omitempty"`

// Where was the configuration loaded from
source string
}
Expand Down
4 changes: 4 additions & 0 deletions exporter-upload.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name: SSH Upload
after:
- log: [Uploading, blue, 'to {{ index .Data "ssh" }}']
- run: scp -r {{ .Ctx.OutputDatabaseFile }} {{ index .Data "ssh" }}
124 changes: 124 additions & 0 deletions exporter_custom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package ortfodb

import (
"fmt"
"os/exec"
"strings"
"text/template"

jsoniter "github.com/json-iterator/go"
)

type CustomExporter struct {
data map[string]any
name string
manifest ExporterManifest
verbose bool
dryRun bool
cwd string
}

func (e *CustomExporter) Name() string {
return e.name
}

func (e *CustomExporter) OptionsType() any {
return e.manifest.Data
}

func (e *CustomExporter) Before(ctx *RunContext, opts ExporterOptions) error {
return e.runCommands(ctx, e.manifest.Verbose, e.manifest.Before, map[string]any{})

}

func (e *CustomExporter) Export(ctx *RunContext, opts ExporterOptions, work *AnalyzedWork) error {
return e.runCommands(ctx, e.verbose, e.manifest.Work, map[string]any{
"Work": work,
})
}

func (e *CustomExporter) After(ctx *RunContext, opts ExporterOptions, db *Database) error {

return e.runCommands(ctx, e.verbose, e.manifest.After, map[string]any{
"Database": db,
})
}

func (e *CustomExporter) runCommands(ctx *RunContext, verbose bool, commands []ExporterCommand, additionalData map[string]any) error {
for _, command := range commands {
if command.Run != "" {
commandline := e.renderCommandParts(ctx, []string{command.Run}, additionalData)[0]
if commandline == "" {
continue
}
if verbose {
ExporterLogCustom(e, "Running", "yellow", commandline)
}

var stderrBuf strings.Builder
var stdoutBuf strings.Builder

proc := exec.Command("bash", "-c", commandline)
proc.Dir = e.cwd
proc.Stderr = &stderrBuf
proc.Stdout = &stdoutBuf
err := proc.Run()

stdout := strings.TrimSpace(stdoutBuf.String())
stderr := strings.TrimSpace(stderrBuf.String())

if stdout != "" {
ExporterLogCustom(e, ">", "blue", stdout)
}
if stderr != "" {
if !verbose {
ExporterLogCustom(e, "Error", "red", "While running %s\n%s", commandline, stderr)
} else {
ExporterLogCustom(e, "!", "red", stderr)
}
}

if err != nil {
return fmt.Errorf("while running %s: %w", commandline, err)
}
} else {
logParts := e.renderCommandParts(ctx, command.Log, additionalData)
ExporterLogCustom(e, logParts[0], logParts[1], logParts[2])
}
}
return nil
}

var funcmap = template.FuncMap{
"json": func(data any) string {
bytes, err := jsoniter.ConfigFastest.Marshal(data)
if err != nil {
return "{}"
}
return string(bytes)
},
}

func (e *CustomExporter) renderCommandParts(ctx *RunContext, commands []string, additionalData map[string]any) []string {
output := make([]string, 0, len(commands))
for _, command := range commands {
tmpl, err := template.New("top").Funcs(funcmap).Parse(command)
if err != nil {
ctx.DisplayError("custom exporter: while parsing command %s", err, command)
return []string{}
}
var buf strings.Builder
err = tmpl.Execute(&buf, merge(additionalData, map[string]any{
"Data": e.data,
"Ctx": ctx,
"Verbose": e.verbose,
"DryRun": e.dryRun,
}))
if err != nil {
ctx.DisplayError("custom exporter: while rendering command %s", err, command)
return []string{}
}
output = append(output, buf.String())
}
return output
}
70 changes: 70 additions & 0 deletions exporter_sql.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package ortfodb

import (
"fmt"
"os"
"strings"

"github.com/mitchellh/mapstructure"
)

type SqlExporterOptions struct {
Language string `yaml:"language"`
Output string `yaml:"output,omitempty"`
}

type SqlExporter struct {
result string
}

func (e *SqlExporter) OptionsType() any {
return SqlExporterOptions{}
}

func (e *SqlExporter) Name() string {
return "sql"
}

func (e *SqlExporter) Before(ctx *RunContext, opts ExporterOptions) error {
e.result = ""
if !fileExists(e.outputFilename(ctx)) {
e.result += `CREATE TABLE works (
id TEXT PRIMARY KEY,
title TEXT,
summary TEXT,
start_date TEXT,
end_date TEXT,
tags TEXT,
technologies TEXT
);`
}
return nil
}

func (e *SqlExporter) Export(ctx *RunContext, opts ExporterOptions, work *AnalyzedWork) error {
options := SqlExporterOptions{}
mapstructure.Decode(opts, &options)

_, summary := work.FirstParagraph(options.Language)
e.result += fmt.Sprintf(
"INSERT INTO works (id, title, summary, start_date, end_date, tags, technologies) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n",
work.ID,
work.Content.Localize(options.Language).Title,
summary.Content.Markdown(),
work.Metadata.Started,
work.Metadata.Finished,
strings.Join(work.Metadata.Tags, ","),
strings.Join(work.Metadata.MadeWith, ","),
)
return nil
}

func (e *SqlExporter) outputFilename(ctx *RunContext) string {
return strings.Replace(ctx.OutputDatabaseFile, ".json", ".sql", 1)
}

func (e *SqlExporter) After(ctx *RunContext, opts ExporterOptions, built *Database) error {
os.WriteFile(e.outputFilename(ctx), []byte(e.result), 0o644)
ExporterLogCustom(e, "Exported", "green", "SQL file to %s", e.outputFilename(ctx))
return nil
}
Loading

0 comments on commit e09f29a

Please sign in to comment.