Refactor to ease development of tasks in external Git repositories

Closes #163.

build/package/scripts/build-go.sh

@@ -16,7 +16,7 @@ ARTIFACT_PREFIX=""
 PRE_TEST_SCRIPT=""
 DEBUG="${DEBUG:-false}"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --working-dir) WORKING_DIR="$2"; shift;;
@@ -131,4 +131,4 @@ if [ $exitcode != 0 ]; then
   exit $exitcode
 fi
 echo "Building ..."
-go build -gcflags "all=-trimpath=$(pwd)" -o "${OUTPUT_DIR}/app"
+go build -gcflags "all=-trimpath=$(pwd)" -o "${OUTPUT_DIR}/app"

build/package/scripts/build-gradle.sh

@@ -14,7 +14,7 @@ gradle_build_dir="build"
 gradle_additional_tasks=
 gradle_options=
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --working-dir) working_dir="$2"; shift;;

build/package/scripts/build-npm.sh

@@ -30,7 +30,7 @@ ARTIFACT_PREFIX=""
 DEBUG="${DEBUG:-false}"
 COPY_NODE_MODULES="false"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --working-dir) WORKING_DIR="$2"; shift;;

build/package/scripts/build-python.sh

@@ -20,7 +20,7 @@ ARTIFACT_PREFIX=""
 PRE_TEST_SCRIPT=""
 DEBUG="${DEBUG:-false}"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --working-dir) WORKING_DIR="$2"; shift;;

build/package/scripts/cache-build.sh

@@ -15,7 +15,7 @@ CACHE_BUILD_KEY=
 CACHE_LOCATION_USED_PATH=
 DEBUG="${DEBUG:-false}"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --working-dir) WORKING_DIR="$2"; shift;;

build/package/scripts/configure-truststore.sh

@@ -7,7 +7,7 @@ src_truststore="${JAVA_HOME}/lib/security/cacerts"
 src_pass="changeit"
 dest_pass="changeit"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --src-store) src_truststore="$2"; shift;;

build/package/scripts/copy-artifacts.sh

@@ -8,7 +8,7 @@ CP="${GNU_CP:-cp}"
 
 DEBUG="${DEBUG:-false}"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
     --debug) DEBUG="$2"; shift;;
     --debug=*) DEBUG="${1#*=}";;

build/package/scripts/copy-build-if-cached.sh

@@ -17,7 +17,7 @@ CACHE_LOCATION_USED_PATH=
 WORKING_DIR="."
 DEBUG="${DEBUG:-false}"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --cache-build) CACHE_BUILD="$2"; shift;;

build/package/scripts/download-aqua-scanner.sh

@@ -5,7 +5,7 @@ md5_bin="${MD5_BIN:-"md5sum"}"
 aqua_scanner_url=""
 bin_dir=".ods-cache/bin"
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
 
     --bin-dir) bin_dir="$2"; shift;;

build/package/scripts/supply-sonar-project-properties-default.sh

@@ -3,7 +3,7 @@ set -eu
 
 working_dir="."
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     case $1 in
     --working-dir) working_dir="$2"; shift;;
     --working-dir=*) working_dir="${1#*=}";;

cmd/taskdoc/main.go

@@ -0,0 +1,69 @@
+// Package taskdoc implements documentation rendering for tasks.
+// It is intended to be run via `go run`, passing a task YAML manifest
+// and a description in Asciidoctor format. The combined result will be
+// written to the specified destination.
+//
+// Example invocation:
+//
+//	go run github.com/opendevstack/ods-pipeline/cmd/taskdoc \
+//		-task tasks/my-task.yaml \
+//		-description build/docs/my-task.adoc \
+//		-destination docs/my-task.adoc
+//
+// By default, taskdoc will use the template located at
+// docs/tasks/template.adoc.tmpl to produce the resulting file. Another
+// template can be specified via -template:
+//
+//	go run github.com/opendevstack/ods-pipeline/cmd/taskdoc \
+//		-task tasks/my-task.yaml \
+//		-description build/docs/my-task.adoc \
+//		-template /path/to/my-custom-template.adoc.tmpl \
+//		-destination docs/my-task.adoc
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"text/template"
+
+	"github.com/opendevstack/ods-pipeline/internal/projectpath"
+	"github.com/opendevstack/ods-pipeline/pkg/taskdoc"
+)
+
+func main() {
+	taskFile := flag.String("task", "", "Task manifest")
+	descriptionFile := flag.String("description", "", "Description snippet")
+	templateFile := flag.String("template", projectpath.RootedPath("docs/tasks/template.adoc.tmpl"), "Template file")
+	destinationFile := flag.String("destination", "", "Destination file")
+	flag.Parse()
+	if err := render(*taskFile, *descriptionFile, *templateFile, *destinationFile); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func render(taskFile, descriptionFile, templateFile, destinationFile string) error {
+	t, err := os.ReadFile(taskFile)
+	if err != nil {
+		return err
+	}
+	d, err := os.ReadFile(descriptionFile)
+	if err != nil {
+		return err
+	}
+	tmpl, err := template.ParseFiles(templateFile)
+	if err != nil {
+		return err
+	}
+
+	task, err := taskdoc.ParseTask(t, d)
+	if err != nil {
+		return err
+	}
+
+	w, err := os.Create(destinationFile)
+	if err != nil {
+		return err
+	}
+	return taskdoc.RenderTaskDocumentation(w, tmpl, task)
+}

cmd/taskmanifest/main.go

@@ -0,0 +1,69 @@
+// Package taskmanifest implements manifest rendering for tasks.
+// It is intended to be run via `go run`, passing a task YAML template
+// and data to be rendered. The combined result will be
+// written to the specified destination. The -data flag can be passed
+// multiple times and may specify any key-value combination, which can then
+// be consumed in the template through Go's text/template package. E.g.
+// passing -data Foo=bar will replace {{.Foo}} in the template with bar.
+//
+// Example invocation:
+//
+//	go run github.com/opendevstack/ods-pipeline/cmd/taskmanifest \
+//		-data ImageRepository=ghcr.io/my-org/my-repo \
+//		-data Version=latest \
+//		-template build/tasks/my-task.yaml \
+//		-destination tasks/my-task.yaml
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/opendevstack/ods-pipeline/pkg/taskmanifest"
+	"github.com/opendevstack/ods-pipeline/pkg/tektontaskrun"
+)
+
+func main() {
+	templateFile := flag.String("template", "", "Template file")
+	destinationFile := flag.String("destination", "", "Destination file")
+	cc := tektontaskrun.NewClusterConfig()
+	mf := &MapFlag{v: cc.DefaultTaskTemplateData()}
+	flag.Var(mf, "data", "Key-value pairs")
+	flag.Parse()
+	if err := render(*templateFile, *destinationFile, mf.v); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func render(templateFile, destinationFile string, data map[string]string) error {
+	tmpl, err := template.ParseFiles(templateFile)
+	if err != nil {
+		return err
+	}
+
+	w, err := os.Create(destinationFile)
+	if err != nil {
+		return err
+	}
+	return taskmanifest.RenderTask(w, tmpl, data)
+}
+
+type MapFlag struct {
+	v map[string]string
+}
+
+func (mf *MapFlag) String() string {
+	return fmt.Sprintf("%v", mf.v)
+}
+func (mf *MapFlag) Set(v string) error {
+	key, value, ok := strings.Cut(v, "=")
+	if !ok {
+		return fmt.Errorf("must have = sign")
+	}
+	mf.v[key] = value
+	return nil
+}

deploy/install.sh

@@ -63,7 +63,7 @@ function usage {
       \n\t\t--sonar-auth 'auth-token' \n\n" "$0"
 }
 
-while [[ "$#" -gt 0 ]]; do
+while [ "$#" -gt 0 ]; do
     # shellcheck disable=SC2034
     case $1 in
 

docs/authoring-tasks.adoc

@@ -102,9 +102,7 @@ In theory you can use pretty much any image that works in OpenShift (e.g. the im
 
 === How do I create my own container image to use in a task?
 
-In OpenShift, the easiest way is by creating an `ImageStream` and a `BuildConfig`. See the link:https://docs.openshift.com/container-platform/latest/cicd/builds/understanding-image-builds.html[OpenShift documentation on builds] for more information. You may also use the YAML definitions in `deploy/ods-pipeline/charts/images` as an example.
-
-Occasionally, you might want to extend the images used in an official tasks, e.g. to deploy additional CA certificates, configure proxy settings, etc. The `images` subchart of `ods-pipeline` provides build configurations that allow you to create images that are based on the official `ods-pipeline` images from ghcr.io. The build configurations include inline Dockerfiles that you can adjust to suit your specific needs.
+In OpenShift, the easiest way is by creating an `ImageStream` and a `BuildConfig`. See the link:https://docs.openshift.com/container-platform/latest/cicd/builds/understanding-image-builds.html[OpenShift documentation on builds] for more information.
 
 === How can I test my tasks?
 

docs/tasks/ods-build-go.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-build-go
 

docs/tasks/ods-build-gradle.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-build-gradle
 

docs/tasks/ods-build-npm.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-build-npm
 

docs/tasks/ods-build-python.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-build-python
 

docs/tasks/ods-deploy-helm.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-deploy-helm
 

docs/tasks/ods-finish.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-finish
 

docs/tasks/ods-package-image.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-package-image
 

docs/tasks/ods-start.adoc

@@ -1,4 +1,4 @@
-// Document generated by internal/documentation/tasks.go from template.adoc.tmpl; DO NOT EDIT.
+// File is generated; DO NOT EDIT.
 
 = ods-start
 

internal/docs/tasks.go

@@ -2,11 +2,12 @@ package docs
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 	"text/template"
 
 	"github.com/opendevstack/ods-pipeline/internal/command"
@@ -37,20 +38,11 @@ func renderTemplate(targetDir, targetFilename string, data Task) error {
 	if err != nil {
 		return err
 	}
-	templateFilename := filepath.Join(targetDir, "template.adoc.tmpl")
-	templateFileParts := strings.Split(templateFilename, "/")
-	templateDisplayname := templateFileParts[len(templateFileParts)-1]
-	_, err = targetFile.WriteString(
-		"// Document generated by internal/documentation/tasks.go from " + templateDisplayname + "; DO NOT EDIT.\n\n",
-	)
+	tmpl, err := template.ParseFiles(filepath.Join(targetDir, "template.adoc.tmpl"))
 	if err != nil {
 		return err
 	}
-	tmpl, err := template.ParseFiles(templateFilename)
-	if err != nil {
-		return err
-	}
-	return tmpl.Execute(targetFile, data)
+	return RenderTaskDocumentation(targetFile, tmpl, &data)
 }
 
 func parseTasks(helmTemplateOutput []byte) ([]*tekton.Task, error) {
@@ -131,3 +123,47 @@ func RenderTasks(tasksSourceDir, descriptionsSourceDir, targetDir string) error
 	}
 	return nil
 }
+
+func ParseTask(f []byte, desc []byte) (*Task, error) {
+	var t tekton.Task
+	err := yaml.Unmarshal(f, &t)
+	if err != nil {
+		return nil, err
+	}
+	if t.Name == "" {
+		return nil, errors.New("encountered empty name, something is wrong with the task")
+	}
+	task := &Task{
+		Name:        t.Name,
+		Description: string(desc),
+		Params:      []Param{},
+		Results:     []Result{},
+	}
+	for _, p := range t.Spec.Params {
+		defaultValue := ""
+		if p.Default != nil {
+			defaultValue = p.Default.StringVal
+		}
+		task.Params = append(task.Params, Param{
+			Name:        p.Name,
+			Default:     defaultValue,
+			Description: p.Description,
+		})
+	}
+	for _, r := range t.Spec.Results {
+		task.Results = append(task.Results, Result{
+			Name:        r.Name,
+			Description: r.Description,
+		})
+	}
+	return task, nil
+}
+
+func RenderTaskDocumentation(w io.Writer, tmpl *template.Template, task *Task) error {
+	if _, err := w.Write(
+		[]byte("// File is generated; DO NOT EDIT.\n\n"),
+	); err != nil {
+		return err
+	}
+	return tmpl.Execute(w, task)
+}

internal/projectpath/root.go

@@ -11,3 +11,7 @@ var (
 	// Root folder of this project
 	Root = filepath.Join(filepath.Dir(b), "../..")
 )
+
+func RootedPath(path string) string {
+	return filepath.Join(Root, path)
+}