diff --git a/.dockerignore b/.dockerignore index d913cc9..a6a9bb6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ web/content/datasets/* charts/ -dev_utils/ \ No newline at end of file +dev_utils/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 49e38a1..8c66d00 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ web/public/* web/content/datasets/test* web/data/* -dev_utils/config.yaml \ No newline at end of file +web/static/pagefind/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0419f24..efff833 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,10 @@ RUN go build -o app . FROM alpine:3.17 RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community hugo +RUN wget https://github.com/CloudCannon/pagefind/releases/download/v1.1.0/pagefind-v1.1.0-x86_64-unknown-linux-musl.tar.gz && \ + tar -xvf pagefind-v1.1.0-x86_64-unknown-linux-musl.tar.gz && \ + mv pagefind /usr/bin && \ + rm pagefind-v1.1.0-x86_64-unknown-linux-musl.tar.gz COPY --from=build /lp_app/app . COPY --from=build /lp_app/web web/ CMD ["./app"] diff --git a/download_metadata.go b/download_metadata.go index 374b848..466e3ca 100644 --- a/download_metadata.go +++ b/download_metadata.go @@ -35,11 +35,9 @@ func metadataDownloader(Metadataclient *MetadataBackend) { err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)) if err != nil { log.Fatal("Error while downloading metadata files from metadata bucket", err) - } else { - log.Infoln("Completed downloading metadatafiles") - } } + log.Infoln("Completed downloading metadata files") } diff --git a/header_extractor.go b/header_extractor.go new file mode 100644 index 0000000..6dc97dc --- /dev/null +++ b/header_extractor.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/xml" + "log" + "os" +) + +// Struct definitions +type LandingPageSet struct { + LandingPage LandingPage `xml:"LANDING_PAGE"` +} + +type LandingPage struct { + Attributes Attributes `xml:"ATTRIBUTES"` +} + +type Attributes struct { + StringAttributes []StringAttribute `xml:"STRING_ATTRIBUTE"` +} + +type StringAttribute struct { + Tag string `xml:"TAG"` + Value string `xml:"VALUE"` +} + +// Function to extract header value from XML content +func getHeaderValueFromXMLContent(xmlContent []byte) (string, error) { + var landingPageSet LandingPageSet + err := xml.Unmarshal(xmlContent, &landingPageSet) + if err != nil { + log.Fatal("Unmarshalling XML file for header failed", err) + } + + // Iterate over the string attributes to find the header value + for _, attr := range landingPageSet.LandingPage.Attributes.StringAttributes { + if attr.Tag == "header" { + return attr.Value, nil + } + } + return "sf", err + +} + +// Function to read the XML file and return its content +func readXMLFile(filePath string) ([]byte, error) { + return os.ReadFile(filePath) +} diff --git a/main.go b/main.go index 8301cd3..d8d01ad 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "os" "os/exec" log "github.com/sirupsen/logrus" @@ -11,16 +12,28 @@ func main() { log.Infoln("started app successfully") mConf := getMetadataConfig() Metadataclient := connectMetadatas3(mConf) - log.Infof("Connection to the bucket established") + log.Infof("Writing markdownfiles for respective XMLs") metadataDownloader(Metadataclient) markDownCreator() - cmd := exec.Command("hugo") - cmd.Dir = "./web/" - cmd.Run() + hugo_cmd := exec.Command("hugo") + hugo_cmd.Dir = "./web/" + hugo_cmd.Stdout = os.Stdout + hugo_cmd.Stderr = os.Stderr + err := hugo_cmd.Run() + if err != nil { + log.Fatal(err) + } log.Infof("Hugo successfully built") + pagefind_cmd := exec.Command("pagefind", "--site", "web/public/", "--output-path", "web/static/pagefind/") + pagefind_cmd.Stdout = os.Stdout + pagefind_cmd.Stderr = os.Stderr + pagefind_err := pagefind_cmd.Run() + if pagefind_err != nil { + log.Fatal(pagefind_err) + } + log.Infof("Pagefind modules successfully built") dConf := getDeploymentConfig() DeploymenClient := connectDeployments3(dConf) - log.Infof("Connection to the bucket established") - test(DeploymenClient) + staticSiteUploader(DeploymenClient) } diff --git a/markdown_writer.go b/markdown_writer.go index c6676dd..a309937 100644 --- a/markdown_writer.go +++ b/markdown_writer.go @@ -35,20 +35,30 @@ func markDownCreator() { // Read XML file name xmlFileName := filepath.Base(xmlFilePath) + log.Debug(xmlFilePath) + xmlContent, err := readXMLFile(xmlFilePath) + if err != nil { + log.Fatalf("Error reading the XML file %V", err) + } + headerValue, err := getHeaderValueFromXMLContent(xmlContent) + if err != nil { + log.Fatal("Error while getting header value from XML file", err) + } + log.Debugln("Header value: %V", headerValue) // Remove file extension fileNameWithoutExt := strings.TrimSuffix(xmlFileName, filepath.Ext(xmlFileName)) // Markdown content - markdownContent := fmt.Sprintf(` ---- + markdownContent := fmt.Sprintf(`--- +title: "%s" --- {{< datafetch variable="%s" >}} Filename of the associated XML file: %s -`, fileNameWithoutExt, xmlFileName) +`, headerValue, fileNameWithoutExt, xmlFileName) // Create Markdown file mdFileName := filepath.Join(markdownDir, fileNameWithoutExt+".md") @@ -66,7 +76,7 @@ Filename of the associated XML file: %s return nil } - log.Infoln("Markdown file %S created successfully!\n", mdFileName) + log.Debug("Markdown file %S created successfully!\n", mdFileName) return nil }) @@ -74,4 +84,5 @@ Filename of the associated XML file: %s if err != nil { log.Fatal("Error walking through directory:", err) } + } diff --git a/static_files_uploader.go b/static_files_uploader.go index 0369b63..60c2e16 100644 --- a/static_files_uploader.go +++ b/static_files_uploader.go @@ -14,7 +14,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -func test(DeploymenClient *DeploymentBackend) { +func staticSiteUploader(DeploymenClient *DeploymentBackend) { var ( localPath = "web/public/" bucket = DeploymenClient.Bucket @@ -50,6 +50,9 @@ func test(DeploymenClient *DeploymentBackend) { if ext == ".css" { contentType = "text/css;" } + if ext == ".svg" { + contentType = "image/svg+xml;" + } if err != nil { log.Fatalln("File bytes empty", path, err) @@ -68,8 +71,10 @@ func test(DeploymenClient *DeploymentBackend) { if err != nil { log.Fatalln("Failed to upload", path, err) } - log.Infoln("Uploaded", path, result.Location) + log.Debugln("Uploaded", path, result.Location) } + log.Infoln("Successfully uploaded built static site to the bucket") + } type fileWalk chan string diff --git a/web/content/datasets/_index.md b/web/content/datasets/_index.md index 3facddc..618f4a7 100644 --- a/web/content/datasets/_index.md +++ b/web/content/datasets/_index.md @@ -1,5 +1,5 @@ --- -title: "Test" +title: "Search" date: 2024-03-20T10:31:33+01:00 layout: test --- diff --git a/web/layouts/_default/baseof.html b/web/layouts/_default/baseof.html index d759183..6d2e967 100644 --- a/web/layouts/_default/baseof.html +++ b/web/layouts/_default/baseof.html @@ -2,17 +2,16 @@ {{ partial "head.html" . }} -{{ partial "nav.html" . }} + + + {{ partial "nav.html" . }} -{{ block "header" .}}

{{ .Site.Title}}

{{ .Site.Params.slogan }}

-{{ end }} - -
+
{{ block "main" . }} {{ end }} diff --git a/web/layouts/datasets/list.html b/web/layouts/datasets/list.html index 494b337..8b6e3fc 100644 --- a/web/layouts/datasets/list.html +++ b/web/layouts/datasets/list.html @@ -1,11 +1,17 @@ {{ define "main" }}

Bigpicture Datasets

-