implment STAC specification (#2)

README.md

@@ -56,9 +56,27 @@ geocatalogo index --file=/path/to/record.xml
 # index a directory of metadata records
 geocatalogo index --dir=/path/to/dir
 
+# dedicated importers
+
+# Landsat on AWS (https://aws.amazon.com/public-datasets/landsat/)
+curl http://landsat-pds.s3.amazonaws.com/c1/L8/scene_list.gz | gunzip > /tmp/scene_list
+landsat-aws-importer --file /tmp/scene_list
+
 # search index
 geocatalogo search --term=landsat
 
+# search by bbox
+geocatalogo search --bbox -152,42,-52,84
+
+# search by time instant
+geocatalogo search --time 2018-01-19T18:28:02Z
+
+# search by time range
+geocatalogo search --time 2007-11-11T12:43:29Z,2018-01-19T18:28:02Z
+
+# search by any combination exclusively (term, bbox, time)
+geocatalogo search --time 2007-11-11T12:43:29Z,2018-01-19T18:28:02Z --bbox -152,42,-52,84 --term landsat
+
 # get a metadata record by id
 geocatalogo get --id=12345
 
@@ -68,7 +86,9 @@ geocatalogo get --id=12345,67890
 # run as an HTTP server (default port 8000)
 geocatalogo serve
 # run as an HTTP server on a custom port
-geocatalogo serve -port 8001
+geocatalogo serve --port 8001
+# run as an HTTP server honouring the STAC API
+geocatalogo serve --api stac
 
 # get version
 geocatalogo version

cmd/geocatalogo/main.go

@@ -34,19 +34,25 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
 	"flag"
+	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
+
 	"github.com/tomkralidis/geocatalogo"
 	"github.com/tomkralidis/geocatalogo/config"
 	"github.com/tomkralidis/geocatalogo/metadata/parsers"
 	"github.com/tomkralidis/geocatalogo/repository"
 )
 
 func main() {
+	var router *mux.Router
 	var plural = ""
+	var bbox []float64
+	var time_ []time.Time
 	var fileCount = 0
 	var fileCounter = 1
 	fileList := []string{}
@@ -71,6 +77,8 @@ func main() {
 
 	searchCommand := flag.NewFlagSet("search", flag.ExitOnError)
 	termFlag := searchCommand.String("term", "", "Search term(s)")
+	bboxFlag := searchCommand.String("bbox", "", "Bounding box (minx,miny,maxx,maxy)")
+	timeFlag := searchCommand.String("time", "", "Time (t1[,t2]), RFC3339 format")
 	fromFlag := searchCommand.Int("from", 0, "Start position / offset (default=0)")
 	sizeFlag := searchCommand.Int("size", 10, "Number of results to return (default=10)")
 
@@ -79,6 +87,7 @@ func main() {
 
 	serveCommand := flag.NewFlagSet("serve", flag.ExitOnError)
 	portFlag := serveCommand.Int("port", 8000, "port")
+	apiFlag := serveCommand.String("api", "default", "API to serve (default, stac)")
 
 	versionCommand := flag.NewFlagSet("version", flag.ExitOnError)
 
@@ -183,28 +192,47 @@ func main() {
 			fileCounter++
 		}
 	} else if searchCommand.Parsed() {
-		if *termFlag == "" {
-			fmt.Println("Please provide search term")
-			os.Exit(10005)
+		if *bboxFlag != "" {
+			bboxTokens := strings.Split(*bboxFlag, ",")
+			if len(bboxTokens) != 4 {
+				fmt.Println("bbox format error (should be minx,miny,maxx,maxy)")
+				os.Exit(10006)
+			}
+			for _, b := range bboxTokens {
+				b_, _ := strconv.ParseFloat(b, 64)
+				bbox = append(bbox, b_)
+			}
 		}
-		results := cat.Search(*termFlag, *fromFlag, *sizeFlag)
+		if *timeFlag != "" {
+			for _, t := range strings.Split(*timeFlag, ",") {
+				timestep, err := time.Parse(time.RFC3339, t)
+				if err != nil {
+					fmt.Println("time format error (should be ISO 8601/RFC3339)")
+					os.Exit(10007)
+				}
+				time_ = append(time_, timestep)
+			}
+		}
+		results := cat.Search(*termFlag, bbox, time_, *fromFlag, *sizeFlag)
 		fmt.Printf("Found %d records\n", results.Matches)
 		for _, result := range results.Records {
 			fmt.Printf("    %s - %s\n", result.Properties.Identifier, result.Properties.Title)
 		}
 	} else if serveCommand.Parsed() {
 		fmt.Printf("Serving on port %d\n", *portFlag)
-		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-			geocatalogo.Handler(w, r, cat)
-		})
-		if err := http.ListenAndServe(fmt.Sprintf(":%d", *portFlag), nil); err != nil {
+		if *apiFlag == "stac" {
+			router = geocatalogo.STACRouter(cat)
+		} else { // csw3-opensearch is the default
+			router = geocatalogo.CSW3OpenSearchRouter(cat)
+		}
+		if err := http.ListenAndServe(fmt.Sprintf(":%d", *portFlag), router); err != nil {
 			fmt.Println(err)
-			os.Exit(10006)
+			os.Exit(10008)
 		}
 	} else if getCommand.Parsed() {
 		if *idFlag == "" {
 			fmt.Println("Please provide identifier")
-			os.Exit(10007)
+			os.Exit(10009)
 		}
 		recordids := strings.Split(*idFlag, ",")
 		results := cat.Get(recordids)

cmd/landsat-aws-importer/main.go

@@ -108,13 +108,14 @@ func main() {
 
 		metadataRecord.Geometry.Type = "Polygon"
 
-		var coordinates = [][2]float64{
+		var coordinates = [][][2]float64{{
 			{min_lon, min_lat},
 			{min_lon, max_lat},
 			{max_lon, max_lat},
-			{max_lon, min_lon},
+			{max_lon, min_lat},
 			{min_lon, min_lat},
-		}
+		}}
+
 		metadataRecord.Geometry.Coordinates = coordinates
 
 		metadataRecord.Properties.Geocatalogo.Schema = "local"

config/config.go

@@ -37,12 +37,13 @@ import (
 // Config provides an object model for configuration.
 type Config struct {
 	Server struct {
-		URL         string
-		MimeType    string
-		Encoding    string
-		Language    string
-		PrettyPrint bool
-		Limit       int
+		ApiDataBasedir string
+		URL            string
+		MimeType       string
+		Encoding       string
+		Language       string
+		PrettyPrint    bool
+		Limit          int
 	}
 	Logging struct {
 		Level   string
@@ -93,6 +94,8 @@ func LoadFromEnv() Config {
 		pair := strings.Split(e, "=")
 
 		switch pair[0] {
+		case "GEOCATALOGO_SERVER_API_DATA_BASEDIR":
+			cfg.Server.ApiDataBasedir = pair[1]
 		case "GEOCATALOGO_SERVER_URL":
 			cfg.Server.URL = pair[1]
 		case "GEOCATALOGO_SERVER_MIMETYPE":

geocatalogo-config.env

@@ -1,4 +1,5 @@
 
+export GEOCATALOGO_SERVER_API_DATA_BASEDIR=/path/to/basedir
 export GEOCATALOGO_SERVER_URL=http://localhost:8001/
 export GEOCATALOGO_SERVER_MIMETYPE=application/json; charset=UTF-8
 export GEOCATALOGO_SERVER_ENCODING=utf-8

geocatalogo-config.yml

@@ -24,6 +24,7 @@
 ##############################################################################
 
 server:
+    api_data_basedir: /path/to/basedir
     url: http://localhost/geocatalogo
     mimetype: application/json; charset=UTF-8
     encoding: utf-8

geocatalogo.go

@@ -29,6 +29,7 @@ package geocatalogo
 
 import (
 	"os"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -95,10 +96,10 @@ func (c *GeoCatalogue) UnIndex() bool {
 }
 
 // Search performs a search/query against the Index
-func (c *GeoCatalogue) Search(term string, from int, size int) search.Results {
+func (c *GeoCatalogue) Search(term string, bbox []float64, timeVal []time.Time, from int, size int) search.Results {
 	sr := search.Results{}
 	log.Info("Searching index")
-	err := c.Repository.Query(term, &sr, from, size)
+	err := c.Repository.Query(term, bbox, timeVal, from, size, &sr)
 	if err != nil {
 		return sr
 	}

metadata/model.go

@@ -72,8 +72,8 @@ type Link struct {
 }
 
 type geometry struct {
-	Type        string       `json:"type"`
-	Coordinates [][2]float64 `json:"coordinates"`
+	Type        string         `json:"type"`
+	Coordinates [][][2]float64 `json:"coordinates"`
 }
 
 type geocatalogo struct {

metadata/parsers/csw_record.go

@@ -72,18 +72,18 @@ func (e *boundingBox) Maxy() (float64, error) {
 }
 
 // BBox generates a list of minx,miny,maxx,maxy
-func (e *boundingBox) BBox() [][2]float64 {
+func (e *boundingBox) BBox() [][][2]float64 {
 	minx, _ := e.Minx()
 	miny, _ := e.Miny()
 	maxx, _ := e.Maxx()
 	maxy, _ := e.Maxy()
-	var a = [][2]float64{
+	var a = [][][2]float64{{
 		{minx, miny},
 		{minx, maxy},
 		{maxx, maxy},
 		{maxx, miny},
 		{minx, miny},
-	}
+	}}
 	return a
 }
 

repository/bleve.go

@@ -114,7 +114,7 @@ func (r *Bleve) Delete() bool {
 }
 
 // Query performs a search against the repository
-func (r *Bleve) Query(term string, sr *search.Results, from int, size int) error {
+func (r *Bleve) Query(term string, from int, size int, sr *search.Results) error {
 	query := bleve.NewQueryStringQuery(term)
 
 	searchRequest := bleve.NewSearchRequest(query)
@@ -190,7 +190,7 @@ func transformResultToRecord(rec *bleveSearch.DocumentMatch) metadata.Record {
 		lr[0] = coords[6].(float64)
 		lr[1] = coords[7].(float64)
 
-		mr.Geometry.Coordinates = [][2]float64{ll, ul, ur, lr, ll}
+		//mr.Geometry.Coordinates = [][2]float64{ll, ul, ur, lr, ll}
 	}
 
 	mr.Properties.Identifier = fmt.Sprintf("%v", rec.Fields["properties.identifier"])