From 84d0589c5c289661ae1b53c1e841fd7d4c99864b Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:49:30 +0200 Subject: [PATCH 1/9] Feat: Experimental support for file handling --- internal/server/server.go | 178 +++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 2 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 17e05fe..235f95e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,11 +1,18 @@ package server import ( + "bufio" + "encoding/json" + "errors" "fmt" + "io" "net/http" + "os" "sort" "time" + "encoding/base64" + "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/mdouchement/standardfile/internal/database" @@ -29,13 +36,37 @@ type Controller struct { RefreshTokenExpirationTime time.Duration } +type Resource struct { + RemoteIdentifier string `json:"remoteIdentifier"` +} + +type ValetRequestParams struct { + Operation string `json:"operation"` + Resources []Resource +} + +type ValetToken struct { + Authorization string `json:"authorization"` + FileId string `json:"fileId"` +} + +func (token *ValetToken) GetFilePath() string { + // TODO check format of fileID + return "/etc/standardfile/database/" + token.FileId +} + // EchoEngine instantiates the wep server. func EchoEngine(ctrl Controller) *echo.Echo { engine := echo.New() engine.Use(middleware.Recover()) // engine.Use(middleware.CSRF()) // not supported by StandardNotes engine.Use(middleware.Secure()) - engine.Use(middleware.CORSWithConfig(middleware.DefaultCORSConfig)) + + // Expose headers for file download + cors := middleware.DefaultCORSConfig + cors.ExposeHeaders = append(cors.ExposeHeaders, "Content-Range", "Accept-Ranges") + engine.Use(middleware.CORSWithConfig(cors)) + engine.Use(middleware.Gzip()) engine.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ @@ -68,6 +99,7 @@ func EchoEngine(ctrl Controller) *echo.Echo { v1 := router.Group("/v1") v1restricted := restricted.Group("/v1") + // // generic handlers // router.GET("/version", func(c echo.Context) error { @@ -142,7 +174,149 @@ func EchoEngine(ctrl Controller) *echo.Echo { v2 := router.Group("/v2") v2.POST("/login", auth.LoginPKCE) v2.POST("/login-params", auth.ParamsPKCE) - //v2restricted := restricted.Group("/v2") + + // + // files + // + v1restricted.POST("/files/valet-tokens", func(c echo.Context) error { + var params ValetRequestParams + if err := c.Bind(¶ms); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + if len(params.Resources) != 1 { + return c.JSON(http.StatusBadRequest, "Multi file requests not supported") + } + + // {"operation":"write","resources":[{"remoteIdentifier":"2ef2a4af-2a3c-41ac-b409-78471e6f4a81","unencryptedFileSize":3427}]} + // {"operation":"delete","resources":[{"remoteIdentifier":"b0383bfa-8d9f-4023-8aa1-5c9e3011a0ef","unencryptedFileSize":0}]} + // {"operation":"read","resources":[{"remoteIdentifier":"2ef2a4af-2a3c-41ac-b409-78471e6f4a81","unencryptedFileSize":0}]} + + var token ValetToken + token.Authorization = c.Request().Header.Get(echo.HeaderAuthorization) + token.FileId = params.Resources[0].RemoteIdentifier + valetTokenJson, err := json.Marshal(token) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + // token := auth + " " + fileId + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "valetToken": base64.StdEncoding.EncodeToString(valetTokenJson), + }) + }) + v1.POST("/files/upload/create-session", func(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + fmt.Println("create-session. valet_token: " + valetTokenJson) + + if _, err := os.Create(token.GetFilePath()); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "uploadId": token.FileId, + }) + }) + v1.POST("/files/upload/close-session", func(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { + return c.JSON(http.StatusBadRequest, "File not created") + } + + fmt.Println("close-session. valet_token: " + valetTokenJson) + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "File uploaded successfully", + }) + }) + v1.POST("/files/upload/chunk", func(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { + return c.JSON(http.StatusBadRequest, "File not created") + } + + chunk_id := c.Request().Header.Get("x-chunk-id") + fmt.Println("chunk. valet_token: " + valetTokenJson + " chunk_id: " + chunk_id) + + f, err := os.OpenFile(token.GetFilePath(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + // remember to close the file + defer f.Close() + + // create new buffer + writer := bufio.NewWriter(f) + reader := c.Request().Body + io.Copy(writer, reader) + + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "Chunk uploaded successfully", + }) + }) + v1.DELETE("/files", func(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + err = os.Remove(token.GetFilePath()) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "File removed successfully", + }) + }) + v1.GET("/files", func(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + return c.File(token.GetFilePath()) + }) // // subscription handlers From 89b164dfc89427eab9604c8c75fc8a19d4acbd05 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Sun, 23 Oct 2022 23:17:33 +0200 Subject: [PATCH 2/9] Refactor: Encapsulate file handling --- internal/server/files.go | 182 ++++++++++++++++++++++++++++++++++++++ internal/server/server.go | 173 ++---------------------------------- 2 files changed, 190 insertions(+), 165 deletions(-) create mode 100644 internal/server/files.go diff --git a/internal/server/files.go b/internal/server/files.go new file mode 100644 index 0000000..c472c14 --- /dev/null +++ b/internal/server/files.go @@ -0,0 +1,182 @@ +package server + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + + "github.com/labstack/echo/v4" +) + +type files struct{} + +type Resource struct { + RemoteIdentifier string `json:"remoteIdentifier"` +} + +type ValetRequestParams struct { + Operation string `json:"operation"` + Resources []Resource +} + +type ValetToken struct { + Authorization string `json:"authorization"` + FileId string `json:"fileId"` +} + +func (token *ValetToken) GetFilePath() string { + // TODO: Check format of fileId (Security) + // TODO: Allow custom path in config + // TODO: Subfolders for each user (Compatible format with official server) + return "/etc/standardfile/database/" + token.FileId +} + +// Provides a valet token that is required to execute an operation +func (h *files) ValetTokens(c echo.Context) error { + var params ValetRequestParams + if err := c.Bind(¶ms); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + if len(params.Resources) != 1 { + return c.JSON(http.StatusBadRequest, "Multi file requests not supported") + } + + var token ValetToken + token.Authorization = c.Request().Header.Get(echo.HeaderAuthorization) + token.FileId = params.Resources[0].RemoteIdentifier + valetTokenJson, err := json.Marshal(token) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "valetToken": base64.StdEncoding.EncodeToString(valetTokenJson), + }) +} + +// Called before uploading chunks of a file +func (h *files) CreateUploadSession(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + fmt.Println("create-session. valet_token: " + valetTokenJson) + + if _, err := os.Create(token.GetFilePath()); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "uploadId": token.FileId, + }) +} + +// Called when all chunks a file uploaded +func (h *files) CloseUploadSession(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { + return c.JSON(http.StatusBadRequest, "File not created") + } + + fmt.Println("close-session. valet_token: " + valetTokenJson) + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "File uploaded successfully", + }) +} + +// Upload parts of a file in an existing session +func (h *files) UploadChunk(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { + return c.JSON(http.StatusBadRequest, "File not created") + } + + chunk_id := c.Request().Header.Get("x-chunk-id") + fmt.Println("chunk. valet_token: " + valetTokenJson + " chunk_id: " + chunk_id) + + f, err := os.OpenFile(token.GetFilePath(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + defer f.Close() + + // Create new buffer + writer := bufio.NewWriter(f) + reader := c.Request().Body + io.Copy(writer, reader) + + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "Chunk uploaded successfully", + }) +} + +// Delete file from server +func (h *files) Delete(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + err = os.Remove(token.GetFilePath()) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + return c.JSON(http.StatusOK, echo.Map{ + "success": true, + "message": "File removed successfully", + }) +} + +// Send encrypted file to client +func (h *files) Download(c echo.Context) error { + valetTokenBase64 := c.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return c.JSON(http.StatusBadRequest, err) + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return c.JSON(http.StatusBadRequest, err) + } + return c.File(token.GetFilePath()) +} diff --git a/internal/server/server.go b/internal/server/server.go index 235f95e..478c5bf 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,18 +1,11 @@ package server import ( - "bufio" - "encoding/json" - "errors" "fmt" - "io" "net/http" - "os" "sort" "time" - "encoding/base64" - "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/mdouchement/standardfile/internal/database" @@ -36,25 +29,6 @@ type Controller struct { RefreshTokenExpirationTime time.Duration } -type Resource struct { - RemoteIdentifier string `json:"remoteIdentifier"` -} - -type ValetRequestParams struct { - Operation string `json:"operation"` - Resources []Resource -} - -type ValetToken struct { - Authorization string `json:"authorization"` - FileId string `json:"fileId"` -} - -func (token *ValetToken) GetFilePath() string { - // TODO check format of fileID - return "/etc/standardfile/database/" + token.FileId -} - // EchoEngine instantiates the wep server. func EchoEngine(ctrl Controller) *echo.Echo { engine := echo.New() @@ -178,145 +152,14 @@ func EchoEngine(ctrl Controller) *echo.Echo { // // files // - v1restricted.POST("/files/valet-tokens", func(c echo.Context) error { - var params ValetRequestParams - if err := c.Bind(¶ms); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - - if len(params.Resources) != 1 { - return c.JSON(http.StatusBadRequest, "Multi file requests not supported") - } - - // {"operation":"write","resources":[{"remoteIdentifier":"2ef2a4af-2a3c-41ac-b409-78471e6f4a81","unencryptedFileSize":3427}]} - // {"operation":"delete","resources":[{"remoteIdentifier":"b0383bfa-8d9f-4023-8aa1-5c9e3011a0ef","unencryptedFileSize":0}]} - // {"operation":"read","resources":[{"remoteIdentifier":"2ef2a4af-2a3c-41ac-b409-78471e6f4a81","unencryptedFileSize":0}]} - - var token ValetToken - token.Authorization = c.Request().Header.Get(echo.HeaderAuthorization) - token.FileId = params.Resources[0].RemoteIdentifier - valetTokenJson, err := json.Marshal(token) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - - // token := auth + " " + fileId - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "valetToken": base64.StdEncoding.EncodeToString(valetTokenJson), - }) - }) - v1.POST("/files/upload/create-session", func(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - valetTokenJson := string(valetTokenBytes) - - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - - fmt.Println("create-session. valet_token: " + valetTokenJson) - - if _, err := os.Create(token.GetFilePath()); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "uploadId": token.FileId, - }) - }) - v1.POST("/files/upload/close-session", func(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { - return c.JSON(http.StatusBadRequest, "File not created") - } - - fmt.Println("close-session. valet_token: " + valetTokenJson) - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "File uploaded successfully", - }) - }) - v1.POST("/files/upload/chunk", func(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { - return c.JSON(http.StatusBadRequest, "File not created") - } - - chunk_id := c.Request().Header.Get("x-chunk-id") - fmt.Println("chunk. valet_token: " + valetTokenJson + " chunk_id: " + chunk_id) - - f, err := os.OpenFile(token.GetFilePath(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - // remember to close the file - defer f.Close() - - // create new buffer - writer := bufio.NewWriter(f) - reader := c.Request().Body - io.Copy(writer, reader) - - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "Chunk uploaded successfully", - }) - }) - v1.DELETE("/files", func(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - err = os.Remove(token.GetFilePath()) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "File removed successfully", - }) - }) - v1.GET("/files", func(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) - if err != nil { - return c.JSON(http.StatusBadRequest, err) - } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - return c.File(token.GetFilePath()) - }) + files := &files{} + v1restricted.POST("/files/valet-tokens", files.ValetTokens) + // Following endpoints are authorized via valet token + v1.POST("/files/upload/create-session", files.CreateUploadSession) + v1.POST("/files/upload/close-session", files.CloseUploadSession) + v1.POST("/files/upload/chunk", files.UploadChunk) + v1.DELETE("/files", files.Delete) + v1.GET("/files", files.Download) // // subscription handlers From 86eb6a30c73c105916271bd2967bd50eb932d079 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:59:53 +0100 Subject: [PATCH 3/9] Fix: Handle errors in file-upload --- internal/server/files.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/server/files.go b/internal/server/files.go index c472c14..7a7a9f0 100644 --- a/internal/server/files.go +++ b/internal/server/files.go @@ -136,7 +136,9 @@ func (h *files) UploadChunk(c echo.Context) error { // Create new buffer writer := bufio.NewWriter(f) reader := c.Request().Body - io.Copy(writer, reader) + if _, err := io.Copy(writer, reader); err != nil { + return c.JSON(http.StatusBadRequest, err) + } return c.JSON(http.StatusOK, echo.Map{ "success": true, From bebfa14f7b19dfbb8eec0701795102127268d803 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Thu, 3 Nov 2022 15:38:11 +0100 Subject: [PATCH 4/9] Style: Rewrite comment --- internal/server/files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/files.go b/internal/server/files.go index 7a7a9f0..9dea158 100644 --- a/internal/server/files.go +++ b/internal/server/files.go @@ -87,7 +87,7 @@ func (h *files) CreateUploadSession(c echo.Context) error { }) } -// Called when all chunks a file uploaded +// Called when uploaded all chunks of a file func (h *files) CloseUploadSession(c echo.Context) error { valetTokenBase64 := c.Request().Header.Get("x-valet-token") valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) From 10c222a1b76b85d39a58968d021d9b789008dfd9 Mon Sep 17 00:00:00 2001 From: Simon <26634807+Crusader99@users.noreply.github.com> Date: Thu, 3 Nov 2022 23:25:08 +0100 Subject: [PATCH 5/9] Chore: Add todo comments to GetFilePath function Co-authored-by: mdouchement --- internal/server/files.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/server/files.go b/internal/server/files.go index 9dea158..3be06dc 100644 --- a/internal/server/files.go +++ b/internal/server/files.go @@ -33,6 +33,8 @@ func (token *ValetToken) GetFilePath() string { // TODO: Check format of fileId (Security) // TODO: Allow custom path in config // TODO: Subfolders for each user (Compatible format with official server) + // TODO: use filepath.Join function + // TODO: Fobbid "../../" pattern, may be with https://pkg.go.dev/io/fs#ValidPath return "/etc/standardfile/database/" + token.FileId } From 74a7e5286dfca7225383d37234e4cb452fa6d90d Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Thu, 3 Nov 2022 23:36:00 +0100 Subject: [PATCH 6/9] Style: Use golang's convention in variable names --- internal/server/files.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/server/files.go b/internal/server/files.go index 3be06dc..e3e96c8 100644 --- a/internal/server/files.go +++ b/internal/server/files.go @@ -26,16 +26,16 @@ type ValetRequestParams struct { type ValetToken struct { Authorization string `json:"authorization"` - FileId string `json:"fileId"` + FileID string `json:"fileId"` } func (token *ValetToken) GetFilePath() string { - // TODO: Check format of fileId (Security) + // TODO: Check format of FileID (Security) // TODO: Allow custom path in config // TODO: Subfolders for each user (Compatible format with official server) // TODO: use filepath.Join function // TODO: Fobbid "../../" pattern, may be with https://pkg.go.dev/io/fs#ValidPath - return "/etc/standardfile/database/" + token.FileId + return "/etc/standardfile/database/" + token.FileID } // Provides a valet token that is required to execute an operation @@ -51,7 +51,7 @@ func (h *files) ValetTokens(c echo.Context) error { var token ValetToken token.Authorization = c.Request().Header.Get(echo.HeaderAuthorization) - token.FileId = params.Resources[0].RemoteIdentifier + token.FileID = params.Resources[0].RemoteIdentifier valetTokenJson, err := json.Marshal(token) if err != nil { return c.JSON(http.StatusBadRequest, err) @@ -85,7 +85,7 @@ func (h *files) CreateUploadSession(c echo.Context) error { return c.JSON(http.StatusOK, echo.Map{ "success": true, - "uploadId": token.FileId, + "uploadId": token.FileID, }) } From 88841855e58f7d5f3694316b8336c613c57d7c32 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Thu, 3 Nov 2022 23:57:48 +0100 Subject: [PATCH 7/9] Chore: Use v1valet routes for file-api --- internal/server/server.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 478c5bf..10ad38a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -154,12 +154,14 @@ func EchoEngine(ctrl Controller) *echo.Echo { // files := &files{} v1restricted.POST("/files/valet-tokens", files.ValetTokens) + v1valet := v1.Group("") + // TODO: v1valet.Use(a valet middleware for authentication) // Following endpoints are authorized via valet token - v1.POST("/files/upload/create-session", files.CreateUploadSession) - v1.POST("/files/upload/close-session", files.CloseUploadSession) - v1.POST("/files/upload/chunk", files.UploadChunk) - v1.DELETE("/files", files.Delete) - v1.GET("/files", files.Download) + v1valet.POST("/files/upload/create-session", files.CreateUploadSession) + v1valet.POST("/files/upload/close-session", files.CloseUploadSession) + v1valet.POST("/files/upload/chunk", files.UploadChunk) + v1valet.DELETE("/files", files.Delete) + v1valet.GET("/files", files.Download) // // subscription handlers From ce45445e63b04bdb4e3bd4a1de7ab733cb21c069 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Fri, 4 Nov 2022 00:32:06 +0100 Subject: [PATCH 8/9] Fix: Correct error handling for file-api --- internal/server/files.go | 162 ++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 79 deletions(-) diff --git a/internal/server/files.go b/internal/server/files.go index e3e96c8..f638443 100644 --- a/internal/server/files.go +++ b/internal/server/files.go @@ -5,11 +5,13 @@ import ( "encoding/base64" "encoding/json" "errors" - "fmt" "io" + "io/fs" "net/http" "os" + "path/filepath" + "github.com/gofrs/uuid" "github.com/labstack/echo/v4" ) @@ -19,6 +21,28 @@ type Resource struct { RemoteIdentifier string `json:"remoteIdentifier"` } +func respond(context echo.Context, code int, message string) error { + success := code < 300 + return context.JSON(code, echo.Map{ + "success": success, + "message": message, + }) +} + +func readValetToken(context echo.Context) (*ValetToken, error) { + valetTokenBase64 := context.Request().Header.Get("x-valet-token") + valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + if err != nil { + return nil, errors.New("Unable to parse base64 valet token") + } + valetTokenJson := string(valetTokenBytes) + var token ValetToken + if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { + return nil, errors.New("Unable to parse json valet token") + } + return &token, nil +} + type ValetRequestParams struct { Operation string `json:"operation"` Resources []Resource @@ -29,26 +53,29 @@ type ValetToken struct { FileID string `json:"fileId"` } -func (token *ValetToken) GetFilePath() string { - // TODO: Check format of FileID (Security) +func (token *ValetToken) getFilePath() (*string, error) { + id, err := uuid.FromString(token.FileID) + if err != nil { + return nil, errors.New("Unable to parse json valet token") + } else if !fs.ValidPath(id.String()) { + return nil, errors.New("Invalid path") + } // TODO: Allow custom path in config // TODO: Subfolders for each user (Compatible format with official server) - // TODO: use filepath.Join function - // TODO: Fobbid "../../" pattern, may be with https://pkg.go.dev/io/fs#ValidPath - return "/etc/standardfile/database/" + token.FileID + path := filepath.Join("etc", "standardfile", "database", id.String()) + return &path, nil } // Provides a valet token that is required to execute an operation func (h *files) ValetTokens(c echo.Context) error { var params ValetRequestParams if err := c.Bind(¶ms); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - - if len(params.Resources) != 1 { - return c.JSON(http.StatusBadRequest, "Multi file requests not supported") + return respond(c, http.StatusBadRequest, "Unable to parse request") + } else if len(params.Resources) != 1 { + return respond(c, http.StatusBadRequest, "Multi file requests unsupported") } + // Generate valet token. Used for actual file operations var token ValetToken token.Authorization = c.Request().Header.Get(echo.HeaderAuthorization) token.FileID = params.Resources[0].RemoteIdentifier @@ -56,7 +83,6 @@ func (h *files) ValetTokens(c echo.Context) error { if err != nil { return c.JSON(http.StatusBadRequest, err) } - return c.JSON(http.StatusOK, echo.Map{ "success": true, "valetToken": base64.StdEncoding.EncodeToString(valetTokenJson), @@ -65,21 +91,19 @@ func (h *files) ValetTokens(c echo.Context) error { // Called before uploading chunks of a file func (h *files) CreateUploadSession(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + token, err := readValetToken(c) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) + // Validate file path + path, err := token.getFilePath() + if err != nil { + return respond(c, http.StatusBadRequest, err.Error()) } - fmt.Println("create-session. valet_token: " + valetTokenJson) - - if _, err := os.Create(token.GetFilePath()); err != nil { + // Create empty file + if _, err := os.Create(*path); err != nil { return c.JSON(http.StatusBadRequest, err) } @@ -91,96 +115,76 @@ func (h *files) CreateUploadSession(c echo.Context) error { // Called when uploaded all chunks of a file func (h *files) CloseUploadSession(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + token, err := readValetToken(c) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { - return c.JSON(http.StatusBadRequest, "File not created") + path, err := token.getFilePath() + if err != nil { + return respond(c, http.StatusBadRequest, err.Error()) + } else if _, err := os.Stat(*path); errors.Is(err, os.ErrNotExist) { + return respond(c, http.StatusBadRequest, "File not created") } - - fmt.Println("close-session. valet_token: " + valetTokenJson) - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "File uploaded successfully", - }) + return respond(c, http.StatusOK, "File uploaded successfully") } // Upload parts of a file in an existing session func (h *files) UploadChunk(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + token, err := readValetToken(c) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } else if _, err := os.Stat(token.GetFilePath()); errors.Is(err, os.ErrNotExist) { + + // Validate file path + path, err := token.getFilePath() + if err != nil { + return respond(c, http.StatusBadRequest, err.Error()) + } else if _, err := os.Stat(*path); errors.Is(err, os.ErrNotExist) { return c.JSON(http.StatusBadRequest, "File not created") } - chunk_id := c.Request().Header.Get("x-chunk-id") - fmt.Println("chunk. valet_token: " + valetTokenJson + " chunk_id: " + chunk_id) - - f, err := os.OpenFile(token.GetFilePath(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + // Open file + f, err := os.OpenFile(*path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, "Unable to open file") } defer f.Close() - // Create new buffer + // Append chunk to file via buffer writer := bufio.NewWriter(f) reader := c.Request().Body if _, err := io.Copy(writer, reader); err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, "Unable to store file") } - - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "Chunk uploaded successfully", - }) + return respond(c, http.StatusOK, "Chunk uploaded successfully") } // Delete file from server func (h *files) Delete(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + token, err := readValetToken(c) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) - } - err = os.Remove(token.GetFilePath()) + path, err := token.getFilePath() if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) + } else if err := os.Remove(*path); err != nil { + return respond(c, http.StatusBadRequest, "Unable to remove file") } - return c.JSON(http.StatusOK, echo.Map{ - "success": true, - "message": "File removed successfully", - }) + return respond(c, http.StatusOK, "File removed successfully") } // Send encrypted file to client func (h *files) Download(c echo.Context) error { - valetTokenBase64 := c.Request().Header.Get("x-valet-token") - valetTokenBytes, err := base64.StdEncoding.DecodeString(valetTokenBase64) + token, err := readValetToken(c) if err != nil { - return c.JSON(http.StatusBadRequest, err) + return respond(c, http.StatusBadRequest, err.Error()) } - valetTokenJson := string(valetTokenBytes) - var token ValetToken - if err := json.Unmarshal([]byte(valetTokenJson), &token); err != nil { - return c.JSON(http.StatusBadRequest, err) + + // Validate file path + path, err := token.getFilePath() + if err != nil { + return respond(c, http.StatusBadRequest, err.Error()) } - return c.File(token.GetFilePath()) + return c.File(*path) } From 3ebd92c41ed03f379babe6cfd1e34c06c96b85a8 Mon Sep 17 00:00:00 2001 From: Simon Forschner <26634807+Crusader99@users.noreply.github.com> Date: Thu, 10 Nov 2022 02:02:00 +0100 Subject: [PATCH 9/9] Feat: Configurable filesServerUrl --- cmd/standardfile/main.go | 1 + internal/server/server.go | 5 ++++- internal/server/subscription_handler.go | 10 +++++++++- standardfile.yml | 12 +++++++++++- 4 files changed, 25 insertions(+), 3 deletions(-) mode change 100644 => 100755 standardfile.yml diff --git a/cmd/standardfile/main.go b/cmd/standardfile/main.go index 4799a8d..df1734f 100644 --- a/cmd/standardfile/main.go +++ b/cmd/standardfile/main.go @@ -172,6 +172,7 @@ var ( NoRegistration: konf.Bool("no_registration"), ShowRealVersion: konf.Bool("show_real_version"), EnableSubscription: konf.Bool("enable_subscription"), + FilesServerUrl: konf.String("files_server_url"), SigningKey: configSecretKey, SessionSecret: kdf(32, configSessionSecret), AccessTokenExpirationTime: konf.MustDuration("session.access_token_ttl"), diff --git a/internal/server/server.go b/internal/server/server.go index 10ad38a..98ca8ad 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -21,6 +21,7 @@ type Controller struct { NoRegistration bool ShowRealVersion bool EnableSubscription bool + FilesServerUrl string // JWT params SigningKey []byte // Session params @@ -167,7 +168,9 @@ func EchoEngine(ctrl Controller) *echo.Echo { // subscription handlers // if ctrl.EnableSubscription { - subscription := &subscription{} + subscription := &subscription{ + filesServerUrl: ctrl.FilesServerUrl, + } router.GET("/v2/subscriptions", func(c echo.Context) error { return c.HTML(http.StatusInternalServerError, "getaddrinfo EAI_AGAIN payments") }) diff --git a/internal/server/subscription_handler.go b/internal/server/subscription_handler.go index dd201c6..e27ec7e 100644 --- a/internal/server/subscription_handler.go +++ b/internal/server/subscription_handler.go @@ -7,7 +7,9 @@ import ( "github.com/labstack/echo/v4" ) -type subscription struct{} +type subscription struct { + filesServerUrl string +} func (h *subscription) SubscriptionV1(c echo.Context) error { user := currentUser(c) @@ -31,6 +33,9 @@ func (h *subscription) SubscriptionV1(c echo.Context) error { }, }, }, + "server": echo.Map{ + "filesServerUrl": h.filesServerUrl, + }, }, "data": echo.Map{ "success": true, @@ -70,6 +75,9 @@ func (h *subscription) Features(c echo.Context) error { }, }, }, + "server": echo.Map{ + "filesServerUrl": h.filesServerUrl, + }, }, "data": echo.Map{ "success": true, diff --git a/standardfile.yml b/standardfile.yml old mode 100644 new mode 100755 index f3d230c..5245044 --- a/standardfile.yml +++ b/standardfile.yml @@ -7,21 +7,27 @@ # Address to bind address: "localhost:5000" + # Disable registration no_registration: false + # Show real version in `GET /version' show_real_version: false + # Database folder path; empty value means current directory database_path: "" + # Secret key used for JWT authentication (before 004 and 20200115) # If missing, will be read from $CREDENTIALS_DIRECTORY/secret_key file secret_key: jwt-development + # Session used for authentication (since 004 and 20200115) session: # If missing, will be read from $CREDENTIALS_DIRECTORY/session.secret file secret: paseto-development access_token_ttl: 1440h # 60 days expressed in Golang's time.Duration format refresh_token_ttl: 8760h # 1 year + # This option enables paid features in the official StandardNotes client. # If you want to enables these features, you should consider to # donate to the StandardNotes project as they say: @@ -33,4 +39,8 @@ session: # # This project https://github.com/mdouchement/standardfile does not intend to # conflict with the business model of StandardNotes project or seek compensation. -enable_subscription: false \ No newline at end of file +enable_subscription: false + +# Insert a publicly accessible URL to this server and set enable_subscription +# to 'true' for working file upload/download. +files_server_url: "http://localhost:5000" \ No newline at end of file