From 05b696c0260e507564b8c96e73a32bf23b1d84d3 Mon Sep 17 00:00:00 2001 From: amrita Date: Mon, 16 Oct 2023 15:00:24 +0545 Subject: [PATCH 1/3] tus docs --- docs/apis/http/tus_upload.md | 189 ++++++++++++++++++++++++++++++++ docs/helpers/extended_vars.yaml | 4 +- 2 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 docs/apis/http/tus_upload.md diff --git a/docs/apis/http/tus_upload.md b/docs/apis/http/tus_upload.md new file mode 100644 index 00000000000..96227c3b27f --- /dev/null +++ b/docs/apis/http/tus_upload.md @@ -0,0 +1,189 @@ +--- +title: "tus upload" +date: 2023-10-10T00:00:00+00:00 +weight: 21 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/apis/http +geekdocFilePath: tus_upload.md +geekdocCollapseSection: true +--- + +Infinite Scale supports tus resumable-upload protocol. +tus is a robust, modular, and open protocol designed to resume large file uploads reliably over HTTP. +In situations where file uploads might be interrupted due to network issues, browser crashes, or other unforeseen interruptions, +tus ensures that uploads can be resumed from the point of failure without losing data. +This documentation shows some basic examples, refer [tus official site](https://tus.io/protocols/resumable-upload) for more details. + +## Upload in Chunks +### Create an Upload URL +The client must send a POST request against a known upload creation URL to request a new upload resource. +The filename has to be provided in base64-encoded format. + +Example: +```shell +# base64 encoded filename 'tustest.txt' is 'dHVzdGVzdC50eHQ=' +echo -n 'tustest.txt' | base64 +``` + +{{< tabs "create-upload-url" >}} +{{< tab "Request" >}} +```shell +curl -ks -XPOST https://ocis.test/remote.php/dav/spaces/8d72036d-14a5-490f-889e-414064156402$196ac304-7b88-44ce-a4db-c4becef0d2e0 \ +-H "Authorization: Bearer eyJhbGciOiJQUzI..."\ +-H "Tus-Resumable: 1.0.0" \ +-H "Upload-Length: 10" \ +-H "Upload-Metadata: filename dHVzdGVzdC50eHQ=" +``` +{{< /tab >}} + +{{< tab "Response - 201 Created" >}} +``` +< HTTP/1.1 201 Created +< Access-Control-Allow-Headers: Tus-Resumable, Upload-Length, Upload-Metadata, If-Match +< Access-Control-Allow-Origin: * +< Access-Control-Expose-Headers: Tus-Resumable, Upload-Offset, Location +< Content-Length: 0 +< Content-Security-Policy: default-src 'none'; +< Date: Mon, 16 Oct 2023 08:49:39 GMT +< Location: https://ocis.test/data/eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZXZhIiwiZXhwIjoxNjk3NTMyNTc5LCJpYXQiOjE2OTc0NDYxNzksInRhcmdldCI6Imh0dHA6Ly9sb2NhbGhvc3Q6OTE1OC9kYXRhL3R1cy8zYTU3ZWZlMS04MzE0LTQ4MGEtOWY5Ny04N2Q1YzBjYTJhMTgifQ.FbrlY7mdOfsbFgMrP8OtcHlCEq72a2ZVnPD2iBo9MfM +< Tus-Extension: creation,creation-with-upload,checksum,expiration +< Tus-Resumable: 1.0.0 +< Vary: Origin +< X-Content-Type-Options: nosniff +< X-Download-Options: noopen +< X-Frame-Options: SAMEORIGIN +< X-Permitted-Cross-Domain-Policies: none +< X-Request-Id: xxxxxxxxxxxxxxxxxxxxxx +< X-Robots-Tag: none +< X-Xss-Protection: 1; mode=block +< +* Connection #0 to host localhost left intact +``` +{{< /tab >}} +{{< /tabs >}} + +The server will return a temporary upload URL in the location header of the response: +``` +< Location: https://ocis.test/data/eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZXZhIiwiZXhwIjoxNjk3NTMyNTc5LCJpYXQiOjE2OTc0NDYxNzksInRhcmdldCI6Imh0dHA6Ly9sb2NhbGhvc3Q6OTE1OC9kYXRhL3R1cy8zYTU3ZWZlMS04MzE0LTQ4MGEtOWY5Ny04N2Q1YzBjYTJhMTgifQ.FbrlY7mdOfsbFgMrP8OtcHlCEq72a2ZVnPD2iBo9MfM +``` + +### Upload the First Chunk +Once a temporary upload URL has been created, a client can send a PATCH request to upload a file. The file content should be sent in the body of the request: +{{< tabs "upload-the-first-chunk" >}} +{{< tab "Request" >}} +```shell +curl -ks -XPATCH https://temporary-upload-url \ +-H "Authorization: Bearer eyJhbGciOiJQUzI..." \ +-H "Tus-Resumable: 1.0.0" \ +-H "Upload-Offset: 0" \ +-H "Content-Type: application/offset+octet-stream" -d "01234" +``` +{{< /tab >}} + +{{< tab "Response - 204 No Content" >}} +``` +< HTTP/1.1 204 No Content +< Date: Tue, 17 Oct 2023 04:10:52 GMT +< Oc-Fileid: 8d72036d-14a5-490f-889e-414064156402$73bb5450-816b-4cae-90aa-1f96adc95bd4!84e319e4-de1d-4dd8-bbd0-e51d933cdbcd +< Tus-Resumable: 1.0.0 +< Upload-Expires: 1697602157 +< Upload-Offset: 5 +< Vary: Origin +< X-Content-Type-Options: nosniff +< X-Request-Id: xxxxxxxxxxxxxxxxxxxxxx +< +* Connection #0 to host localhost left intact +``` +{{< /tab >}} +{{< /tabs >}} + +### Upload Further Chunks +After the first chunk is uploaded, the second chunk can be uploaded by pointing `Upload-Offset` to exact position that was returned in the first response. +Upload process will not be marked as complete until the total uploaded content size matches the `Upload-Length` specified during the creation of the temporary URL. + +{{< tabs "upload-the-second-chunk" >}} +{{< tab "Request" >}} +```shell +curl -ks -XPATCH https://temporary-upload-url \ +-H "Authorization: Bearer eyJhbGciOiJQUzI..." \ +-H "Tus-Resumable: 1.0.0" \ +-H "Upload-Offset: 5" \ +-H "Content-Type: application/offset+octet-stream" -d "56789" +``` +{{< /tab >}} + +{{< tab "Response - 204 No Content" >}} +``` +< HTTP/1.1 204 No Content +< Date: Tue, 17 Oct 2023 04:11:00 GMT +< Oc-Fileid: 8d72036d-14a5-490f-889e-414064156402$73bb5450-816b-4cae-90aa-1f96adc95bd4!84e319e4-de1d-4dd8-bbd0-e51d933cdbcd +< Tus-Resumable: 1.0.0 +< Upload-Expires: 1697602157 +< Upload-Offset: 10 +< Vary: Origin +< X-Content-Type-Options: nosniff +< X-Request-Id: xxxxxxxxxxxxxxxxxxxxxx +< +* Connection #0 to host localhost left intact +``` +{{< /tab >}} +{{< /tabs >}} +{{< hint type=warning title="Important Warning" >}} +`Upload-Offset` header indicates the byte position in the target file where the server should start writing the upload content. +It ensures data integrity and order during the upload process. +{{< /hint >}} + +## Creation with Upload +{{< tabs "creation-with-upload" >}} +{{< tab "Request" >}} +```shell +curl -ks -XPOST https://ocis.test/remote.php/dav/spaces/{space-id} \ +-H "Authorization: Bearer eyJhbGciOiJQUzI..." \ +-H "Tus-Resumable: 1.0.0" \ +-H "Upload-Length: 14" \ +-H "Content-Type: application/offset+octet-stream" \ +-H "Upload-Metadata: filename dGVzdC50eHQ=" \ +-H "Tus-Extension: creation-with-upload" \ +-d "upload content" +``` +{{< /tab >}} + +{{< tab "Response - 201 Created" >}} +```shell +< HTTP/1.1 201 Created +< Access-Control-Allow-Headers: Tus-Resumable, Upload-Length, Upload-Metadata, If-Match +< Access-Control-Allow-Origin: * +< Access-Control-Expose-Headers: Tus-Resumable, Upload-Offset, Location +< Content-Length: 0 +< Content-Security-Policy: default-src 'none'; +< Content-Type: text/plain +< Date: Mon, 16 Oct 2023 04:18:25 GMT +< Etag: "372c96743f68bc40e789124d30567371" +< Last-Modified: Mon, 16 Oct 2023 04:18:25 +0000 +< Location: https://ocis.test/data/eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZXZhIiwiZXhwIjoxNjk3NTE2MzA1LCJpYXQiOjE2OTc0Mjk5MDUsInRhcmdldCI6Imh0dHA6Ly9sb2NhbGhvc3Q6OTE1OC9kYXRhL3R1cy82NjlhODBlZi1hN2VjLTQwYTAtOGNmOS05MTgwNTVhYzlkZjAifQ.yq-ofJYnJ9FLML7Z_jki1FJQ7Ulbt9O_cmLe6V411A4 +< Oc-Etag: "372c96743f68bc40e789124d30567371" +< Oc-Fileid: 44d3e1e0-6c01-4b94-9145-9d0068239fcd$446bdad4-4b27-41f1-afce-0881f202a214!d7c292a6-c395-4e92-bf07-2c1663aec8dd +< Oc-Perm: RDNVWZP +< Tus-Extension: creation,creation-with-upload,checksum,expiration +< Tus-Resumable: 1.0.0 +< Upload-Expires: 1697516305 +< Upload-Offset: 14 +< Vary: Origin +< X-Content-Type-Options: nosniff +< X-Download-Options: noopen +< X-Frame-Options: SAMEORIGIN +* TLSv1.2 (IN), TLS header, Supplemental data (23): +{ [5 bytes data] +< X-Permitted-Cross-Domain-Policies: none +< X-Request-Id: xxxxxxxxxxxxxxxxxxxxxx +< X-Robots-Tag: none +< X-Xss-Protection: 1; mode=block +< +* Connection #0 to host localhost left intact +``` +{{< /tab >}} +{{< /tabs >}} + +{{< hint type=warning title="Important Warning" >}} +`Upload-Length` header should contain the exact byte as upload content. +{{< /hint >}} diff --git a/docs/helpers/extended_vars.yaml b/docs/helpers/extended_vars.yaml index 3cdb41777aa..d30d070b6b2 100644 --- a/docs/helpers/extended_vars.yaml +++ b/docs/helpers/extended_vars.yaml @@ -39,8 +39,8 @@ variables: type: string default_value: "" description: 'Go micro registry type to use. Supported types are: ''nats'', ''kubernetes'', - ''etcd'', ''consul'', ''memory'' and ''mdns''. Will be selected automatically. - Only change under supervision of ownCloud Support.' + ''etcd'', ''consul'', ''memory'' and ''mdns''. Will be selected automatically. Only change + on supervision of ownCloud Support.' do_ignore: false - rawname: registryAddressEnv path: ocis-pkg/registry/registry.go:42 From ec982c5d916b390d9a10cb613d0f2ae18a3a7101 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Mon, 6 Nov 2023 14:04:46 +0545 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Martin --- docs/apis/http/tus_upload.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/apis/http/tus_upload.md b/docs/apis/http/tus_upload.md index 96227c3b27f..f367f204f80 100644 --- a/docs/apis/http/tus_upload.md +++ b/docs/apis/http/tus_upload.md @@ -1,5 +1,5 @@ --- -title: "tus upload" +title: "tus Upload" date: 2023-10-10T00:00:00+00:00 weight: 21 geekdocRepo: https://github.com/owncloud/ocis @@ -8,14 +8,16 @@ geekdocFilePath: tus_upload.md geekdocCollapseSection: true --- -Infinite Scale supports tus resumable-upload protocol. +Infinite Scale supports the tus resumable-upload protocol. tus is a robust, modular, and open protocol designed to resume large file uploads reliably over HTTP. In situations where file uploads might be interrupted due to network issues, browser crashes, or other unforeseen interruptions, tus ensures that uploads can be resumed from the point of failure without losing data. This documentation shows some basic examples, refer [tus official site](https://tus.io/protocols/resumable-upload) for more details. ## Upload in Chunks + ### Create an Upload URL + The client must send a POST request against a known upload creation URL to request a new upload resource. The filename has to be provided in base64-encoded format. @@ -68,6 +70,7 @@ The server will return a temporary upload URL in the location header of the resp ``` ### Upload the First Chunk + Once a temporary upload URL has been created, a client can send a PATCH request to upload a file. The file content should be sent in the body of the request: {{< tabs "upload-the-first-chunk" >}} {{< tab "Request" >}} @@ -98,6 +101,7 @@ curl -ks -XPATCH https://temporary-upload-url \ {{< /tabs >}} ### Upload Further Chunks + After the first chunk is uploaded, the second chunk can be uploaded by pointing `Upload-Offset` to exact position that was returned in the first response. Upload process will not be marked as complete until the total uploaded content size matches the `Upload-Length` specified during the creation of the temporary URL. @@ -134,6 +138,7 @@ It ensures data integrity and order during the upload process. {{< /hint >}} ## Creation with Upload + {{< tabs "creation-with-upload" >}} {{< tab "Request" >}} ```shell From 3093499f20a9bb4857cb89dd910c71dfde90d867 Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 7 Nov 2023 14:50:27 +0100 Subject: [PATCH 3/3] add reference in frontend ennvar description --- services/frontend/pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/frontend/pkg/config/config.go b/services/frontend/pkg/config/config.go index 663d87c4768..f65080f5fb2 100644 --- a/services/frontend/pkg/config/config.go +++ b/services/frontend/pkg/config/config.go @@ -32,7 +32,7 @@ type Config struct { MaxQuota uint64 `yaml:"max_quota" env:"OCIS_SPACES_MAX_QUOTA;FRONTEND_MAX_QUOTA" desc:"Set the global max quota value in bytes. A value of 0 equals unlimited. The value is provided via capabilities."` UploadMaxChunkSize int `yaml:"upload_max_chunk_size" env:"FRONTEND_UPLOAD_MAX_CHUNK_SIZE" desc:"Sets the max chunk sizes in bytes for uploads via the clients."` UploadHTTPMethodOverride string `yaml:"upload_http_method_override" env:"FRONTEND_UPLOAD_HTTP_METHOD_OVERRIDE" desc:"Advise TUS to replace PATCH requests by POST requests."` - DefaultUploadProtocol string `yaml:"default_upload_protocol" env:"FRONTEND_DEFAULT_UPLOAD_PROTOCOL" desc:"The default upload protocol to use in the clients (e.g. tus)."` + DefaultUploadProtocol string `yaml:"default_upload_protocol" env:"FRONTEND_DEFAULT_UPLOAD_PROTOCOL" desc:"The default upload protocol to use in clients. Currently only 'tus' is avaliable. See the developer API documentation for more details about TUS."` EnableResharing bool `yaml:"enable_resharing" env:"OCIS_ENABLE_RESHARING;FRONTEND_ENABLE_RESHARING" desc:"Changing this value is NOT supported. Enables the support for resharing in the clients."` EnableFederatedSharingIncoming bool `yaml:"enable_federated_sharing_incoming" env:"FRONTEND_ENABLE_FEDERATED_SHARING_INCOMING" desc:"Changing this value is NOT supported. Enables support for incoming federated sharing for clients. The backend behaviour is not changed."` EnableFederatedSharingOutgoing bool `yaml:"enable_federated_sharing_outgoing" env:"FRONTEND_ENABLE_FEDERATED_SHARING_OUTGOING" desc:"Changing this value is NOT supported. Enables support for outgoing federated sharing for clients. The backend behaviour is not changed."`