From fa2080f1b4d66831e0edc902635e1e91bfdf4757 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Tue, 23 Jan 2024 21:01:48 +0300 Subject: [PATCH 1/4] link: Create link object payload message It describes future protocol version's link object payload. Child objects list will be moved from the header to the payload. This is done due to the header size restrictions. Closes #263. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + link/types.proto | 20 ++++++++++++++ object/types.proto | 2 ++ proto-docs/link.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ proto-docs/object.md | 2 +- 5 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 link/types.proto create mode 100644 proto-docs/link.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ec0e2..d1999bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - Numeric operands for object search queries (#265) +- Link object payload message (#263) ### Changed diff --git a/link/types.proto b/link/types.proto new file mode 100644 index 0000000..2bbd5e2 --- /dev/null +++ b/link/types.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package neo.fs.v2.link; + +option go_package = "github.com/nspcc-dev/neofs-api-go/v2/link/grpc;link"; +option csharp_namespace = "Neo.FileStorage.API.Link"; + +import "refs/types.proto"; + +// Link is a payload of helper objects that contain the full list of the split +// chain objects' IDs. It is created only after the whole split chain is known +// and signed. This object is the only object that refers to every "child object" +// ID. It is NOT required for the original object assembling. It MUST have ALL +// the "child objects" IDs. IDs MUST be ordered according to the original payload +// split, meaning the first payload part holder MUST be placed at the first place +// in the corresponding link object. +message Link { + // Full list of the "child" object IDs. + repeated neo.fs.v2.refs.ObjectID children = 1 [json_name = "children"]; +} diff --git a/object/types.proto b/object/types.proto index 7a85fcb..c94ebda 100644 --- a/object/types.proto +++ b/object/types.proto @@ -185,6 +185,8 @@ message Header { // `header` field of the parent object. Used to reconstruct parent. Header parent_header = 4 [json_name = "parentHeader"]; + // DEPRECATED. Was used before creating the separate LINK object type. Keep + // child objects list in the LINK object's payload. // List of identifiers of the objects generated by splitting current one. repeated neo.fs.v2.refs.ObjectID children = 5 [json_name = "children"]; diff --git a/proto-docs/link.md b/proto-docs/link.md new file mode 100644 index 0000000..a1596a0 --- /dev/null +++ b/proto-docs/link.md @@ -0,0 +1,66 @@ +# Protocol Documentation + + +## Table of Contents + +- [link/types.proto](#link/types.proto) + + - Messages + - [Link](#neo.fs.v2.link.Link) + + +- [Scalar Value Types](#scalar-value-types) + + + + +

Top

+ +## link/types.proto + + + + + + + +### Message Link +Link is a payload of helper objects that contain the full list of the split +chain objects' IDs. It is created only after the whole split chain is known +and signed. This object is the only object that refers to every "child object" +ID. It is NOT required for the original object assembling. It MUST have ALL +the "child objects" IDs. IDs MUST be ordered according to the original payload +split, meaning the first payload part holder MUST be placed at the first place +in the corresponding link object. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| children | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | repeated | Full list of the "child" object IDs. | + + + + + + + +## Scalar Value Types + +| .proto Type | Notes | C++ Type | Java Type | Python Type | +| ----------- | ----- | -------- | --------- | ----------- | +| double | | double | double | float | +| float | | float | float | float | +| int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int | +| int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long | +| uint32 | Uses variable-length encoding. | uint32 | int | int/long | +| uint64 | Uses variable-length encoding. | uint64 | long | int/long | +| sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int | +| sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long | +| fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int | +| fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long | +| sfixed32 | Always four bytes. | int32 | int | int | +| sfixed64 | Always eight bytes. | int64 | long | int/long | +| bool | | bool | boolean | boolean | +| string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode | +| bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str | + diff --git a/proto-docs/object.md b/proto-docs/object.md index 0f1adc6..9f5f5f5 100644 --- a/proto-docs/object.md +++ b/proto-docs/object.md @@ -954,7 +954,7 @@ must be within the same container. | previous | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | Identifier of the left split neighbor | | parent_signature | [neo.fs.v2.refs.Signature](#neo.fs.v2.refs.Signature) | | `signature` field of the parent object. Used to reconstruct parent. | | parent_header | [Header](#neo.fs.v2.object.Header) | | `header` field of the parent object. Used to reconstruct parent. | -| children | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | repeated | List of identifiers of the objects generated by splitting current one. | +| children | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | repeated | DEPRECATED. Was used before creating the separate LINK object type. Keep child objects list in the LINK object's payload. List of identifiers of the objects generated by splitting current one. | | split_id | [bytes](#bytes) | | 16 byte UUIDv4 used to identify the split object hierarchy parts. Must be unique inside container. All objects participating in the split must have the same `split_id` value. | From 39d130d907beaaa772186855c80516b60041c13b Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 24 Jan 2024 17:58:03 +0300 Subject: [PATCH 2/4] link: Add children sizes information It allows faster seeking through a split object without fetching the whole chain. Closes #264. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + link/types.proto | 20 +++++++++++++++----- proto-docs/link.md | 22 ++++++++++++++++++---- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1999bc..5269f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Numeric operands for object search queries (#265) - Link object payload message (#263) +- Children sizes index to the link objects (#264) ### Changed diff --git a/link/types.proto b/link/types.proto index 2bbd5e2..b7d78b5 100644 --- a/link/types.proto +++ b/link/types.proto @@ -11,10 +11,20 @@ import "refs/types.proto"; // chain objects' IDs. It is created only after the whole split chain is known // and signed. This object is the only object that refers to every "child object" // ID. It is NOT required for the original object assembling. It MUST have ALL -// the "child objects" IDs. IDs MUST be ordered according to the original payload -// split, meaning the first payload part holder MUST be placed at the first place -// in the corresponding link object. +// the "child objects" IDs. Child objects MUST be ordered according to the +// original payload split, meaning the first payload part holder MUST be placed +// at the first place in the corresponding link object. Sizes MUST NOT be omitted +// and MUST be a real object payload size in bytes. message Link { - // Full list of the "child" object IDs. - repeated neo.fs.v2.refs.ObjectID children = 1 [json_name = "children"]; + // Object ID with its object's payload size. + message MeasuredObject { + // Object ID. + neo.fs.v2.refs.ObjectID id = 1 [json_name = "id"]; + + // Object size in bytes. + uint32 size = 2 [json_name = "size"]; + } + + // Full list of the "child" object descriptors. + repeated MeasuredObject children = 1 [json_name = "children"]; } diff --git a/proto-docs/link.md b/proto-docs/link.md index a1596a0..a71626b 100644 --- a/proto-docs/link.md +++ b/proto-docs/link.md @@ -7,6 +7,7 @@ - Messages - [Link](#neo.fs.v2.link.Link) + - [Link.MeasuredObject](#neo.fs.v2.link.Link.MeasuredObject) - [Scalar Value Types](#scalar-value-types) @@ -29,14 +30,27 @@ Link is a payload of helper objects that contain the full list of the split chain objects' IDs. It is created only after the whole split chain is known and signed. This object is the only object that refers to every "child object" ID. It is NOT required for the original object assembling. It MUST have ALL -the "child objects" IDs. IDs MUST be ordered according to the original payload -split, meaning the first payload part holder MUST be placed at the first place -in the corresponding link object. +the "child objects" IDs. Child objects MUST be ordered according to the +original payload split, meaning the first payload part holder MUST be placed +at the first place in the corresponding link object. Sizes MUST NOT be omitted +and MUST be a real object payload size in bytes. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| children | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | repeated | Full list of the "child" object IDs. | +| children | [Link.MeasuredObject](#neo.fs.v2.link.Link.MeasuredObject) | repeated | Full list of the "child" object descriptors. | + + + + +### Message Link.MeasuredObject +Object ID with its object's payload size. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| id | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | Object ID. | +| size | [uint32](#uint32) | | Object size in bytes. | From f816d8add8a02de94cd3c6694a64a3b6db2c1002 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Tue, 23 Jan 2024 20:42:59 +0300 Subject: [PATCH 3/4] object: Add `LINK` object type This commit makes it easier to differ link objects from the other types. Object split hierarchy rework increases the link object's structure and makes it more strictly formatted, so now it plays a more important role in the split chains (and the split rules became more complex too). Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + object/types.proto | 4 ++++ proto-docs/object.md | 2 ++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5269f2f..ca6c76c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Numeric operands for object search queries (#265) - Link object payload message (#263) - Children sizes index to the link objects (#264) +- `LINK` split chain object type (#283) ### Changed diff --git a/object/types.proto b/object/types.proto index c94ebda..f3dca1c 100644 --- a/object/types.proto +++ b/object/types.proto @@ -17,6 +17,7 @@ import "session/types.proto"; // * TOMBSTONE // * STORAGE_GROUP // * LOCK +// * LINK enum ObjectType { // Just a normal object REGULAR = 0; @@ -29,6 +30,9 @@ enum ObjectType { // Object lock LOCK = 3; + + // Object that stores child object IDs for the split objects. + LINK = 4; } // Type of match expression diff --git a/proto-docs/object.md b/proto-docs/object.md index 9f5f5f5..9c475af 100644 --- a/proto-docs/object.md +++ b/proto-docs/object.md @@ -1043,6 +1043,7 @@ String presentation of object type is the same as definition: * TOMBSTONE * STORAGE_GROUP * LOCK +* LINK | Name | Number | Description | | ---- | ------ | ----------- | @@ -1050,6 +1051,7 @@ String presentation of object type is the same as definition: | TOMBSTONE | 1 | Used internally to identify deleted objects | | STORAGE_GROUP | 2 | StorageGroup information | | LOCK | 3 | Object lock | +| LINK | 4 | Object that stores child object IDs for the split objects. | From a825a7ecd07725cbb9caa37f2181c7aa8e285fc2 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Mon, 5 Feb 2024 18:22:35 +0300 Subject: [PATCH 4/4] object: Use the first object part as a split ID There is no need to generate some UUID with non-specified rules if the first object part allows the same identification routines but uses hashes widely accepted in the protocol. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + object/types.proto | 11 +++++++++++ proto-docs/object.md | 6 ++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca6c76c..f45bd16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - `LINK` split chain object type (#283) ### Changed +- Split ID is deprecated, the first child part is used instead and is known to the other parts (#283) ### Removed diff --git a/object/types.proto b/object/types.proto index f3dca1c..46d33e7 100644 --- a/object/types.proto +++ b/object/types.proto @@ -194,11 +194,16 @@ message Header { // List of identifiers of the objects generated by splitting current one. repeated neo.fs.v2.refs.ObjectID children = 5 [json_name = "children"]; + // DEPRECATED. Was used as an identifier of a split chain. Use the first + // part ID instead. // 16 byte UUIDv4 used to identify the split object hierarchy parts. Must be // unique inside container. All objects participating in the split must have // the same `split_id` value. bytes split_id = 6 [json_name = "splitID"]; + // Identifier of the first part of the origin object. Known to all the split + // parts except the first one. Identifies the split and allows to differ them. + neo.fs.v2.refs.ObjectID first = 7 [json_name = "first"]; } // Position of the object in the split hierarchy Split split = 11 [json_name = "split"]; @@ -229,6 +234,8 @@ message Object { // assemble the original object. With a linking object one can assemble an object // right from the object parts. message SplitInfo { + // DEPRECATED. Was used as an identifier of a split chain. Use the first + // part ID instead. // 16 byte UUID used to identify the split object hierarchy parts. bytes split_id = 1; @@ -240,4 +247,8 @@ message SplitInfo { // split header with the original object header and a sorted list of // object parts. neo.fs.v2.refs.ObjectID link = 3; + + // Identifier of the first part of the origin object. Known to all the split + // parts except the first one. Identifies the split and allows to differ them. + neo.fs.v2.refs.ObjectID first_part = 4; } diff --git a/proto-docs/object.md b/proto-docs/object.md index 9c475af..9ee7d17 100644 --- a/proto-docs/object.md +++ b/proto-docs/object.md @@ -955,7 +955,8 @@ must be within the same container. | parent_signature | [neo.fs.v2.refs.Signature](#neo.fs.v2.refs.Signature) | | `signature` field of the parent object. Used to reconstruct parent. | | parent_header | [Header](#neo.fs.v2.object.Header) | | `header` field of the parent object. Used to reconstruct parent. | | children | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | repeated | DEPRECATED. Was used before creating the separate LINK object type. Keep child objects list in the LINK object's payload. List of identifiers of the objects generated by splitting current one. | -| split_id | [bytes](#bytes) | | 16 byte UUIDv4 used to identify the split object hierarchy parts. Must be unique inside container. All objects participating in the split must have the same `split_id` value. | +| split_id | [bytes](#bytes) | | DEPRECATED. Was used as an identifier of a split chain. Use the first part ID instead. 16 byte UUIDv4 used to identify the split object hierarchy parts. Must be unique inside container. All objects participating in the split must have the same `split_id` value. | +| first | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | Identifier of the first part of the origin object. Known to all the split parts except the first one. Identifies the split and allows to differ them. | @@ -1005,9 +1006,10 @@ right from the object parts. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| split_id | [bytes](#bytes) | | 16 byte UUID used to identify the split object hierarchy parts. | +| split_id | [bytes](#bytes) | | DEPRECATED. Was used as an identifier of a split chain. Use the first part ID instead. 16 byte UUID used to identify the split object hierarchy parts. | | last_part | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | The identifier of the last object in split hierarchy parts. It contains split header with the original object header. | | link | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | The identifier of a linking object for split hierarchy parts. It contains split header with the original object header and a sorted list of object parts. | +| first_part | [neo.fs.v2.refs.ObjectID](#neo.fs.v2.refs.ObjectID) | | Identifier of the first part of the origin object. Known to all the split parts except the first one. Identifies the split and allows to differ them. |