From db175989a340767f44ac20d6ef246133a0fb4748 Mon Sep 17 00:00:00 2001 From: Richard Hua Date: Wed, 24 Jul 2024 10:09:30 -0700 Subject: [PATCH 1/5] feat: initial protos for decentralization --- proto/mls/api/v1/mls.proto | 2 +- proto/xmtpv4/message_api/message_api.proto | 116 +++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 proto/xmtpv4/message_api/message_api.proto diff --git a/proto/mls/api/v1/mls.proto b/proto/mls/api/v1/mls.proto index 32b9642..42865cb 100644 --- a/proto/mls/api/v1/mls.proto +++ b/proto/mls/api/v1/mls.proto @@ -161,7 +161,7 @@ message GroupMessage { message GroupMessageInput { // Version 1 of the GroupMessageInput payload format message V1 { - bytes data = 1; + bytes data = 1; // Serialized MlsProtocolMessage bytes sender_hmac = 2; } diff --git a/proto/xmtpv4/message_api/message_api.proto b/proto/xmtpv4/message_api/message_api.proto new file mode 100644 index 0000000..3277e9d --- /dev/null +++ b/proto/xmtpv4/message_api/message_api.proto @@ -0,0 +1,116 @@ +message AuthenticatedData { + repeated uint64 last_originator_sids = 1; +} + +// From MLS spec +message MlsPrivateMessage { + bytes group_id; + uint64 epoch; + // One of Application, Proposal, Commit + ContentType content_type; + bytes authenticated_data; + bytes encrypted_sender_data; + bytes ciphertext; +} + +// Replaces GroupMessageInput V1 +// To rename or not to rename? +message ClientEnvelope { + // TLS serialized MlsMessageIn, which contains MlsPrivateMessage + bytes data = 1; + bytes sender_hmac = 2; +} + +message PayerEnvelope { + bytes unsigned_client_envelope = 1; // Protobuf serialized + RecoverableEcdsaSignature payer_signature = 2; +} + +// For blockchain envelopes, the originator_sid is set by the smart contract, +// but the originator_ns is set by the publishing node +message UnsignedOriginatorEnvelope { + uint64 originator_sid = 1; + uint64 originator_ns = 2; + PayerEnvelope payer_envelope = 3; +} + +message BlockchainProof { + uint64 block_number = 1; + uint32 publisher_id = 2; +} + +message OriginatorEnvelope { + bytes unsigned_originator_envelope = 1; // Protobuf serialized + oneof proof { + RecoverableEcdsaSignature originator_signature = 2; + BlockchainProof blockchain_proof = 3; + } +} + +message GatewayEnvelope { + uint64 gateway_sid = 1; + OriginatorEnvelope originator_envelope = 2; +} + + + + +message EnvelopesQuery { + optional oneof last_seen { + uint64 originator_sid = 1; + uint64 gateway_sid = 2; + } + optional oneof filter { + bytes topic = 3; + uint32 originator_id = 4; + } +} + +message BatchSubscribeEnvelopesRequest { + message SubscribeEnvelopesRequest { + EnvelopesQuery query = 1; + } + repeated SubscribeEnvelopesRequest requests = 1; +} + +rpc SubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream GatewayEnvelope) { + option (google.api.http) = { + post: "/mls/v2/subscribe-envelopes" + body: "*" + }; +} + +// Pagination config for queries +message PagingInfo { + SortDirection direction = 1; + uint32 limit = 2; +} + +message QueryEnvelopesRequest { + EnvelopesQuery query = 1; + PagingInfo paging_info = 2; +} + +message QueryEnvelopesResponse { + repeated GatewayEnvelope envelopes = 1; +} + +rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { + option (google.api.http) = { + post: "/mls/v2/query-envelopes" + body: "*" + }; +} + +enum Misbehavior { + MISBEHAVIOR_UNSPECIFIED = 0; + MISBEHAVIOR_UNAVAILABLE_NODE = 1; + MISBEHAVIOR_OUT_OF_ORDER_ORIGINATOR_SID = 2; + MISBEHAVIOR_DUPLICATE_ORIGINATOR_SID = 3; + MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING; +} + +message MisbehaviorReport { + MisbehaviorType type; + repeated OriginatorEnvelope envelopes; +} From 03569966ed870f08912c458af25c87c634cf1c72 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Wed, 24 Jul 2024 10:28:33 -0700 Subject: [PATCH 2/5] feat: fix syntax errors --- proto/xmtpv4/message_api/message_api.proto | 127 ++++++++++----------- 1 file changed, 62 insertions(+), 65 deletions(-) diff --git a/proto/xmtpv4/message_api/message_api.proto b/proto/xmtpv4/message_api/message_api.proto index 3277e9d..6cc9239 100644 --- a/proto/xmtpv4/message_api/message_api.proto +++ b/proto/xmtpv4/message_api/message_api.proto @@ -1,89 +1,93 @@ -message AuthenticatedData { - repeated uint64 last_originator_sids = 1; -} +syntax = "proto3"; + +package xmtp.xmtpv4; + +import "google/api/annotations.proto"; +import "identity/associations/signature.proto"; -// From MLS spec -message MlsPrivateMessage { - bytes group_id; - uint64 epoch; - // One of Application, Proposal, Commit - ContentType content_type; - bytes authenticated_data; - bytes encrypted_sender_data; - bytes ciphertext; +option go_package = "github.com/xmtp/proto/v3/go/xmtpv4/message_api"; + +message AuthenticatedData { + repeated uint64 last_originator_sids = 1; } // Replaces GroupMessageInput V1 // To rename or not to rename? message ClientEnvelope { - // TLS serialized MlsMessageIn, which contains MlsPrivateMessage - bytes data = 1; - bytes sender_hmac = 2; + // TLS serialized MlsMessageIn, which contains MlsPrivateMessage + bytes data = 1; + bytes sender_hmac = 2; } message PayerEnvelope { - bytes unsigned_client_envelope = 1; // Protobuf serialized - RecoverableEcdsaSignature payer_signature = 2; + bytes unsigned_client_envelope = 1; // Protobuf serialized + xmtp.identity.associations.RecoverableEcdsaSignature payer_signature = 2; } // For blockchain envelopes, the originator_sid is set by the smart contract, // but the originator_ns is set by the publishing node message UnsignedOriginatorEnvelope { - uint64 originator_sid = 1; - uint64 originator_ns = 2; - PayerEnvelope payer_envelope = 3; + uint64 originator_sid = 1; + uint64 originator_ns = 2; + PayerEnvelope payer_envelope = 3; } message BlockchainProof { - uint64 block_number = 1; - uint32 publisher_id = 2; + uint64 block_number = 1; + uint32 publisher_id = 2; } message OriginatorEnvelope { - bytes unsigned_originator_envelope = 1; // Protobuf serialized - oneof proof { - RecoverableEcdsaSignature originator_signature = 2; - BlockchainProof blockchain_proof = 3; - } + bytes unsigned_originator_envelope = 1; // Protobuf serialized + oneof proof { + xmtp.identity.associations.RecoverableEcdsaSignature originator_signature = 2; + BlockchainProof blockchain_proof = 3; + } } message GatewayEnvelope { - uint64 gateway_sid = 1; - OriginatorEnvelope originator_envelope = 2; + uint64 gateway_sid = 1; + OriginatorEnvelope originator_envelope = 2; } - - - message EnvelopesQuery { - optional oneof last_seen { - uint64 originator_sid = 1; - uint64 gateway_sid = 2; - } - optional oneof filter { - bytes topic = 3; - uint32 originator_id = 4; - } + oneof last_seen { + uint64 originator_sid = 1; + uint64 gateway_sid = 2; + } + + oneof filter { + bytes topic = 3; + uint32 originator_id = 4; + } } message BatchSubscribeEnvelopesRequest { - message SubscribeEnvelopesRequest { - EnvelopesQuery query = 1; - } - repeated SubscribeEnvelopesRequest requests = 1; + message SubscribeEnvelopesRequest { + EnvelopesQuery query = 1; + } + repeated SubscribeEnvelopesRequest requests = 1; } -rpc SubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream GatewayEnvelope) { - option (google.api.http) = { - post: "/mls/v2/subscribe-envelopes" - body: "*" - }; +service ReplicationApi { + rpc SubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream GatewayEnvelope) { + option (google.api.http) = { + post: "/mls/v2/subscribe-envelopes" + body: "*" + }; + } + + rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { + option (google.api.http) = { + post: "/mls/v2/query-envelopes" + body: "*" + }; + } } // Pagination config for queries message PagingInfo { - SortDirection direction = 1; - uint32 limit = 2; + uint32 limit = 1; } message QueryEnvelopesRequest { @@ -95,22 +99,15 @@ message QueryEnvelopesResponse { repeated GatewayEnvelope envelopes = 1; } -rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { - option (google.api.http) = { - post: "/mls/v2/query-envelopes" - body: "*" - }; -} - enum Misbehavior { - MISBEHAVIOR_UNSPECIFIED = 0; - MISBEHAVIOR_UNAVAILABLE_NODE = 1; - MISBEHAVIOR_OUT_OF_ORDER_ORIGINATOR_SID = 2; - MISBEHAVIOR_DUPLICATE_ORIGINATOR_SID = 3; - MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING; + MISBEHAVIOR_UNSPECIFIED = 0; + MISBEHAVIOR_UNAVAILABLE_NODE = 1; + MISBEHAVIOR_OUT_OF_ORDER_ORIGINATOR_SID = 2; + MISBEHAVIOR_DUPLICATE_ORIGINATOR_SID = 3; + MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING = 4; } message MisbehaviorReport { - MisbehaviorType type; - repeated OriginatorEnvelope envelopes; + Misbehavior type = 1; + repeated OriginatorEnvelope envelopes = 2; } From c2ed711169ff01a2dd86943a2d9fde4d73a1686d Mon Sep 17 00:00:00 2001 From: Richard Hua Date: Tue, 30 Jul 2024 14:11:22 -0700 Subject: [PATCH 3/5] chore: fix lints --- .vscode/settings.json | 6 +++++- proto/xmtpv4/message_api/message_api.proto | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7911ace..727ae2c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,9 @@ "editor.formatOnSave": true }, // protolint complains if lines are longer than length 80, display a ruler - "editor.rulers": [80] + "editor.rulers": [ + 80 + ], +"editor.detectIndentation": false, +"editor.tabSize": 2 } diff --git a/proto/xmtpv4/message_api/message_api.proto b/proto/xmtpv4/message_api/message_api.proto index 6cc9239..7c6db35 100644 --- a/proto/xmtpv4/message_api/message_api.proto +++ b/proto/xmtpv4/message_api/message_api.proto @@ -1,3 +1,4 @@ +// Message API for XMTP V4 syntax = "proto3"; package xmtp.xmtpv4; @@ -7,6 +8,7 @@ import "identity/associations/signature.proto"; option go_package = "github.com/xmtp/proto/v3/go/xmtpv4/message_api"; +// Authenticated data within the MlsPrivateMessage message AuthenticatedData { repeated uint64 last_originator_sids = 1; } @@ -19,6 +21,7 @@ message ClientEnvelope { bytes sender_hmac = 2; } +// Wraps client envelope with payer signature message PayerEnvelope { bytes unsigned_client_envelope = 1; // Protobuf serialized xmtp.identity.associations.RecoverableEcdsaSignature payer_signature = 2; @@ -32,11 +35,13 @@ message UnsignedOriginatorEnvelope { PayerEnvelope payer_envelope = 3; } +// An alternative to a signature for blockchain payloads message BlockchainProof { uint64 block_number = 1; uint32 publisher_id = 2; } +// Signed originator envelope message OriginatorEnvelope { bytes unsigned_originator_envelope = 1; // Protobuf serialized oneof proof { @@ -45,11 +50,13 @@ message OriginatorEnvelope { } } +// Wraps originator envelope with a sequence ID assigned by the gateway message GatewayEnvelope { uint64 gateway_sid = 1; OriginatorEnvelope originator_envelope = 2; } +// Query for envelopes, shared by query and subscribe endpoints message EnvelopesQuery { oneof last_seen { uint64 originator_sid = 1; @@ -62,21 +69,30 @@ message EnvelopesQuery { } } +// Batch subscribe to envelopes message BatchSubscribeEnvelopesRequest { + // Single subscription request for envelopes message SubscribeEnvelopesRequest { EnvelopesQuery query = 1; } repeated SubscribeEnvelopesRequest requests = 1; } +message BatchSubscribeEnvelopesResponse { + repeated GatewayEnvelope envelopes = 1; +} + +// Replication API service ReplicationApi { - rpc SubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream GatewayEnvelope) { + // Subscribe to envelopes + rpc BatchSubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream BatchSubscribeEnvelopesResponse) { option (google.api.http) = { post: "/mls/v2/subscribe-envelopes" body: "*" }; } + // Query envelopes rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { option (google.api.http) = { post: "/mls/v2/query-envelopes" @@ -90,15 +106,18 @@ message PagingInfo { uint32 limit = 1; } +// Query envelopes request message QueryEnvelopesRequest { EnvelopesQuery query = 1; PagingInfo paging_info = 2; } +// Query envelopes response message QueryEnvelopesResponse { repeated GatewayEnvelope envelopes = 1; } +// Misbehavior types enum Misbehavior { MISBEHAVIOR_UNSPECIFIED = 0; MISBEHAVIOR_UNAVAILABLE_NODE = 1; @@ -107,6 +126,7 @@ enum Misbehavior { MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING = 4; } +// Reports node misbehavior, submittable by nodes or by clients message MisbehaviorReport { Misbehavior type = 1; repeated OriginatorEnvelope envelopes = 2; From 66cec7a9fa6920aa976194e77b2ea9c0d72d4a83 Mon Sep 17 00:00:00 2001 From: Richard Hua Date: Tue, 30 Jul 2024 14:18:36 -0700 Subject: [PATCH 4/5] chore: reorder types logically --- proto/xmtpv4/message_api/message_api.proto | 63 +++++++++++----------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/proto/xmtpv4/message_api/message_api.proto b/proto/xmtpv4/message_api/message_api.proto index 7c6db35..11e8bb0 100644 --- a/proto/xmtpv4/message_api/message_api.proto +++ b/proto/xmtpv4/message_api/message_api.proto @@ -56,6 +56,21 @@ message GatewayEnvelope { OriginatorEnvelope originator_envelope = 2; } +// Misbehavior types +enum Misbehavior { + MISBEHAVIOR_UNSPECIFIED = 0; + MISBEHAVIOR_UNAVAILABLE_NODE = 1; + MISBEHAVIOR_OUT_OF_ORDER_ORIGINATOR_SID = 2; + MISBEHAVIOR_DUPLICATE_ORIGINATOR_SID = 3; + MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING = 4; +} + +// Reports node misbehavior, submittable by nodes or by clients +message MisbehaviorReport { + Misbehavior type = 1; + repeated OriginatorEnvelope envelopes = 2; +} + // Query for envelopes, shared by query and subscribe endpoints message EnvelopesQuery { oneof last_seen { @@ -78,29 +93,11 @@ message BatchSubscribeEnvelopesRequest { repeated SubscribeEnvelopesRequest requests = 1; } +// Streamed response for batch subscribe - can be multiple envelopes at once message BatchSubscribeEnvelopesResponse { repeated GatewayEnvelope envelopes = 1; } -// Replication API -service ReplicationApi { - // Subscribe to envelopes - rpc BatchSubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream BatchSubscribeEnvelopesResponse) { - option (google.api.http) = { - post: "/mls/v2/subscribe-envelopes" - body: "*" - }; - } - - // Query envelopes - rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { - option (google.api.http) = { - post: "/mls/v2/query-envelopes" - body: "*" - }; - } -} - // Pagination config for queries message PagingInfo { uint32 limit = 1; @@ -117,17 +114,21 @@ message QueryEnvelopesResponse { repeated GatewayEnvelope envelopes = 1; } -// Misbehavior types -enum Misbehavior { - MISBEHAVIOR_UNSPECIFIED = 0; - MISBEHAVIOR_UNAVAILABLE_NODE = 1; - MISBEHAVIOR_OUT_OF_ORDER_ORIGINATOR_SID = 2; - MISBEHAVIOR_DUPLICATE_ORIGINATOR_SID = 3; - MISBEHAVIOR_CYCLICAL_MESSAGE_ORDERING = 4; -} +// Replication API +service ReplicationApi { + // Subscribe to envelopes + rpc BatchSubscribeEnvelopes(BatchSubscribeEnvelopesRequest) returns (stream BatchSubscribeEnvelopesResponse) { + option (google.api.http) = { + post: "/mls/v2/subscribe-envelopes" + body: "*" + }; + } -// Reports node misbehavior, submittable by nodes or by clients -message MisbehaviorReport { - Misbehavior type = 1; - repeated OriginatorEnvelope envelopes = 2; + // Query envelopes + rpc QueryEnvelopes(QueryEnvelopesRequest) returns (QueryEnvelopesResponse) { + option (google.api.http) = { + post: "/mls/v2/query-envelopes" + body: "*" + }; + } } From d4eb6d72821b06a13b3127971504479209e5c3da Mon Sep 17 00:00:00 2001 From: Richard Hua Date: Tue, 30 Jul 2024 14:19:37 -0700 Subject: [PATCH 5/5] fix: remove paging_info --- proto/xmtpv4/message_api/message_api.proto | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/proto/xmtpv4/message_api/message_api.proto b/proto/xmtpv4/message_api/message_api.proto index 11e8bb0..be6499f 100644 --- a/proto/xmtpv4/message_api/message_api.proto +++ b/proto/xmtpv4/message_api/message_api.proto @@ -98,15 +98,10 @@ message BatchSubscribeEnvelopesResponse { repeated GatewayEnvelope envelopes = 1; } -// Pagination config for queries -message PagingInfo { - uint32 limit = 1; -} - // Query envelopes request message QueryEnvelopesRequest { EnvelopesQuery query = 1; - PagingInfo paging_info = 2; + uint32 limit = 2; } // Query envelopes response