From 6d92a3ddaa2b92fa153a9558e04c8c93327f19ff Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 30 Jun 2025 09:34:54 +0100 Subject: [PATCH 01/16] Add cluster support (#228) ## Release notes: usage and product changes Introduce cluster support for TypeDB 3.x, featuring server replicas and the new version message. ## Implementation Remove database replicas. Instead, add server replicas (by extending the usual `Server` message) with a similar set of fields. Instead of adding a `is_primary` flag, introduce an extensible enum `ReplicaType` for more granular split between primary and supporting nodes. To reduce the network overhead, this new information is provided in the initial server's connection response. --- proto/BUILD | 1 + proto/connection.proto | 9 ++------- proto/database.proto | 20 ++++---------------- proto/error.proto | 2 +- proto/server.proto | 20 +++++++++++++++++++- proto/typedb-service.proto | 3 +++ 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index 1eaa241b..d6e31acd 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -66,6 +66,7 @@ proto_library( deps = [ ":authentication-proto", ":database-proto", + ":server-proto", ":version-proto" ], ) diff --git a/proto/connection.proto b/proto/connection.proto index d288eb46..b083d7f1 100644 --- a/proto/connection.proto +++ b/proto/connection.proto @@ -6,35 +6,30 @@ syntax = "proto3"; import "proto/authentication.proto"; import "proto/database.proto"; +import "proto/server.proto"; import "proto/version.proto"; package typedb.protocol; message Connection { - message Open { message Req { Version version = 1; ExtensionVersion extension_version = 5; string driver_lang = 2; string driver_version = 3; - Authentication.Token.Create.Req authentication = 4; } message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - - // pre-send all databases and replica info - DatabaseManager.All.Res databases_all = 3; - + ServerManager.All.Res servers_all = 3; Authentication.Token.Create.Res authentication = 4; } } } -// Connection ID and Token are expected in all message metadata message ConnectionID { bytes id = 1; } diff --git a/proto/database.proto b/proto/database.proto index eefa6d4a..46940921 100644 --- a/proto/database.proto +++ b/proto/database.proto @@ -16,14 +16,14 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } message All { message Req {} message Res { - repeated DatabaseReplicas databases = 1; + repeated Database databases = 1; } } @@ -43,7 +43,7 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } @@ -58,20 +58,8 @@ message DatabaseManager { } } -message DatabaseReplicas { - - string name = 1; - repeated Replica replicas = 2; - - message Replica { - string address = 1; - bool primary = 2; - bool preferred = 3; - int64 term = 4; - } -} - message Database { + string name = 1; message Schema { message Req { diff --git a/proto/error.proto b/proto/error.proto index eaa7a557..7543d286 100644 --- a/proto/error.proto +++ b/proto/error.proto @@ -7,7 +7,7 @@ syntax = "proto3"; package typedb.protocol; // This is an emulation of the google ErrorDetails message. Generally, ErrorDetails are submitted via the GRPC error -// mechanism, but a manual error sending is required in streams +// mechanism, but a manual error sending is useful to differentiate error levels in streams message Error { string error_code = 1; string domain = 2; diff --git a/proto/server.proto b/proto/server.proto index 40d2c989..b1c0cc35 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -7,7 +7,6 @@ syntax = "proto3"; package typedb.protocol; message ServerManager { - message All { message Req {} message Res { @@ -18,4 +17,23 @@ message ServerManager { message Server { string address = 1; + optional ReplicaStatus replica_status = 2; + + message ReplicaStatus { + ReplicaType replica_type = 1; + int64 term = 2; + + enum ReplicaType { + Primary = 0; + Secondary = 1; + } + } + + message Version { + message Req {} + message Res { + string distribution = 1; + string version = 2; + } + } } diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 95f0c239..79e69365 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -25,6 +25,9 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + // Server API + rpc server_version (Server.Version.Req) returns (Server.Version.Res); + // User Manager API rpc users_get (UserManager.Get.Req) returns (UserManager.Get.Res); rpc users_all (UserManager.All.Req) returns (UserManager.All.Res); From 6e473ebf84e5482701529f919a2d0018c796f3f0 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:20:59 +0100 Subject: [PATCH 02/16] Add server register/deregister endpoint --- proto/server.proto | 16 ++++++++++++++++ proto/typedb-service.proto | 2 ++ 2 files changed, 18 insertions(+) diff --git a/proto/server.proto b/proto/server.proto index b1c0cc35..7dd1914b 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -13,6 +13,21 @@ message ServerManager { repeated Server servers = 1; } } + + message Register { + message Req { + uint64 replica_id = 1; + string address = 2; + } + message Res {} + } + + message Deregister { + message Req { + uint64 replica_id = 1; + } + message Res {} + } } message Server { @@ -20,6 +35,7 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { + uint64 replica_id = 3; ReplicaType replica_type = 1; int64 term = 2; diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 79e69365..3b401c76 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -24,6 +24,8 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); + rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); // Server API rpc server_version (Server.Version.Req) returns (Server.Version.Res); From a3f6586d5e91c52e617db1304941a90aa6d57676 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:24:49 +0100 Subject: [PATCH 03/16] Update the numbering of ReplicaStatus fields --- proto/server.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/server.proto b/proto/server.proto index 7dd1914b..980731fc 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -35,9 +35,9 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { - uint64 replica_id = 3; - ReplicaType replica_type = 1; - int64 term = 2; + uint64 replica_id = 1; + ReplicaType replica_type = 2; + int64 term = 3; enum ReplicaType { Primary = 0; From 3088a14247ef31cae8107f85e335f96991fedd95 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Fri, 4 Jul 2025 09:29:27 +0100 Subject: [PATCH 04/16] Change type of term from in64 to uint64 --- proto/server.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/server.proto b/proto/server.proto index 980731fc..07afb72d 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -37,7 +37,7 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; ReplicaType replica_type = 2; - int64 term = 3; + uint64 term = 3; enum ReplicaType { Primary = 0; From aad93a5abff6842838844bda231609aeb2cfc7a0 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Wed, 9 Jul 2025 13:51:43 +0100 Subject: [PATCH 05/16] Move raft peering protocol to this repo --- grpc/rust/BUILD | 44 +++++++++++++++++--------------- grpc/rust/build.rs | 2 ++ proto/BUILD | 29 +++++++++++++++------ proto/raft-peering-service.proto | 9 +++++++ proto/raft-peering.proto | 12 +++++++++ 5 files changed, 67 insertions(+), 29 deletions(-) create mode 100644 proto/raft-peering-service.proto create mode 100644 proto/raft-peering.proto diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index d3e84fe6..6d9fbdd7 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -10,64 +10,66 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", - packages = ["typedb.protocol"], srcs = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", + "//proto:analyze-proto", + "//proto:analyzed-conjunction-proto", "//proto:answer-proto", + "//proto:authentication-proto", "//proto:concept-proto", "//proto:connection-proto", - "//proto:authentication-proto", + "//proto:database-proto", + "//proto:error-proto", + "//proto:migration-proto", "//proto:options-proto", "//proto:query-proto", + "//proto:raft-peering-proto", + "//proto:raft-peering-service", + "//proto:server-proto", "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", + "//proto:typedb-service", + "//proto:user-proto", "//proto:version-proto", - ] + ], + packages = ["typedb.protocol"], ) rust_library( name = "typedb_protocol", - crate_root = ":typedb_protocol_src", srcs = [ ":typedb_protocol_src", ], - deps = [ - "@crates//:tonic", - "@crates//:prost", - ], + crate_root = ":typedb_protocol_src", tags = [ "crate-name=typedb-protocol", ], visibility = ["//visibility:public"], + deps = [ + "@crates//:prost", + "@crates//:tonic", + ], ) assemble_crate( name = "assemble_crate", - target = ":typedb_protocol", description = "TypeDB Protocol", homepage = "https://github.com/typedb/typedb-protocol", - readme_file = "//:README.md", license = "MPL-2.0", license_file = "//:LICENSE", + readme_file = "//:README.md", repository = "https://github.com/typedb/typedb-protocol", + target = ":typedb_protocol", ) deploy_crate( name = "deploy_crate", - target = ":assemble_crate", + release = deployment["crate"]["release"], snapshot = deployment["crate"]["snapshot"], - release = deployment["crate"]["release"] + target = ":assemble_crate", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index 8ea60f8f..ad54f3c0 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -20,6 +20,8 @@ fn main() -> std::io::Result<()> { "../../proto/typedb-service.proto", "../../proto/user.proto", "../../proto/version.proto", + "../../proto/raft-peering.proto", + "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index d6e31acd..fbdd41b3 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,20 +6,33 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") +proto_library( + name = "raft-peering-service", + srcs = [":raft-peering-service.proto"], + deps = [ + ":raft-peering-proto", + ], +) + proto_library( name = "typedb-service", srcs = [":typedb-service.proto"], deps = [ - ":connection-proto", ":authentication-proto", - ":server-proto", - ":user-proto", + ":connection-proto", ":database-proto", - ":transaction-proto", ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", ], ) +proto_library( + name = "raft-peering-proto", + srcs = [":raft-peering.proto"], +) + proto_library( name = "server-proto", srcs = [":server.proto"], @@ -67,7 +80,7 @@ proto_library( ":authentication-proto", ":database-proto", ":server-proto", - ":version-proto" + ":version-proto", ], ) @@ -103,7 +116,7 @@ proto_library( ":error-proto", ":options-proto", ":query-proto", - ] + ], ) proto_library( @@ -146,12 +159,12 @@ filegroup( "query.proto", "transaction.proto", "version.proto", - ] + ], ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto new file mode 100644 index 00000000..e87a8e05 --- /dev/null +++ b/proto/raft-peering-service.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +import "proto/raft-peering.proto"; + +package typedb.protocol; + +service RaftPeering { + rpc Peering(PeeringRequest) returns (PeeringResponse); +} \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto new file mode 100644 index 00000000..33a1add6 --- /dev/null +++ b/proto/raft-peering.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package typedb.protocol; + +message PeeringRequest { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; +} + +message PeeringResponse { +} From a78e444b07d839b5148fe848f72afc7e08fd112f Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 21 Jul 2025 16:46:09 +0100 Subject: [PATCH 06/16] Add server_status message to the raft peering service (#229) ## Release notes: usage and product changes Add server_status message to the raft peering service and rename the old raft peering messages to adjust to the existing naming conventions. ## Implementation --- proto/BUILD | 3 +++ proto/raft-peering-service.proto | 7 ++++++- proto/raft-peering.proto | 27 ++++++++++++++++++++++----- proto/server.proto | 3 ++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index fbdd41b3..4be34bfb 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -31,6 +31,9 @@ proto_library( proto_library( name = "raft-peering-proto", srcs = [":raft-peering.proto"], + deps = [ + ":server-proto", + ], ) proto_library( diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto index e87a8e05..82301fd6 100644 --- a/proto/raft-peering-service.proto +++ b/proto/raft-peering-service.proto @@ -1,3 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; import "proto/raft-peering.proto"; @@ -5,5 +9,6 @@ import "proto/raft-peering.proto"; package typedb.protocol; service RaftPeering { - rpc Peering(PeeringRequest) returns (PeeringResponse); + rpc peering (Peering.Req) returns (Peering.Res); + rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); } \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto index 33a1add6..cc6411e7 100644 --- a/proto/raft-peering.proto +++ b/proto/raft-peering.proto @@ -1,12 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; package typedb.protocol; -message PeeringRequest { - uint64 from_id = 1; - string from_address = 2; - bytes message = 3; +import "proto/server.proto"; + +// TODO: Rename to Replication +message Peering { + message Req { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; + } + + message Res {} } -message PeeringResponse { +message ServerStatus { + message Req {} + + message Res { + string connection_address = 1; + Server.ReplicaStatus replica_status = 2; + } } diff --git a/proto/server.proto b/proto/server.proto index 07afb72d..e29d01e9 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -41,7 +41,8 @@ message Server { enum ReplicaType { Primary = 0; - Secondary = 1; + Candidate = 1; + Secondary = 2; } } From af19442e7c2b8367c1c568b4bb14a2075c00b7b4 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 14 Aug 2025 14:47:42 +0100 Subject: [PATCH 07/16] Split protocol into 3 separate services (#230) Split the protocol into 3 services: * TypeDBService (contains common TypeDB logic): server, database, user, and transaction management, etc. * TypeDBClusteringService (contains TypeDB logic regarding clustering): replica registration, server status. * RaftService (contains Raft logic): replication operations. Split the old RaftPeeringService into TypeDBClusteringService and RaftService. Move separate services and their implementation files into separate subpackages. Rename methods. Introduce `ServersGet` method for single server status retrieval. Introduce `registration` method to verify the replica registration operation before submitting it to Raft. --- grpc/java/BUILD | 34 ++-- grpc/nodejs/BUILD | 34 ++-- grpc/rust/BUILD | 36 +++-- grpc/rust/build.rs | 36 +++-- proto/BUILD | 152 +----------------- proto/raft_service/BUILD | 28 ++++ proto/raft_service/raft_service.proto | 14 ++ .../replication.proto} | 14 +- proto/typedb_clustering_service/BUILD | 30 ++++ .../typedb_clustering.proto | 25 +++ .../typedb_clustering_service.proto} | 8 +- proto/typedb_service/BUILD | 134 +++++++++++++++ proto/{ => typedb_service}/analyze.proto | 8 +- .../analyzed-conjunction.proto | 4 +- proto/{ => typedb_service}/answer.proto | 2 +- .../{ => typedb_service}/authentication.proto | 0 proto/{ => typedb_service}/concept.proto | 0 proto/{ => typedb_service}/connection.proto | 6 +- proto/{ => typedb_service}/database.proto | 2 +- proto/{ => typedb_service}/error.proto | 0 proto/{ => typedb_service}/migration.proto | 2 +- proto/{ => typedb_service}/options.proto | 0 proto/{ => typedb_service}/query.proto | 8 +- proto/{ => typedb_service}/server.proto | 7 + proto/{ => typedb_service}/transaction.proto | 8 +- .../typedb_service.proto} | 15 +- proto/{ => typedb_service}/user.proto | 0 27 files changed, 355 insertions(+), 252 deletions(-) create mode 100644 proto/raft_service/BUILD create mode 100644 proto/raft_service/raft_service.proto rename proto/{raft-peering.proto => raft_service/replication.proto} (61%) create mode 100644 proto/typedb_clustering_service/BUILD create mode 100644 proto/typedb_clustering_service/typedb_clustering.proto rename proto/{raft-peering-service.proto => typedb_clustering_service/typedb_clustering_service.proto} (66%) create mode 100644 proto/typedb_service/BUILD rename proto/{ => typedb_service}/analyze.proto (95%) rename proto/{ => typedb_service}/analyzed-conjunction.proto (97%) rename proto/{ => typedb_service}/answer.proto (97%) rename proto/{ => typedb_service}/authentication.proto (100%) rename proto/{ => typedb_service}/concept.proto (100%) rename proto/{ => typedb_service}/connection.proto (84%) rename proto/{ => typedb_service}/database.proto (97%) rename proto/{ => typedb_service}/error.proto (100%) rename proto/{ => typedb_service}/migration.proto (98%) rename proto/{ => typedb_service}/options.proto (100%) rename proto/{ => typedb_service}/query.proto (92%) rename proto/{ => typedb_service}/server.proto (91%) rename proto/{ => typedb_service}/transaction.proto (93%) rename proto/{typedb-service.proto => typedb_service/typedb_service.proto} (86%) rename proto/{ => typedb_service}/user.proto (100%) diff --git a/grpc/java/BUILD b/grpc/java/BUILD index cf6f2316..a793641b 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -9,22 +9,26 @@ load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") java_grpc_library( name = "typedb-protocol", protos = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], # TypeDB Core bundles JARs by maven coordinate, we can remove this when Core is rewritten in Rust tags = ["maven_coordinates=com.typedb:typedb-protocol:{pom_version}"], diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index c9b88271..8e00bae1 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -25,22 +25,26 @@ protoc_gen_ts_bin.protoc_gen_ts_binary( ts_grpc_compile( name = "typedb-protocol-src", deps = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ] ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 6d9fbdd7..200ec898 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -11,24 +11,26 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", srcs = [ - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", - "//proto:answer-proto", - "//proto:authentication-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:database-proto", - "//proto:error-proto", - "//proto:migration-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:raft-peering-proto", - "//proto:raft-peering-service", - "//proto:server-proto", - "//proto:transaction-proto", - "//proto:typedb-service", - "//proto:user-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], packages = ["typedb.protocol"], ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index ad54f3c0..8ba15f05 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -4,24 +4,26 @@ fn main() -> std::io::Result<()> { let protos = vec![ - "../../proto/analyze.proto", - "../../proto/answer.proto", - "../../proto/authentication.proto", - "../../proto/concept.proto", - "../../proto/connection.proto", - "../../proto/analyzed-conjunction.proto", - "../../proto/database.proto", - "../../proto/error.proto", - "../../proto/migration.proto", - "../../proto/options.proto", - "../../proto/query.proto", - "../../proto/server.proto", - "../../proto/transaction.proto", - "../../proto/typedb-service.proto", - "../../proto/user.proto", + "../../proto/raft_service/replication.proto", + "../../proto/raft_service/raft_service.proto", + "../../proto/typedb_clustering_service/typedb_clustering.proto", + "../../proto/typedb_clustering_service/typedb_clustering_service.proto", + "../../proto/typedb_service/analyze.proto", + "../../proto/typedb_service/analyzed-conjunction.proto", + "../../proto/typedb_service/answer.proto", + "../../proto/typedb_service/authentication.proto", + "../../proto/typedb_service/concept.proto", + "../../proto/typedb_service/connection.proto", + "../../proto/typedb_service/database.proto", + "../../proto/typedb_service/error.proto", + "../../proto/typedb_service/migration.proto", + "../../proto/typedb_service/options.proto", + "../../proto/typedb_service/query.proto", + "../../proto/typedb_service/server.proto", + "../../proto/typedb_service/transaction.proto", + "../../proto/typedb_service/typedb_service.proto", + "../../proto/typedb_service/user.proto", "../../proto/version.proto", - "../../proto/raft-peering.proto", - "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index 4be34bfb..3b6d63bf 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,142 +6,6 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") -proto_library( - name = "raft-peering-service", - srcs = [":raft-peering-service.proto"], - deps = [ - ":raft-peering-proto", - ], -) - -proto_library( - name = "typedb-service", - srcs = [":typedb-service.proto"], - deps = [ - ":authentication-proto", - ":connection-proto", - ":database-proto", - ":migration-proto", - ":server-proto", - ":transaction-proto", - ":user-proto", - ], -) - -proto_library( - name = "raft-peering-proto", - srcs = [":raft-peering.proto"], - deps = [ - ":server-proto", - ], -) - -proto_library( - name = "server-proto", - srcs = [":server.proto"], -) - -proto_library( - name = "database-proto", - srcs = [":database.proto"], - deps = [ - ":migration-proto", - ], -) - -proto_library( - name = "migration-proto", - srcs = [":migration.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "user-proto", - srcs = [":user.proto"], -) - -proto_library( - name = "answer-proto", - srcs = ["answer.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "authentication-proto", - srcs = ["authentication.proto"], -) - -proto_library( - name = "concept-proto", - srcs = ["concept.proto"], -) - -proto_library( - name = "connection-proto", - srcs = ["connection.proto"], - deps = [ - ":authentication-proto", - ":database-proto", - ":server-proto", - ":version-proto", - ], -) - -proto_library( - name = "options-proto", - srcs = ["options.proto"], -) - -proto_library( - name = "query-proto", - srcs = ["query.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "error-proto", - srcs = ["error.proto"], -) - -proto_library( - name = "transaction-proto", - srcs = ["transaction.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ":query-proto", - ], -) - -proto_library( - name = "analyze-proto", - srcs = ["analyze.proto"], - deps = [ - ":analyzed-conjunction-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "analyzed-conjunction-proto", - srcs = ["analyzed-conjunction.proto"], - deps = [ - ":answer-proto", - ":concept-proto", - ], -) - proto_library( name = "version-proto", srcs = ["version.proto"], @@ -153,14 +17,14 @@ proto_library( filegroup( name = "proto-raw-buffers", srcs = [ - "analyze.proto", - "answer.proto", - "concept.proto", - "connection.proto", - "analyzed-conjunction.proto", - "options.proto", - "query.proto", - "transaction.proto", + "//proto/typedb_service:analyze.proto", + "//proto/typedb_service:analyzed-conjunction.proto", + "//proto/typedb_service:answer.proto", + "//proto/typedb_service:concept.proto", + "//proto/typedb_service:connection.proto", + "//proto/typedb_service:options.proto", + "//proto/typedb_service:query.proto", + "//proto/typedb_service:transaction.proto", "version.proto", ], ) diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD new file mode 100644 index 00000000..62000b0e --- /dev/null +++ b/proto/raft_service/BUILD @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "raft-service", + srcs = ["raft_service.proto"], + deps = [ + ":replication-proto", + ], +) + +proto_library( + name = "replication-proto", + srcs = ["replication.proto"], + deps = [], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto new file mode 100644 index 00000000..51886f28 --- /dev/null +++ b/proto/raft_service/raft_service.proto @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +import "proto/raft_service/replication.proto"; + +package typedb.protocol; + +// TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? +service Raft { + rpc replication (Replication.Req) returns (Replication.Res); +} diff --git a/proto/raft-peering.proto b/proto/raft_service/replication.proto similarity index 61% rename from proto/raft-peering.proto rename to proto/raft_service/replication.proto index cc6411e7..7d9dc921 100644 --- a/proto/raft-peering.proto +++ b/proto/raft_service/replication.proto @@ -6,10 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/server.proto"; - -// TODO: Rename to Replication -message Peering { +message Replication { message Req { uint64 from_id = 1; string from_address = 2; @@ -18,12 +15,3 @@ message Peering { message Res {} } - -message ServerStatus { - message Req {} - - message Res { - string connection_address = 1; - Server.ReplicaStatus replica_status = 2; - } -} diff --git a/proto/typedb_clustering_service/BUILD b/proto/typedb_clustering_service/BUILD new file mode 100644 index 00000000..4a5caf09 --- /dev/null +++ b/proto/typedb_clustering_service/BUILD @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-clustering-service", + srcs = ["typedb_clustering_service.proto"], + deps = [ + ":typedb-clustering-proto", + ], +) + +proto_library( + name = "typedb-clustering-proto", + srcs = ["typedb_clustering.proto"], + deps = [ + "//proto/typedb_service:server-proto", + ], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/typedb_clustering_service/typedb_clustering.proto b/proto/typedb_clustering_service/typedb_clustering.proto new file mode 100644 index 00000000..74ab151a --- /dev/null +++ b/proto/typedb_clustering_service/typedb_clustering.proto @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +import "proto/typedb_service/server.proto"; + +message CanRegister { + message Req { + uint64 replica_id = 1; + } + + message Res {} +} + +message ServerStatus { + message Req {} + + message Res { + Server server = 1; + } +} diff --git a/proto/raft-peering-service.proto b/proto/typedb_clustering_service/typedb_clustering_service.proto similarity index 66% rename from proto/raft-peering-service.proto rename to proto/typedb_clustering_service/typedb_clustering_service.proto index 82301fd6..cc83c61a 100644 --- a/proto/raft-peering-service.proto +++ b/proto/typedb_clustering_service/typedb_clustering_service.proto @@ -4,11 +4,11 @@ syntax = "proto3"; -import "proto/raft-peering.proto"; +import "proto/typedb_clustering_service/typedb_clustering.proto"; package typedb.protocol; -service RaftPeering { - rpc peering (Peering.Req) returns (Peering.Res); +service TypeDBClustering { + rpc can_register (CanRegister.Req) returns (CanRegister.Res); rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); -} \ No newline at end of file +} diff --git a/proto/typedb_service/BUILD b/proto/typedb_service/BUILD new file mode 100644 index 00000000..7246b159 --- /dev/null +++ b/proto/typedb_service/BUILD @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-service", + srcs = ["typedb_service.proto"], + deps = [ + ":authentication-proto", + ":connection-proto", + ":database-proto", + ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", + ], +) + +proto_library( + name = "analyze-proto", + srcs = ["analyze.proto"], + deps = [ + ":analyzed-conjunction-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "analyzed-conjunction-proto", + srcs = ["analyzed-conjunction.proto"], + deps = [ + ":answer-proto", + ":concept-proto", + ], +) + +proto_library( + name = "answer-proto", + srcs = ["answer.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "authentication-proto", + srcs = ["authentication.proto"], +) + +proto_library( + name = "concept-proto", + srcs = ["concept.proto"], +) + +proto_library( + name = "connection-proto", + srcs = ["connection.proto"], + deps = [ + ":authentication-proto", + ":database-proto", + ":server-proto", + "//proto:version-proto", + ], +) + +proto_library( + name = "database-proto", + srcs = ["database.proto"], + deps = [ + ":migration-proto", + ], +) + +proto_library( + name = "error-proto", + srcs = ["error.proto"], +) + +proto_library( + name = "migration-proto", + srcs = ["migration.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "options-proto", + srcs = ["options.proto"], +) + +proto_library( + name = "query-proto", + srcs = ["query.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "server-proto", + srcs = ["server.proto"], +) + +proto_library( + name = "transaction-proto", + srcs = ["transaction.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ":query-proto", + ], +) + +proto_library( + name = "user-proto", + srcs = ["user.proto"], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/analyze.proto b/proto/typedb_service/analyze.proto similarity index 95% rename from proto/analyze.proto rename to proto/typedb_service/analyze.proto index 7e3eb921..1d3dee84 100644 --- a/proto/analyze.proto +++ b/proto/typedb_service/analyze.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyzed-conjunction.proto"; -import "proto/concept.proto"; -import "proto/error.proto"; -import "proto/options.proto"; +import "proto/typedb_service/analyzed-conjunction.proto"; +import "proto/typedb_service/concept.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; package typedb.protocol; diff --git a/proto/analyzed-conjunction.proto b/proto/typedb_service/analyzed-conjunction.proto similarity index 97% rename from proto/analyzed-conjunction.proto rename to proto/typedb_service/analyzed-conjunction.proto index eef88d0f..a28f01e1 100644 --- a/proto/analyzed-conjunction.proto +++ b/proto/typedb_service/analyzed-conjunction.proto @@ -4,8 +4,8 @@ syntax = "proto3"; -import "proto/answer.proto"; -import "proto/concept.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/answer.proto b/proto/typedb_service/answer.proto similarity index 97% rename from proto/answer.proto rename to proto/typedb_service/answer.proto index 91d48174..2e88e544 100644 --- a/proto/answer.proto +++ b/proto/typedb_service/answer.proto @@ -4,7 +4,7 @@ syntax = "proto3"; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/authentication.proto b/proto/typedb_service/authentication.proto similarity index 100% rename from proto/authentication.proto rename to proto/typedb_service/authentication.proto diff --git a/proto/concept.proto b/proto/typedb_service/concept.proto similarity index 100% rename from proto/concept.proto rename to proto/typedb_service/concept.proto diff --git a/proto/connection.proto b/proto/typedb_service/connection.proto similarity index 84% rename from proto/connection.proto rename to proto/typedb_service/connection.proto index b083d7f1..d4d3207d 100644 --- a/proto/connection.proto +++ b/proto/typedb_service/connection.proto @@ -4,9 +4,9 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/database.proto"; -import "proto/server.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; import "proto/version.proto"; package typedb.protocol; diff --git a/proto/database.proto b/proto/typedb_service/database.proto similarity index 97% rename from proto/database.proto rename to proto/typedb_service/database.proto index 46940921..e3a2ca03 100644 --- a/proto/database.proto +++ b/proto/typedb_service/database.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/migration.proto"; +import "proto/typedb_service/migration.proto"; message DatabaseManager { diff --git a/proto/error.proto b/proto/typedb_service/error.proto similarity index 100% rename from proto/error.proto rename to proto/typedb_service/error.proto diff --git a/proto/migration.proto b/proto/typedb_service/migration.proto similarity index 98% rename from proto/migration.proto rename to proto/typedb_service/migration.proto index 46ec24c2..ba82617e 100644 --- a/proto/migration.proto +++ b/proto/typedb_service/migration.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; message Migration { message Export { diff --git a/proto/options.proto b/proto/typedb_service/options.proto similarity index 100% rename from proto/options.proto rename to proto/typedb_service/options.proto diff --git a/proto/query.proto b/proto/typedb_service/query.proto similarity index 92% rename from proto/query.proto rename to proto/typedb_service/query.proto index 8fd640b1..c8e13474 100644 --- a/proto/query.proto +++ b/proto/typedb_service/query.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/answer.proto"; -import "proto/options.proto"; -import "proto/error.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/error.proto"; package typedb.protocol; diff --git a/proto/server.proto b/proto/typedb_service/server.proto similarity index 91% rename from proto/server.proto rename to proto/typedb_service/server.proto index e29d01e9..695a4098 100644 --- a/proto/server.proto +++ b/proto/typedb_service/server.proto @@ -14,6 +14,13 @@ message ServerManager { } } + message Get { + message Req {} + message Res { + Server server = 1; + } + } + message Register { message Req { uint64 replica_id = 1; diff --git a/proto/transaction.proto b/proto/typedb_service/transaction.proto similarity index 93% rename from proto/transaction.proto rename to proto/typedb_service/transaction.proto index ce5f648b..a47d07b7 100644 --- a/proto/transaction.proto +++ b/proto/typedb_service/transaction.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/error.proto"; -import "proto/options.proto"; -import "proto/query.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/query.proto"; package typedb.protocol; diff --git a/proto/typedb-service.proto b/proto/typedb_service/typedb_service.proto similarity index 86% rename from proto/typedb-service.proto rename to proto/typedb_service/typedb_service.proto index 3b401c76..03d66488 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -4,13 +4,13 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/connection.proto"; -import "proto/database.proto"; -import "proto/server.proto"; -import "proto/user.proto"; -import "proto/transaction.proto"; -import "proto/migration.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/connection.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; +import "proto/typedb_service/user.proto"; +import "proto/typedb_service/transaction.proto"; +import "proto/typedb_service/migration.proto"; package typedb.protocol; @@ -24,6 +24,7 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_get (ServerManager.Get.Req) returns (ServerManager.Get.Res); rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); diff --git a/proto/user.proto b/proto/typedb_service/user.proto similarity index 100% rename from proto/user.proto rename to proto/typedb_service/user.proto From ec2bbc5d7d86dca69622f8f4e1086bc7961343d5 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Fri, 24 Oct 2025 14:26:58 +0100 Subject: [PATCH 08/16] Add protobuf schema for Raft's 'state mutation request' (#232) Add protobuf schema for Raft's 'state mutation request'. The schema defines how the data is transported over the network and stored in the Raft log. - Add a protobuf definition for State Mutation Request - Create a mock RPC endpoint to handle that message as a hack to get the code generator to work --- grpc/java/BUILD | 6 ++- grpc/nodejs/BUILD | 34 ++++++++----- grpc/rust/BUILD | 3 +- proto/raft_service/BUILD | 7 +++ proto/raft_service/raft_service.proto | 2 + proto/raft_service/request.proto | 72 +++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 proto/raft_service/request.proto diff --git a/grpc/java/BUILD b/grpc/java/BUILD index a793641b..b35e96a0 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") @@ -12,8 +13,9 @@ java_grpc_library( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -36,7 +38,7 @@ java_grpc_library( checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index 8e00bae1..4db32940 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@typedb_bazel_distribution//npm:rules.bzl", "assemble_npm", "deploy_npm") load("@typedb_dependencies//distribution:deployment.bzl", "deployment") @@ -14,7 +15,10 @@ load("//grpc/nodejs:rules.bzl", "ts_grpc_compile") npm_link_all_packages(name = "node_modules") -exports_files(["package.json", "pnpm-lock.yaml"]) +exports_files([ + "package.json", + "pnpm-lock.yaml", +]) load("@typedb_protocol_npm//grpc/nodejs:protoc-gen-ts/package_json.bzl", protoc_gen_ts_bin = "bin") @@ -28,8 +32,9 @@ ts_grpc_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -45,12 +50,14 @@ ts_grpc_compile( "//proto/typedb_service:transaction-proto", "//proto/typedb_service:typedb-service", "//proto/typedb_service:user-proto", - ] + ], ) ts_project( name = "typedb-protocol", srcs = [":typedb-protocol-src"], + declaration = True, + transpiler = "tsc", tsconfig = { "compilerOptions": { "target": "es2019", @@ -60,26 +67,27 @@ ts_project( }, "include": [ "proto", - ] + ], }, - declaration = True, + visibility = ["//visibility:public"], deps = [ ":node_modules/@grpc/grpc-js", - ":node_modules/google-protobuf", ":node_modules/@types/google-protobuf", ":node_modules/@types/node", + ":node_modules/google-protobuf", ":node_modules/typescript", ], - transpiler = "tsc", - visibility = ["//visibility:public"], ) npm_package( name = "typedb-protocol-package", - srcs = [":typedb-protocol", "package.json"], + srcs = [ + "package.json", + ":typedb-protocol", + ], + include_external_repositories = ["typedb_protocol"], include_runfiles = False, package = "typedb-protocol", - include_external_repositories = ["typedb_protocol"], visibility = ["//visibility:public"], ) @@ -90,18 +98,18 @@ assemble_npm( deploy_npm( name = "deploy-npm", - target = ":assemble-npm", - snapshot = deployment["npm"]["snapshot"], release = deployment["npm"]["release"], + snapshot = deployment["npm"]["snapshot"], + target = ":assemble-npm", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), exclude = [ "package.json", "pnpm-lock.yaml", ], license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 200ec898..c3f05955 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -14,8 +14,9 @@ rust_tonic_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD index 62000b0e..c9fc4f22 100644 --- a/proto/raft_service/BUILD +++ b/proto/raft_service/BUILD @@ -11,6 +11,7 @@ proto_library( srcs = ["raft_service.proto"], deps = [ ":replication-proto", + ":request-proto", ], ) @@ -20,6 +21,12 @@ proto_library( deps = [], ) +proto_library( + name = "request-proto", + srcs = ["request.proto"], + deps = [], +) + checkstyle_test( name = "checkstyle", size = "small", diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto index 51886f28..d9346878 100644 --- a/proto/raft_service/raft_service.proto +++ b/proto/raft_service/raft_service.proto @@ -5,10 +5,12 @@ syntax = "proto3"; import "proto/raft_service/replication.proto"; +import "proto/raft_service/request.proto"; package typedb.protocol; // TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? service Raft { rpc replication (Replication.Req) returns (Replication.Res); + rpc request (Request) returns (Request); } diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto new file mode 100644 index 00000000..2296a367 --- /dev/null +++ b/proto/raft_service/request.proto @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +message Request { + oneof request { + Load load = 1; + DatabaseManager database_manager = 2; + DatabaseCommit database_commit = 3; + UserManager user_manager = 4; + } + + message Load {} + + message DatabaseManager { + oneof database_manager { + All all = 1; + Get get = 2; + GetUnrestricted get_unrestricted = 3; + Create create = 4; + CreateUnrestricted create_unrestricted = 5; + Delete delete = 6; + } + + message All {} + message Get { string name = 1; } + message GetUnrestricted { string name = 1; } + message Create { string name = 1; } + message CreateUnrestricted { string name = 1; } + message Delete { string name = 1; } + } + + message DatabaseCommit { + oneof database_commit { + SchemaCommit schema = 1; + DataCommit data = 2; + } + + message SchemaCommit { + string name = 1; + bytes commit_record = 2; + } + + message DataCommit { + string name = 1; + bytes commit_record = 2; + } + } + + message UserManager { + oneof user_manager { + All all = 1; + Get get = 2; + Contains contains = 3; + Create create = 4; + Update update = 5; + Delete delete = 6; + } + + message All { string accessor = 1; } + message Get { string name = 1; string accessor = 2; } + message Contains { string name = 1; } + message Create { bytes commit_record = 1; } + message Update { bytes commit_record = 1; } + message Delete { bytes commit_record = 1; } + } +} + From 47c50a9e2b1c6a19067ec775e9e022b9c6b7cac0 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Wed, 19 Nov 2025 17:49:45 +0000 Subject: [PATCH 09/16] Mirror TypeDB service methods in Raft service (#238) ## Release notes: usage and product changes Reorganize Raft service's request messages to mirror the structure of TypeDB service's methods. Additionally, optionalize the Server message to allow returning pure Raft Agent id without other server information available for Clustered operations (in case the replica is not available). ## Implementation I just did it! --------- Co-authored-by: Ganeshwara Hananda Co-authored-by: Ganeshwara Hananda --- proto/raft_service/request.proto | 105 ++++++++++++++++------ proto/typedb_service/server.proto | 6 +- proto/typedb_service/typedb_service.proto | 1 - 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto index 2296a367..ff88beb3 100644 --- a/proto/raft_service/request.proto +++ b/proto/raft_service/request.proto @@ -10,8 +10,9 @@ message Request { oneof request { Load load = 1; DatabaseManager database_manager = 2; - DatabaseCommit database_commit = 3; + Database database = 3; UserManager user_manager = 4; + User user = 5; } message Load {} @@ -19,25 +20,50 @@ message Request { message DatabaseManager { oneof database_manager { All all = 1; - Get get = 2; - GetUnrestricted get_unrestricted = 3; - Create create = 4; - CreateUnrestricted create_unrestricted = 5; - Delete delete = 6; + Contains contains = 2; + Get get = 3; + GetUnrestricted get_unrestricted = 4; + Create create = 5; + CreateUnrestricted create_unrestricted = 6; } message All {} - message Get { string name = 1; } - message GetUnrestricted { string name = 1; } - message Create { string name = 1; } - message CreateUnrestricted { string name = 1; } - message Delete { string name = 1; } + + message Contains { + string name = 1; + } + + message Get { + string name = 1; + } + + message GetUnrestricted { + string name = 1; + } + message Create { + string name = 1; + } + + message CreateUnrestricted { + string name = 1; + } } - message DatabaseCommit { - oneof database_commit { - SchemaCommit schema = 1; - DataCommit data = 2; + message Database { + oneof database { + Schema schema = 1; + TypeSchema type_schema = 2; + SchemaCommit schema_commit = 3; + DataCommit data_commit = 4; + Delete delete = 5; + } + + message Schema { + string name = 1; + } + + message TypeSchema { + string name = 1; } message SchemaCommit { @@ -49,24 +75,51 @@ message Request { string name = 1; bytes commit_record = 2; } + + message Delete { + string name = 1; + } } message UserManager { oneof user_manager { All all = 1; - Get get = 2; - Contains contains = 3; + Contains contains = 2; + Get get = 3; Create create = 4; - Update update = 5; - Delete delete = 6; } - message All { string accessor = 1; } - message Get { string name = 1; string accessor = 2; } - message Contains { string name = 1; } - message Create { bytes commit_record = 1; } - message Update { bytes commit_record = 1; } - message Delete { bytes commit_record = 1; } + message All { + string accessor = 1; + } + + message Contains { + string name = 1; + string accessor = 2; + } + + message Get { + string name = 1; + string accessor = 2; + } + + message Create { + bytes commit_record = 1; + } } -} + message User { + oneof user { + Update update = 1; + Delete delete = 2; + } + + message Update { + bytes commit_record = 1; + } + + message Delete { + bytes commit_record = 1; + } + } +} diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index 695a4098..f75f5f3d 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -38,13 +38,13 @@ message ServerManager { } message Server { - string address = 1; + optional string address = 1; optional ReplicaStatus replica_status = 2; message ReplicaStatus { uint64 replica_id = 1; - ReplicaType replica_type = 2; - uint64 term = 3; + optional ReplicaType replica_type = 2; + optional uint64 term = 3; enum ReplicaType { Primary = 0; diff --git a/proto/typedb_service/typedb_service.proto b/proto/typedb_service/typedb_service.proto index 03d66488..2dec0f6b 100644 --- a/proto/typedb_service/typedb_service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -59,5 +59,4 @@ service TypeDB { // requests and responses back-and-forth. The first transaction client message must // be {Transaction.Open.Req}. Closing the stream closes the transaction. rpc transaction (stream Transaction.Client) returns (stream Transaction.Server); - } From ae75c6edda51a5cccd4b1074bdc3f671ba1da7e2 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 20 Nov 2025 14:14:32 +0000 Subject: [PATCH 10/16] Rename ReplicaType to ReplicaRole --- proto/typedb_service/server.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index f75f5f3d..fa524fa1 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -43,10 +43,10 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; - optional ReplicaType replica_type = 2; + optional ReplicaRole replica_role = 2; optional uint64 term = 3; - enum ReplicaType { + enum ReplicaRole { Primary = 0; Candidate = 1; Secondary = 2; From 788f985129629cad413f063288b5dad48927ad49 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Fri, 28 Nov 2025 19:33:40 +0000 Subject: [PATCH 11/16] Complete replication requests for user management (#240) ## Release notes: usage and product changes Introduce protocol changes required by https://github.com/typedb/typedb-cluster/pull/642 : * User management requests now contain specific request information like username and password hashes (not actual passwords) instead of database commits. ## Implementation --- proto/raft_service/request.proto | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto index ff88beb3..294d24a0 100644 --- a/proto/raft_service/request.proto +++ b/proto/raft_service/request.proto @@ -104,7 +104,9 @@ message Request { } message Create { - bytes commit_record = 1; + string name = 1; + string password_hash = 2; + string accessor = 3; } } @@ -115,11 +117,15 @@ message Request { } message Update { - bytes commit_record = 1; + string name = 1; + optional string name_update = 2; + optional string password_hash_update = 3; + string accessor = 4; } message Delete { - bytes commit_record = 1; + string name = 1; + string accessor = 2; } } } From cf8f3cb3db1a9cf41cfc3f3bbb8959246c368e70 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 15 Dec 2025 16:56:44 +0000 Subject: [PATCH 12/16] Return the single target server instead of all servers in the initial connection (#242) ## Release notes: usage and product changes Substitute `ServerManager.All.Res` with `ServerManager.Get.Res` in the result of the connection opening message. Returning all servers in a cloud environment does not make sense, since you cannot know the actual address of other servers until you connect to them (they are probably hidden behind proxies, and the GRPC protocol does not even know about them). So, it makes more sense to return only the target server's server information, which helps map the connection address with the id of the replica and all the other parameters for the future requests. ## Implementation --- proto/typedb_service/connection.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/typedb_service/connection.proto b/proto/typedb_service/connection.proto index d4d3207d..5c42a0d7 100644 --- a/proto/typedb_service/connection.proto +++ b/proto/typedb_service/connection.proto @@ -24,7 +24,7 @@ message Connection { message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - ServerManager.All.Res servers_all = 3; + ServerManager.Get.Res server = 3; Authentication.Token.Create.Res authentication = 4; } } From 6b83ce686d543f1c73260516adac570c29f8e3d3 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 15 Dec 2025 17:54:54 +0000 Subject: [PATCH 13/16] Update version to 3.7.0-alpha1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 240bba90..f977d07d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.0 \ No newline at end of file +3.7.0-alpha1 \ No newline at end of file From 9dab855e4069e2dd8e481bd57c938ee4d4754b9c Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 12:29:22 +0000 Subject: [PATCH 14/16] Return servers_all to the connection result --- proto/typedb_service/connection.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/typedb_service/connection.proto b/proto/typedb_service/connection.proto index 5c42a0d7..d4d3207d 100644 --- a/proto/typedb_service/connection.proto +++ b/proto/typedb_service/connection.proto @@ -24,7 +24,7 @@ message Connection { message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - ServerManager.Get.Res server = 3; + ServerManager.All.Res servers_all = 3; Authentication.Token.Create.Res authentication = 4; } } From 2ab39b0ff947f598176819affd4132a3169de896 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 15:19:23 +0000 Subject: [PATCH 15/16] Change version to 3.7.0-alpha0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f977d07d..88af4715 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.0-alpha1 \ No newline at end of file +3.7.0-alpha0 \ No newline at end of file From 88d31ba3d236dbddb8dea87c1ab2237204d25f73 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 16:41:27 +0000 Subject: [PATCH 16/16] Add connection address to the Server message --- proto/typedb_service/server.proto | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index fa524fa1..f76792a6 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -38,8 +38,9 @@ message ServerManager { } message Server { - optional string address = 1; - optional ReplicaStatus replica_status = 2; + optional string serving_address = 1; + optional string connection_address = 2; + optional ReplicaStatus replica_status = 3; message ReplicaStatus { uint64 replica_id = 1;