From 898877d60a15b9bd53e7aef39dee94a7c27664cf Mon Sep 17 00:00:00 2001 From: Sergei Galiamichev Date: Mon, 23 Sep 2024 17:27:53 +0200 Subject: [PATCH] Global Unique Index --- WORKSPACE.bazel | 4 +-- .../ydb/yoj/databind/schema/GlobalIndex.java | 5 +++ .../tech/ydb/yoj/databind/schema/Schema.java | 10 +++++- pom.xml | 2 +- .../test/inmemory/InMemoryDataShard.java | 32 +++++++++++++++++-- .../test/inmemory/TxDataShardImpl.java | 2 +- repository-test/BUILD | 1 + .../yoj/repository/test/RepositoryTest.java | 10 ++++++ .../repository/test/entity/TestEntities.java | 3 +- .../test/sample/model/UniqueProject.java | 20 ++++++++++++ repository-ydb-v2/BUILD | 1 + .../ydb/client/YdbSchemaOperations.java | 14 ++++++-- .../YdbSchemaCompatibilityChecker.java | 4 +-- .../yoj/repository/ydb/YqlTypeLegacyTest.java | 4 +-- .../ydb/YqlTypeRecommendedTest.java | 4 +-- 15 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 repository-test/src/main/java/tech/ydb/yoj/repository/test/sample/model/UniqueProject.java diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index e2488f5b..e39c4143 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -39,9 +39,9 @@ SLF4J_VERSION = "2.0.11" SNAKEYAML_VERSION = "1.33" -YDB_PROTOAPI_VERSION = "1.6.0" +YDB_PROTOAPI_VERSION = "1.6.2" -YDB_SDK_VERSION = "2.1.12" +YDB_SDK_VERSION = "2.2.11" maven_install( name = "java_contribs_stable", diff --git a/databind/src/main/java/tech/ydb/yoj/databind/schema/GlobalIndex.java b/databind/src/main/java/tech/ydb/yoj/databind/schema/GlobalIndex.java index e2e277b0..a4fc710b 100644 --- a/databind/src/main/java/tech/ydb/yoj/databind/schema/GlobalIndex.java +++ b/databind/src/main/java/tech/ydb/yoj/databind/schema/GlobalIndex.java @@ -30,4 +30,9 @@ * List of annotated class fields representing index columns. */ String[] fields(); + + /** + * Is unique index type + */ + boolean unique() default false; } diff --git a/databind/src/main/java/tech/ydb/yoj/databind/schema/Schema.java b/databind/src/main/java/tech/ydb/yoj/databind/schema/Schema.java index 4a6cf04c..ed46d2a5 100644 --- a/databind/src/main/java/tech/ydb/yoj/databind/schema/Schema.java +++ b/databind/src/main/java/tech/ydb/yoj/databind/schema/Schema.java @@ -3,6 +3,7 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; import lombok.SneakyThrows; @@ -147,7 +148,7 @@ name, getType(), fieldPath) } columns.add(field.getName()); } - outputIndexes.add(new Index(name, List.copyOf(columns))); + outputIndexes.add(new Index(name, List.copyOf(columns), index.unique())); } return outputIndexes; } @@ -773,13 +774,20 @@ public FieldValueType getFieldValueType() { } @Value + @AllArgsConstructor public static class Index { + public Index (@NonNull String indexName, @NonNull List fieldNames) { + this(indexName, fieldNames, false); + } + @NonNull String indexName; @With @NonNull List fieldNames; + + boolean unique; } @Value diff --git a/pom.xml b/pom.xml index 219bba01..8b3f3f30 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ 1.7.1 - 2.2.8 + 2.2.11 1.6.2 diff --git a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryDataShard.java b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryDataShard.java index 2b394160..d74d585d 100644 --- a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryDataShard.java +++ b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryDataShard.java @@ -1,5 +1,6 @@ package tech.ydb.yoj.repository.test.inmemory; +import tech.ydb.yoj.databind.schema.Schema; import tech.ydb.yoj.repository.db.Entity; import tech.ydb.yoj.repository.db.EntityIdSchema; import tech.ydb.yoj.repository.db.EntitySchema; @@ -15,6 +16,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; @@ -142,16 +144,42 @@ public synchronized void insert(long txId, long version, T entity) { throw new EntityAlreadyExistsException("Entity " + entity.getId() + " already exists"); } - save(txId, entity); + save(txId, version, entity); } - public synchronized void save(long txId, T entity) { + public synchronized void save(long txId, long version, T entity) { InMemoryEntityLine entityLine = entityLines.computeIfAbsent(entity.getId(), __ -> new InMemoryEntityLine()); + validateUniqueness(txId, version, entity); uncommited.computeIfAbsent(txId, __ -> new HashSet<>()).add(entity.getId()); + entityLine.put(txId, Columns.fromEntity(schema, entity)); } + private void validateUniqueness(long txId, long version, T entity) { + List indexes = schema.getGlobalIndexes().stream() + .filter(Schema.Index::isUnique) + .toList(); + for (Schema.Index index : indexes) { + Object[] entityIndexValues = buildIndexValues(index, entity); + for (InMemoryEntityLine line : entityLines.values()) { + Columns columns = line.get(txId, version); + if (columns != null && Objects.deepEquals(entityIndexValues, buildIndexValues(index, columns.toSchema(schema)))) { + throw new EntityAlreadyExistsException("Entity " + entity.getId() + " already exists"); + } + } + } + } + + private Object[] buildIndexValues(Schema.Index index, T entity) { + Object[] objects = new Object[index.getFieldNames().size()]; + Map cells = schema.flatten(entity); + for (int i = 0; i < index.getFieldNames().size(); i++) { + objects[i] = cells.get(index.getFieldNames().get(i)); + } + return objects; + } + public synchronized void delete(long txId, Entity.Id id) { InMemoryEntityLine entityLine = entityLines.get(id); if (entityLine == null) { diff --git a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/TxDataShardImpl.java b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/TxDataShardImpl.java index ba58b673..a79a0404 100644 --- a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/TxDataShardImpl.java +++ b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/TxDataShardImpl.java @@ -38,7 +38,7 @@ public void insert(T entity) { @Override public void save(T entity) { - shard.save(txId, entity); + shard.save(txId, version, entity); } @Override diff --git a/repository-test/BUILD b/repository-test/BUILD index eb251236..ee72f32a 100644 --- a/repository-test/BUILD +++ b/repository-test/BUILD @@ -14,6 +14,7 @@ java_library( "@java_contribs_stable//:com_fasterxml_jackson_core_jackson_databind", "@java_contribs_stable//:com_fasterxml_jackson_datatype_jackson_datatype_jdk8", "@java_contribs_stable//:com_fasterxml_jackson_datatype_jackson_datatype_jsr310", + "@java_contribs_stable//:com_google_code_findbugs_jsr305", "@java_contribs_stable//:com_google_guava_guava", "@java_contribs_stable//:javax_annotation_javax_annotation_api", "@java_contribs_stable//:junit_junit", diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java index 76924a3e..3356ad42 100644 --- a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java @@ -56,6 +56,7 @@ import tech.ydb.yoj.repository.test.sample.model.TypeFreak.A; import tech.ydb.yoj.repository.test.sample.model.TypeFreak.B; import tech.ydb.yoj.repository.test.sample.model.TypeFreak.Embedded; +import tech.ydb.yoj.repository.test.sample.model.UniqueProject; import tech.ydb.yoj.repository.test.sample.model.UpdateFeedEntry; import tech.ydb.yoj.repository.test.sample.model.Version; import tech.ydb.yoj.repository.test.sample.model.VersionedAliasedEntity; @@ -101,6 +102,7 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static tech.ydb.yoj.repository.db.EntityExpressions.newFilterBuilder; @@ -1351,6 +1353,14 @@ private void findInKeysViewFilteredAndOrdered(Set keys, boole ); } + @Test + public void testUniqueIndex() { + UniqueProject ue1 = new UniqueProject(new UniqueProject.Id("id1"), "valuableName"); + db.tx(() -> db.table(UniqueProject.class).save(ue1)); + UniqueProject ue2 = new UniqueProject(new UniqueProject.Id("id2"), "valuableName"); + assertThrows(EntityAlreadyExistsException.class, () -> db.tx(() -> db.table(UniqueProject.class).insert(ue2))); + } + @Test public void doubleTxIsOk() { db.tx(this::findRange); diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java index 5152e1c3..8dc7582b 100644 --- a/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java @@ -21,6 +21,7 @@ import tech.ydb.yoj.repository.test.sample.model.Supabubble2; import tech.ydb.yoj.repository.test.sample.model.Team; import tech.ydb.yoj.repository.test.sample.model.TypeFreak; +import tech.ydb.yoj.repository.test.sample.model.UniqueProject; import tech.ydb.yoj.repository.test.sample.model.UpdateFeedEntry; import tech.ydb.yoj.repository.test.sample.model.VersionedAliasedEntity; import tech.ydb.yoj.repository.test.sample.model.VersionedEntity; @@ -34,7 +35,7 @@ private TestEntities() { @SuppressWarnings("rawtypes") public static final List> ALL = List.of( - Project.class, TypeFreak.class, Complex.class, Referring.class, Primitive.class, + Project.class, UniqueProject.class, TypeFreak.class, Complex.class, Referring.class, Primitive.class, Book.class, Book.ByAuthor.class, Book.ByTitle.class, LogEntry.class, Team.class, BytePkEntity.class, diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/sample/model/UniqueProject.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/sample/model/UniqueProject.java new file mode 100644 index 00000000..338af165 --- /dev/null +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/sample/model/UniqueProject.java @@ -0,0 +1,20 @@ +package tech.ydb.yoj.repository.test.sample.model; + +import lombok.Value; +import lombok.With; +import tech.ydb.yoj.databind.schema.GlobalIndex; +import tech.ydb.yoj.repository.db.Entity; + +@Value +@GlobalIndex(name = "unique_name", fields = {"name"}, unique = true) +public class UniqueProject implements Entity { + Id id; + @With + String name; + + @Value + public static class Id implements Entity.Id { + String value; + } +} + diff --git a/repository-ydb-v2/BUILD b/repository-ydb-v2/BUILD index 18f41e4b..75cbc754 100644 --- a/repository-ydb-v2/BUILD +++ b/repository-ydb-v2/BUILD @@ -21,6 +21,7 @@ java_library( "@java_contribs_stable//:tech_ydb_ydb_auth_api", "@java_contribs_stable//:tech_ydb_ydb_proto_api", "@java_contribs_stable//:tech_ydb_ydb_sdk_bom", + "@java_contribs_stable//:tech_ydb_ydb_sdk_common", "@java_contribs_stable//:tech_ydb_ydb_sdk_core", "@java_contribs_stable//:tech_ydb_ydb_sdk_scheme", "@java_contribs_stable//:tech_ydb_ydb_sdk_table", diff --git a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java index 860ccf4c..5a59f74c 100644 --- a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java +++ b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java @@ -17,6 +17,7 @@ import tech.ydb.scheme.description.ListDirectoryResult; import tech.ydb.table.Session; import tech.ydb.table.description.TableDescription; +import tech.ydb.table.description.TableIndex; import tech.ydb.table.description.TableTtl; import tech.ydb.table.settings.AlterTableSettings; import tech.ydb.table.settings.Changefeed; @@ -90,7 +91,13 @@ public void createTable(String name, List columns, List< }); List primaryKeysNames = primaryKeys.stream().map(Schema.JavaField::getName).collect(toList()); builder.setPrimaryKeys(primaryKeysNames); - globalIndexes.forEach(index -> builder.addGlobalIndex(index.getIndexName(), index.getFieldNames())); + globalIndexes.forEach(index -> { + if (index.isUnique()) { + builder.addGlobalUniqueIndex(index.getIndexName(), index.getFieldNames()); + } else { + builder.addGlobalIndex(index.getIndexName(), index.getFieldNames()); + } + }); Session session = sessionManager.getSession(); try { @@ -163,7 +170,7 @@ public Table describeTable(String name, List columns, Li }) .toList(); List ydbIndexes = indexes.stream() - .map(i -> new Index(i.getIndexName(), i.getFieldNames())) + .map(i -> new Index(i.getIndexName(), i.getFieldNames(), i.isUnique())) .toList(); TtlModifier tableTtl = ttlModifier == null ? null @@ -282,7 +289,7 @@ private Table describeTableInternal(String path) { }) .toList(), table.getIndexes().stream() - .map(i -> new Index(i.getName(), i.getColumns())) + .map(i -> new Index(i.getName(), i.getColumns(), i.getType() == TableIndex.Type.GLOBAL_UNIQUE)) .toList(), table.getTableTtl() == null || table.getTableTtl().getTtlMode() == TableTtl.TtlMode.NOT_SET ? null @@ -428,6 +435,7 @@ public static class Column { public static class Index { String name; List columns; + boolean unique; } @Value diff --git a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/compatibility/YdbSchemaCompatibilityChecker.java b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/compatibility/YdbSchemaCompatibilityChecker.java index 1f92d353..6a68f75c 100644 --- a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/compatibility/YdbSchemaCompatibilityChecker.java +++ b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/compatibility/YdbSchemaCompatibilityChecker.java @@ -345,7 +345,7 @@ private static String indexes(YdbSchemaOperations.Table table) { return "\n"; } return ",\n" + indexes.stream() - .map(idx -> "\tINDEX `" + idx.getName() + "` GLOBAL ON (" + indexColumns(idx.getColumns()) + ")") + .map(idx -> "\tINDEX `" + idx.getName() + "` GLOBAL " + (idx.isUnique() ? "UNIQUE " : "") + "ON (" + indexColumns(idx.getColumns()) + ")") .collect(Collectors.joining(",\n")) + "\n"; } @@ -401,7 +401,7 @@ private void makeMigrationTableIndexInstructions(YdbSchemaOperations.Table from, .collect(toMap(YdbSchemaOperations.Index::getName, Function.identity())); Function createIndex = i -> - String.format("ALTER TABLE `%s` ADD INDEX `%s` GLOBAL ON (%s);", + String.format("ALTER TABLE `%s` ADD INDEX `%s` GLOBAL " + (i.isUnique() ? "UNIQUE ": "") + "ON (%s);", to.getName(), i.getName(), i.getColumns().stream().map(c -> "`" + c + "`").collect(joining(",")) ); diff --git a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeLegacyTest.java b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeLegacyTest.java index b9929121..574d7ab5 100644 --- a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeLegacyTest.java +++ b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeLegacyTest.java @@ -213,7 +213,7 @@ public void testGlobalIndexMultiIndex() { var schema = ObjectSchema.of(GlobalIndexMultiIndex.class); Assert.assertEquals(List.of( new Schema.Index("idx1", List.of("id_id1", "id_3")), - new Schema.Index("idx2", List.of("id_2", "id_3"))), + new Schema.Index("idx2", List.of("id_2", "id_3"), true)), schema.getGlobalIndexes()); } @@ -366,7 +366,7 @@ public static class Id implements Entity.Id { } @GlobalIndex(name = "idx1", fields = {"id.id1", "id3"}) - @GlobalIndex(name = "idx2", fields = {"id.id2", "id3"}) + @GlobalIndex(name = "idx2", fields = {"id.id2", "id3"}, unique = true) @AllArgsConstructor public static class GlobalIndexMultiIndex implements Entity { Id id; diff --git a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeRecommendedTest.java b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeRecommendedTest.java index cb67b6da..6853817e 100644 --- a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeRecommendedTest.java +++ b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YqlTypeRecommendedTest.java @@ -213,7 +213,7 @@ public void testGlobalIndexMultiIndex() { var schema = ObjectSchema.of(GlobalIndexMultiIndex.class); Assert.assertEquals(List.of( new Schema.Index("idx1", List.of("id_id1", "id_3")), - new Schema.Index("idx2", List.of("id_2", "id_3"))), + new Schema.Index("idx2", List.of("id_2", "id_3"), true)), schema.getGlobalIndexes()); } @@ -366,7 +366,7 @@ public static class Id implements Entity.Id { } @GlobalIndex(name = "idx1", fields = {"id.id1", "id3"}) - @GlobalIndex(name = "idx2", fields = {"id.id2", "id3"}) + @GlobalIndex(name = "idx2", fields = {"id.id2", "id3"}, unique = true) @AllArgsConstructor public static class GlobalIndexMultiIndex implements Entity { Id id;