Skip to content

Commit

Permalink
Merge pull request #310 from alex268/master
Browse files Browse the repository at this point in the history 
Full support of table TTL config
  • Loading branch information
@alex268
alex268 authored Sep 8, 2024
2 parents 6444e31 + fd63f5b commit 381bd98
Show file tree
Hide file tree
Showing 8 changed files with 375 additions and 32 deletions.
20 changes: 19 additions & 1 deletion query/src/test/java/tech/ydb/query/TableExampleTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import java.time.Instant;
import java.time.LocalDate;
import java.time.Month;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -316,13 +318,29 @@ public void step07_scanQueryWithParams() {
"$seasonId", PrimitiveValue.newUint64(1)
);

final List<String> episodeTitle = new ArrayList<>();
final List<String> seasonTitle = new ArrayList<>();
final List<String> seriesTitle = new ArrayList<>();

retryCtx.supplyStatus(session -> {
episodeTitle.clear();
seasonTitle.clear();
seriesTitle.clear();

ExecuteScanQuerySettings settings = ExecuteScanQuerySettings.newBuilder().build();
GrpcReadStream<ResultSetReader> scan = session.executeScanQuery(query, params, settings);
return scan.start(rs -> {
Assert.assertTrue(rs.next());
while (rs.next()) {
episodeTitle.add(rs.getColumn("episode_title").getText());
seasonTitle.add(rs.getColumn("season_title").getText());
seriesTitle.add(rs.getColumn("series_title").getText());
}
});
}).join().expectSuccess("scan query problem");

Assert.assertEquals(14, episodeTitle.size());
Assert.assertEquals(14, seasonTitle.size());
Assert.assertEquals(14, seriesTitle.size());
}

@Test
Expand Down
8 changes: 7 additions & 1 deletion table/src/main/java/tech/ydb/table/description/TableDescription.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public static class Builder {
private TableStats tableStats = null;
private PartitioningSettings partitioningSettings = null;
private final List<PartitionStats> partitionStats = new ArrayList<>();
private TableTtl ttlSettings = new TableTtl();
private TableTtl ttlSettings = TableTtl.notSet();

public Builder addNonnullColumn(String name, Type type) {
return addNonnullColumn(name, type, null);
Expand Down Expand Up @@ -222,11 +222,17 @@ public Builder addPartitionStat(long rows, long size) {
return this;
}

@Deprecated
public Builder setTtlSettings(int ttlModeCase, String columnName, int expireAfterSeconds) {
this.ttlSettings = new TableTtl(TtlMode.forCase(ttlModeCase), columnName, expireAfterSeconds);
return this;
}

public Builder setTtlSettings(TableTtl ttl) {
this.ttlSettings = ttl;
return this;
}

private List<TableColumn> buildColumns() {
if (columns.isEmpty()) {
throw new IllegalStateException("cannot build table description with no columns");
Expand Down
96 changes: 90 additions & 6 deletions table/src/main/java/tech/ydb/table/description/TableTtl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,40 +4,80 @@
import javax.annotation.Nullable;

public class TableTtl {
private static final TableTtl NOT_SET = new TableTtl(TtlMode.NOT_SET, TtlUnit.UNSPECIFIED, "", 0, null);

@Nonnull
private final TtlMode ttlMode;
@Nullable
@Nonnull
private final TtlUnit ttlUnit;
@Nonnull
private final String dateTimeColumn;
@Nullable
@Nonnull
private final Integer expireAfterSeconds;
@Nonnull
private final Integer runIntervalSeconds;

@Deprecated
public TableTtl(@Nonnull TtlMode ttlMode, @Nonnull String dateTimeColumn, @Nonnull Integer expireAfterSeconds) {
this.ttlMode = ttlMode;
this.dateTimeColumn = dateTimeColumn;
this.expireAfterSeconds = expireAfterSeconds;
this.ttlUnit = TtlUnit.UNSPECIFIED;
this.runIntervalSeconds = null;
}

@Deprecated
public TableTtl() {
this.ttlMode = TtlMode.NOT_SET;
this.dateTimeColumn = null;
this.expireAfterSeconds = null;
this.dateTimeColumn = "";
this.expireAfterSeconds = 0;
this.ttlUnit = TtlUnit.UNSPECIFIED;
this.runIntervalSeconds = null;
}

private TableTtl(
@Nonnull TtlMode mode,
@Nonnull TtlUnit unit,
@Nonnull String columnName,
int expireAfterSeconds,
Integer runIntervalSeconds
) {
this.ttlMode = mode;
this.dateTimeColumn = columnName;
this.expireAfterSeconds = expireAfterSeconds;
this.ttlUnit = unit;
this.runIntervalSeconds = runIntervalSeconds;
}

@Nonnull
public TtlMode getTtlMode() {
return ttlMode;
}

@Nullable
@Nonnull
public TtlUnit getTtlUnit() {
return ttlUnit;
}

@Nonnull
public String getDateTimeColumn() {
return dateTimeColumn;
}

@Nullable
@Nonnull
public Integer getExpireAfterSeconds() {
return expireAfterSeconds;
}

@Nullable
public Integer getRunIntervaelSeconds() {
return runIntervalSeconds;
}

public TableTtl withRunIntervalSeconds(int seconds) {
return new TableTtl(ttlMode, ttlUnit, dateTimeColumn, expireAfterSeconds, seconds);
}

public enum TtlMode {
DATE_TYPE_COLUMN(1),
VALUE_SINCE_UNIX_EPOCH(2),
Expand All @@ -58,4 +98,48 @@ public static TtlMode forCase(int value) {
throw new IllegalArgumentException("No TTL mode defined for specified value");
}
}

public enum TtlUnit {
UNSPECIFIED,
SECONDS,
MILLISECONDS,
MICROSECONDS,
NANOSECONDS;
}

/**
* Construct an empty TTL configuration
*
* @return instance of TTL configuration
*/
public static TableTtl notSet() {
return NOT_SET;
}

/**
* The row will be considered as expired at the moment of time, when the value stored in <i>columnName</i> is less
* than or equal to the current time (in epoch time format), and <i>expireAfterSeconds</i> has passed since that
* moment; i.e. the expiration threshold is the value of <i>columnName</i>plus <i>expireAfterSeconds</i>.
*
* @param columnName name of column with type Date, Datetime or Timestamp
* @param expireAfterSeconds number of seconds to add to the time in the column
* @return instance of TTL configuration
*/
public static TableTtl dateTimeColumn(@Nonnull String columnName, int expireAfterSeconds) {
return new TableTtl(TtlMode.DATE_TYPE_COLUMN, TtlUnit.UNSPECIFIED, columnName, expireAfterSeconds, null);
}

/**
* The row will be considered as expired at the moment of time, when the value stored in <i>columnName</i> is less
* than or equal to the current time (in epoch time format), and <i>expireAfterSeconds</i> has passed since that
* moment; i.e. the expiration threshold is the value of <i>columnName</i>plus <i>expireAfterSeconds</i>.
*
* @param columnName name of column with type UInt32, UInt64 or DyNumber
* @param unit time unit of column
* @param expireAfterSeconds number of seconds to add to the time in the column
* @return instance of TTL configuration
*/
public static TableTtl valueSinceUnixEpoch(@Nonnull String columnName, TtlUnit unit, int expireAfterSeconds) {
return new TableTtl(TtlMode.VALUE_SINCE_UNIX_EPOCH, unit, columnName, expireAfterSeconds, null);
}
}
128 changes: 107 additions & 21 deletions table/src/main/java/tech/ydb/table/impl/BaseSession.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

import com.google.protobuf.Empty;
import com.google.protobuf.Timestamp;
import io.grpc.Metadata;
import org.slf4j.Logger;
Expand Down Expand Up @@ -45,6 +46,7 @@
import tech.ydb.table.description.TableColumn;
import tech.ydb.table.description.TableDescription;
import tech.ydb.table.description.TableIndex;
import tech.ydb.table.description.TableTtl;
import tech.ydb.table.query.DataQuery;
import tech.ydb.table.query.DataQueryResult;
import tech.ydb.table.query.ExplainDataQueryResult;
Expand Down Expand Up @@ -81,7 +83,6 @@
import tech.ydb.table.settings.ReplicationPolicy;
import tech.ydb.table.settings.RollbackTxSettings;
import tech.ydb.table.settings.StoragePolicy;
import tech.ydb.table.settings.TtlSettings;
import tech.ydb.table.transaction.TableTransaction;
import tech.ydb.table.transaction.Transaction;
import tech.ydb.table.transaction.TxControl;
Expand Down Expand Up @@ -268,16 +269,57 @@ private static YdbTable.ColumnFamily buildColumnFamity(ColumnFamily family) {
.build();
}

private static YdbTable.TtlSettings buildTtlSettings(TtlSettings settings) {
return YdbTable.TtlSettings.newBuilder()
.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(settings.getDateTimeColumn())
.setExpireAfterSeconds(settings.getExpireAfterSeconds())
.build())
.build();
private static YdbTable.TtlSettings buildTtlSettings(TableTtl ttl) {
if (ttl == null || ttl.getTtlMode() == TableTtl.TtlMode.NOT_SET) {
return null;
}

YdbTable.TtlSettings.Builder tb = YdbTable.TtlSettings.newBuilder();

if (ttl.getTtlMode() == TableTtl.TtlMode.DATE_TYPE_COLUMN) {
tb.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(ttl.getDateTimeColumn())
.setExpireAfterSeconds(ttl.getExpireAfterSeconds())
.build());
}

if (ttl.getTtlMode() == TableTtl.TtlMode.VALUE_SINCE_UNIX_EPOCH) {
YdbTable.ValueSinceUnixEpochModeSettings.Unit unit;
switch (ttl.getTtlUnit()) {
case SECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_SECONDS;
break;
case MILLISECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_MILLISECONDS;
break;
case MICROSECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_MICROSECONDS;
break;
case NANOSECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_NANOSECONDS;
break;
case UNSPECIFIED:
default:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_UNSPECIFIED;
break;
}

tb.setValueSinceUnixEpoch(YdbTable.ValueSinceUnixEpochModeSettings.newBuilder()
.setColumnName(ttl.getDateTimeColumn())
.setColumnUnit(unit)
.setExpireAfterSeconds(ttl.getExpireAfterSeconds())
.build());
}

if (ttl.getRunIntervaelSeconds() != null) {
tb.setRunIntervalSeconds(ttl.getRunIntervaelSeconds());
}

return tb.build();
}

@Override
@SuppressWarnings("deprecation")
public CompletableFuture<Status> createTable(
String path,
TableDescription description,
Expand All @@ -301,8 +343,22 @@ public CompletableFuture<Status> createTable(
request.addIndexes(buildIndex(index));
}

if (settings.getTtlSettings() != null) {
request.setTtlSettings(buildTtlSettings(settings.getTtlSettings()));
if (description.getTableTtl() != null) {
YdbTable.TtlSettings ttl = buildTtlSettings(description.getTableTtl());
if (ttl != null) {
request.setTtlSettings(ttl);
}
}
// deprecated variant has high pripority
tech.ydb.table.settings.TtlSettings deprecatedTTL = settings.getTtlSettings();
if (deprecatedTTL != null) {
YdbTable.TtlSettings ttl = YdbTable.TtlSettings.newBuilder()
.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(deprecatedTTL.getDateTimeColumn())
.setExpireAfterSeconds(deprecatedTTL.getExpireAfterSeconds())
.build())
.build();
request.setTtlSettings(ttl);
}

if (description.getPartitioningSettings() != null) {
Expand Down Expand Up @@ -430,8 +486,13 @@ public CompletableFuture<Status> alterTable(String path, AlterTableSettings sett
builder.addAddIndexes(buildIndex(index));
}

if (settings.getTtlSettings() != null) {
builder.setSetTtlSettings(buildTtlSettings(settings.getTtlSettings()));
if (settings.getTableTTL() != null) {
YdbTable.TtlSettings ttl = buildTtlSettings(settings.getTableTTL());
if (ttl != null) {
builder.setSetTtlSettings(ttl);
} else {
builder.setDropTtlSettings(Empty.getDefaultInstance());
}
}

if (settings.getPartitioningSettings() != null) {
Expand Down Expand Up @@ -641,22 +702,47 @@ private static TableDescription mapDescribeTable(
}
}

YdbTable.TtlSettings ttlSettings = result.getTtlSettings();
int ttlModeCase = ttlSettings.getModeCase().getNumber();
switch (ttlSettings.getModeCase()) {
YdbTable.TtlSettings ttl = result.getTtlSettings();
TableTtl tableTtl;
switch (ttl.getModeCase()) {
case DATE_TYPE_COLUMN:
YdbTable.DateTypeColumnModeSettings dateTypeColumn = ttlSettings.getDateTypeColumn();
description.setTtlSettings(ttlModeCase, dateTypeColumn.getColumnName(),
dateTypeColumn.getExpireAfterSeconds());
YdbTable.DateTypeColumnModeSettings dc = ttl.getDateTypeColumn();
tableTtl = TableTtl
.dateTimeColumn(dc.getColumnName(), dc.getExpireAfterSeconds())
.withRunIntervalSeconds(ttl.getRunIntervalSeconds());
break;
case VALUE_SINCE_UNIX_EPOCH:
YdbTable.ValueSinceUnixEpochModeSettings valueSinceUnixEpoch = ttlSettings.getValueSinceUnixEpoch();
description.setTtlSettings(ttlModeCase, valueSinceUnixEpoch.getColumnName(),
valueSinceUnixEpoch.getExpireAfterSeconds());
YdbTable.ValueSinceUnixEpochModeSettings vs = ttl.getValueSinceUnixEpoch();
TableTtl.TtlUnit unit;
switch (vs.getColumnUnit()) {
case UNIT_SECONDS:
unit = TableTtl.TtlUnit.SECONDS;
break;
case UNIT_MILLISECONDS:
unit = TableTtl.TtlUnit.MILLISECONDS;
break;
case UNIT_MICROSECONDS:
unit = TableTtl.TtlUnit.MICROSECONDS;
break;
case UNIT_NANOSECONDS:
unit = TableTtl.TtlUnit.NANOSECONDS;
break;
case UNIT_UNSPECIFIED:
case UNRECOGNIZED:
default:
unit = TableTtl.TtlUnit.UNSPECIFIED;
break;
}
tableTtl = TableTtl
.valueSinceUnixEpoch(vs.getColumnName(), unit, vs.getExpireAfterSeconds())
.withRunIntervalSeconds(ttl.getRunIntervalSeconds());
break;
case MODE_NOT_SET:
default:
tableTtl = TableTtl.notSet();
break;
}
description.setTtlSettings(tableTtl);

return description.build();
}
Expand Down
Loading

0 comments on commit 381bd98

Please sign in to comment.