Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Full support of table TTL config #310

Merged
merged 2 commits into from
Sep 8, 2024
Merged

Full support of table TTL config #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38869a6
Full support of table TTL config
alex268 Sep 7, 2024
fd63f5b
Fixed flaky QueryService test
alex268 Sep 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion query/src/test/java/tech/ydb/query/TableExampleTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import java.time.Instant;
import java.time.LocalDate;
import java.time.Month;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -316,13 +318,29 @@ public void step07_scanQueryWithParams() {
"$seasonId", PrimitiveValue.newUint64(1)
);

final List<String> episodeTitle = new ArrayList<>();
final List<String> seasonTitle = new ArrayList<>();
final List<String> seriesTitle = new ArrayList<>();

retryCtx.supplyStatus(session -> {
episodeTitle.clear();
seasonTitle.clear();
seriesTitle.clear();

ExecuteScanQuerySettings settings = ExecuteScanQuerySettings.newBuilder().build();
GrpcReadStream<ResultSetReader> scan = session.executeScanQuery(query, params, settings);
return scan.start(rs -> {
Assert.assertTrue(rs.next());
while (rs.next()) {
episodeTitle.add(rs.getColumn("episode_title").getText());
seasonTitle.add(rs.getColumn("season_title").getText());
seriesTitle.add(rs.getColumn("series_title").getText());
}
});
}).join().expectSuccess("scan query problem");

Assert.assertEquals(14, episodeTitle.size());
Assert.assertEquals(14, seasonTitle.size());
Assert.assertEquals(14, seriesTitle.size());
}

@Test
Expand Down
8 changes: 7 additions & 1 deletion table/src/main/java/tech/ydb/table/description/TableDescription.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public static class Builder {
private TableStats tableStats = null;
private PartitioningSettings partitioningSettings = null;
private final List<PartitionStats> partitionStats = new ArrayList<>();
private TableTtl ttlSettings = new TableTtl();
private TableTtl ttlSettings = TableTtl.notSet();

public Builder addNonnullColumn(String name, Type type) {
return addNonnullColumn(name, type, null);
Expand Down Expand Up @@ -212,11 +212,17 @@ public Builder addPartitionStat(long rows, long size) {
return this;
}

@Deprecated
public Builder setTtlSettings(int ttlModeCase, String columnName, int expireAfterSeconds) {
this.ttlSettings = new TableTtl(TtlMode.forCase(ttlModeCase), columnName, expireAfterSeconds);
return this;
}

public Builder setTtlSettings(TableTtl ttl) {
this.ttlSettings = ttl;
return this;
}

private List<TableColumn> buildColumns() {
if (columns.isEmpty()) {
throw new IllegalStateException("cannot build table description with no columns");
Expand Down
96 changes: 90 additions & 6 deletions table/src/main/java/tech/ydb/table/description/TableTtl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,40 +4,80 @@
import javax.annotation.Nullable;

public class TableTtl {
private static final TableTtl NOT_SET = new TableTtl(TtlMode.NOT_SET, TtlUnit.UNSPECIFIED, "", 0, null);

@Nonnull
private final TtlMode ttlMode;
@Nullable
@Nonnull
private final TtlUnit ttlUnit;
@Nonnull
private final String dateTimeColumn;
@Nullable
@Nonnull
private final Integer expireAfterSeconds;
@Nonnull
private final Integer runIntervalSeconds;

@Deprecated
public TableTtl(@Nonnull TtlMode ttlMode, @Nonnull String dateTimeColumn, @Nonnull Integer expireAfterSeconds) {
this.ttlMode = ttlMode;
this.dateTimeColumn = dateTimeColumn;
this.expireAfterSeconds = expireAfterSeconds;
this.ttlUnit = TtlUnit.UNSPECIFIED;
this.runIntervalSeconds = null;
}

@Deprecated
public TableTtl() {
this.ttlMode = TtlMode.NOT_SET;
this.dateTimeColumn = null;
this.expireAfterSeconds = null;
this.dateTimeColumn = "";
this.expireAfterSeconds = 0;
this.ttlUnit = TtlUnit.UNSPECIFIED;
this.runIntervalSeconds = null;
}

private TableTtl(
@Nonnull TtlMode mode,
@Nonnull TtlUnit unit,
@Nonnull String columnName,
int expireAfterSeconds,
alex268 marked this conversation as resolved.
Show resolved Hide resolved
Integer runIntervalSeconds
) {
this.ttlMode = mode;
this.dateTimeColumn = columnName;
this.expireAfterSeconds = expireAfterSeconds;
this.ttlUnit = unit;
this.runIntervalSeconds = runIntervalSeconds;
}

@Nonnull
public TtlMode getTtlMode() {
return ttlMode;
}

@Nullable
@Nonnull
public TtlUnit getTtlUnit() {
return ttlUnit;
}

@Nonnull
public String getDateTimeColumn() {
return dateTimeColumn;
}

@Nullable
@Nonnull
public Integer getExpireAfterSeconds() {
return expireAfterSeconds;
}

@Nullable
public Integer getRunIntervaelSeconds() {
return runIntervalSeconds;
}

public TableTtl withRunIntervalSeconds(int seconds) {
return new TableTtl(ttlMode, ttlUnit, dateTimeColumn, expireAfterSeconds, seconds);
}

public enum TtlMode {
DATE_TYPE_COLUMN(1),
VALUE_SINCE_UNIX_EPOCH(2),
Expand All @@ -58,4 +98,48 @@ public static TtlMode forCase(int value) {
throw new IllegalArgumentException("No TTL mode defined for specified value");
}
}

public enum TtlUnit {
UNSPECIFIED,
SECONDS,
MILLISECONDS,
MICROSECONDS,
NANOSECONDS;
}

/**
* Construct an empty TTL configuration
*
* @return instance of TTL configuration
*/
public static TableTtl notSet() {
return NOT_SET;
}

/**
* The row will be considered as expired at the moment of time, when the value stored in <i>columnName</i> is less
* than or equal to the current time (in epoch time format), and <i>expireAfterSeconds</i> has passed since that
* moment; i.e. the expiration threshold is the value of <i>columnName</i>plus <i>expireAfterSeconds</i>.
*
* @param columnName name of column with type Date, Datetime or Timestamp
* @param expireAfterSeconds number of seconds to add to the time in the column
* @return instance of TTL configuration
*/
public static TableTtl dateTimeColumn(@Nonnull String columnName, int expireAfterSeconds) {
return new TableTtl(TtlMode.DATE_TYPE_COLUMN, TtlUnit.UNSPECIFIED, columnName, expireAfterSeconds, null);
}

/**
* The row will be considered as expired at the moment of time, when the value stored in <i>columnName</i> is less
* than or equal to the current time (in epoch time format), and <i>expireAfterSeconds</i> has passed since that
* moment; i.e. the expiration threshold is the value of <i>columnName</i>plus <i>expireAfterSeconds</i>.
*
* @param columnName name of column with type UInt32, UInt64 or DyNumber
* @param unit time unit of column
* @param expireAfterSeconds number of seconds to add to the time in the column
* @return instance of TTL configuration
*/
public static TableTtl valueSinceUnixEpoch(@Nonnull String columnName, TtlUnit unit, int expireAfterSeconds) {
alex268 marked this conversation as resolved.
Show resolved Hide resolved
return new TableTtl(TtlMode.VALUE_SINCE_UNIX_EPOCH, unit, columnName, expireAfterSeconds, null);
}
}
128 changes: 107 additions & 21 deletions table/src/main/java/tech/ydb/table/impl/BaseSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

import com.google.protobuf.Empty;
import com.google.protobuf.Timestamp;
import io.grpc.Metadata;
import org.slf4j.Logger;
Expand Down Expand Up @@ -45,6 +46,7 @@
import tech.ydb.table.description.TableColumn;
import tech.ydb.table.description.TableDescription;
import tech.ydb.table.description.TableIndex;
import tech.ydb.table.description.TableTtl;
import tech.ydb.table.query.DataQuery;
import tech.ydb.table.query.DataQueryResult;
import tech.ydb.table.query.ExplainDataQueryResult;
Expand Down Expand Up @@ -81,7 +83,6 @@
import tech.ydb.table.settings.ReplicationPolicy;
import tech.ydb.table.settings.RollbackTxSettings;
import tech.ydb.table.settings.StoragePolicy;
import tech.ydb.table.settings.TtlSettings;
import tech.ydb.table.transaction.TableTransaction;
import tech.ydb.table.transaction.Transaction;
import tech.ydb.table.transaction.TxControl;
Expand Down Expand Up @@ -265,16 +266,57 @@ private static YdbTable.ColumnFamily buildColumnFamity(ColumnFamily family) {
.build();
}

private static YdbTable.TtlSettings buildTtlSettings(TtlSettings settings) {
return YdbTable.TtlSettings.newBuilder()
.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(settings.getDateTimeColumn())
.setExpireAfterSeconds(settings.getExpireAfterSeconds())
.build())
.build();
private static YdbTable.TtlSettings buildTtlSettings(TableTtl ttl) {
if (ttl == null || ttl.getTtlMode() == TableTtl.TtlMode.NOT_SET) {
return null;
}

YdbTable.TtlSettings.Builder tb = YdbTable.TtlSettings.newBuilder();

if (ttl.getTtlMode() == TableTtl.TtlMode.DATE_TYPE_COLUMN) {
tb.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(ttl.getDateTimeColumn())
.setExpireAfterSeconds(ttl.getExpireAfterSeconds())
.build());
}

if (ttl.getTtlMode() == TableTtl.TtlMode.VALUE_SINCE_UNIX_EPOCH) {
YdbTable.ValueSinceUnixEpochModeSettings.Unit unit;
switch (ttl.getTtlUnit()) {
case SECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_SECONDS;
break;
case MILLISECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_MILLISECONDS;
break;
case MICROSECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_MICROSECONDS;
break;
case NANOSECONDS:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_NANOSECONDS;
break;
case UNSPECIFIED:
default:
unit = YdbTable.ValueSinceUnixEpochModeSettings.Unit.UNIT_UNSPECIFIED;
break;
}

tb.setValueSinceUnixEpoch(YdbTable.ValueSinceUnixEpochModeSettings.newBuilder()
.setColumnName(ttl.getDateTimeColumn())
.setColumnUnit(unit)
.setExpireAfterSeconds(ttl.getExpireAfterSeconds())
.build());
}

if (ttl.getRunIntervaelSeconds() != null) {
tb.setRunIntervalSeconds(ttl.getRunIntervaelSeconds());
}

return tb.build();
}

@Override
@SuppressWarnings("deprecation")
public CompletableFuture<Status> createTable(
String path,
TableDescription description,
Expand All @@ -298,8 +340,22 @@ public CompletableFuture<Status> createTable(
request.addIndexes(buildIndex(index));
}

if (settings.getTtlSettings() != null) {
request.setTtlSettings(buildTtlSettings(settings.getTtlSettings()));
if (description.getTableTtl() != null) {
YdbTable.TtlSettings ttl = buildTtlSettings(description.getTableTtl());
if (ttl != null) {
request.setTtlSettings(ttl);
}
}
// deprecated variant has high pripority
tech.ydb.table.settings.TtlSettings deprecatedTTL = settings.getTtlSettings();
if (deprecatedTTL != null) {
YdbTable.TtlSettings ttl = YdbTable.TtlSettings.newBuilder()
.setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder()
.setColumnName(deprecatedTTL.getDateTimeColumn())
.setExpireAfterSeconds(deprecatedTTL.getExpireAfterSeconds())
.build())
.build();
request.setTtlSettings(ttl);
}

if (description.getPartitioningSettings() != null) {
Expand Down Expand Up @@ -427,8 +483,13 @@ public CompletableFuture<Status> alterTable(String path, AlterTableSettings sett
builder.addAddIndexes(buildIndex(index));
}

if (settings.getTtlSettings() != null) {
builder.setSetTtlSettings(buildTtlSettings(settings.getTtlSettings()));
if (settings.getTableTTL() != null) {
YdbTable.TtlSettings ttl = buildTtlSettings(settings.getTableTTL());
if (ttl != null) {
builder.setSetTtlSettings(ttl);
} else {
builder.setDropTtlSettings(Empty.getDefaultInstance());
}
}

if (settings.getPartitioningSettings() != null) {
Expand Down Expand Up @@ -634,22 +695,47 @@ private static TableDescription mapDescribeTable(
}
}

YdbTable.TtlSettings ttlSettings = result.getTtlSettings();
int ttlModeCase = ttlSettings.getModeCase().getNumber();
switch (ttlSettings.getModeCase()) {
YdbTable.TtlSettings ttl = result.getTtlSettings();
TableTtl tableTtl;
switch (ttl.getModeCase()) {
case DATE_TYPE_COLUMN:
YdbTable.DateTypeColumnModeSettings dateTypeColumn = ttlSettings.getDateTypeColumn();
description.setTtlSettings(ttlModeCase, dateTypeColumn.getColumnName(),
dateTypeColumn.getExpireAfterSeconds());
YdbTable.DateTypeColumnModeSettings dc = ttl.getDateTypeColumn();
tableTtl = TableTtl
.dateTimeColumn(dc.getColumnName(), dc.getExpireAfterSeconds())
.withRunIntervalSeconds(ttl.getRunIntervalSeconds());
break;
case VALUE_SINCE_UNIX_EPOCH:
YdbTable.ValueSinceUnixEpochModeSettings valueSinceUnixEpoch = ttlSettings.getValueSinceUnixEpoch();
description.setTtlSettings(ttlModeCase, valueSinceUnixEpoch.getColumnName(),
valueSinceUnixEpoch.getExpireAfterSeconds());
YdbTable.ValueSinceUnixEpochModeSettings vs = ttl.getValueSinceUnixEpoch();
TableTtl.TtlUnit unit;
switch (vs.getColumnUnit()) {
case UNIT_SECONDS:
unit = TableTtl.TtlUnit.SECONDS;
break;
case UNIT_MILLISECONDS:
unit = TableTtl.TtlUnit.MILLISECONDS;
break;
case UNIT_MICROSECONDS:
unit = TableTtl.TtlUnit.MICROSECONDS;
break;
case UNIT_NANOSECONDS:
unit = TableTtl.TtlUnit.NANOSECONDS;
break;
case UNIT_UNSPECIFIED:
case UNRECOGNIZED:
default:
unit = TableTtl.TtlUnit.UNSPECIFIED;
break;
}
tableTtl = TableTtl
.valueSinceUnixEpoch(vs.getColumnName(), unit, vs.getExpireAfterSeconds())
.withRunIntervalSeconds(ttl.getRunIntervalSeconds());
break;
case MODE_NOT_SET:
default:
tableTtl = TableTtl.notSet();
break;
}
description.setTtlSettings(tableTtl);

return description.build();
}
Expand Down
Loading