Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add max staleness to ExternalTableDefinition #3499

Merged
merged 4 commits into from
Oct 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,20 @@ public Builder setMetadataCacheMode(String metadataCacheMode) {

abstract Builder setMetadataCacheModeInner(String metadataCacheMode);

/**
* [Optional] Metadata Cache Mode for the table. Set this to enable caching of metadata from
* external data source.
*
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource:-table">
* MaxStaleness</a>
*/
public Builder setMaxStaleness(String maxStaleness) {
return setMaxStalenessInner(maxStaleness);
}

abstract Builder setMaxStalenessInner(String maxStaleness);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
Expand Down Expand Up @@ -305,6 +319,22 @@ public String getMetadataCacheMode() {
@Nullable
abstract String getMetadataCacheModeInner();

/**
* Returns the maximum staleness of data that could be returned when the table is queried.
* Staleness encoded as a string encoding of sql IntervalValue type.
*
* @see <a
* href="hhttps://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource:-table">
* MaxStaleness</a>
*/
@Nullable
public String getMaxStaleness() {
return getMaxStalenessInner();
}

@Nullable
abstract String getMaxStalenessInner();

/**
* Returns the source format, and possibly some parsing options, of the external data. Supported
* formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}.
Expand Down Expand Up @@ -351,6 +381,9 @@ public HivePartitioningOptions getHivePartitioningOptions() {
com.google.api.services.bigquery.model.Table toPb() {
Table tablePb = super.toPb();
tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb());
if (getMaxStaleness() != null) {
whuffman36 marked this conversation as resolved.
Show resolved Hide resolved
tablePb.setMaxStaleness(getMaxStaleness());
}
return tablePb;
}

Expand Down Expand Up @@ -616,6 +649,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (externalDataConfiguration.getMetadataCacheMode() != null) {
builder.setMetadataCacheMode(externalDataConfiguration.getMetadataCacheMode());
}
if (tablePb.getMaxStaleness() != null) {
builder.setMaxStaleness(tablePb.getMaxStaleness());
}
}
return builder.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ public class ExternalTableDefinitionTest {
.setSourceUriPrefix(SOURCE_URIS.get(0))
.build();
private static final String OBJECT_METADATA = "SIMPLE";

private static final String METADATA_CACHE_MODE = "AUTOMATIC";
private static final String MAX_STALENESS = "INTERVAL 15 MINUTE";
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH")
Expand All @@ -73,6 +73,7 @@ public class ExternalTableDefinitionTest {
.setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
.setObjectMetadata(OBJECT_METADATA)
.setMetadataCacheMode(METADATA_CACHE_MODE)
.setMaxStaleness(MAX_STALENESS)
.build();

private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO =
Expand Down Expand Up @@ -174,5 +175,6 @@ private void compareExternalTableDefinition(
assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
assertEquals(expected.getObjectMetadata(), value.getObjectMetadata());
assertEquals(expected.getMetadataCacheMode(), value.getMetadataCacheMode());
assertEquals(expected.getMaxStaleness(), value.getMaxStaleness());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2110,9 +2110,13 @@ public void testCreateAndGetTableWithSelectedField() {
public void testCreateExternalTable() throws InterruptedException {
String tableName = "test_create_external_table";
TableId tableId = TableId.of(DATASET, tableName);

ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.of(
"gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json());
"gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json())
.toBuilder()
.setMaxStaleness("INTERVAL 15 MINUTE")
.build();
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
Expand Down
Loading