Skip to content

Commit

Permalink
test: add max staleness to ExternalTableDefinition
Browse files Browse the repository at this point in the history
  • Loading branch information
PhongChuong committed Sep 26, 2024
1 parent 172729e commit 53107c4
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,6 @@ public Table getTable(final String datasetId, final String tableId, TableOption.

@Override
public Table getTable(TableId tableId, TableOption... options) {
System.out.println("CHUONGPH: Ran get table");
// More context about why this:
// https://github.com/googleapis/google-cloud-java/issues/3808
final TableId completeTableId =
Expand Down Expand Up @@ -831,8 +830,6 @@ public com.google.api.services.bigquery.model.Table call() {
if (getOptions().getThrowNotFound() && answer == null) {
throw new BigQueryException(HTTP_NOT_FOUND, "Table not found");
}
// TODO(NOW)
System.out.println("CHUONGPH: table: " + answer);
return answer == null ? null : Table.fromPb(this, answer);
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -379,10 +379,10 @@ public HivePartitioningOptions getHivePartitioningOptions() {
@Override
com.google.api.services.bigquery.model.Table toPb() {
Table tablePb = super.toPb();
tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb());
if (getMaxStaleness() != null) {
tablePb.setMaxStaleness(getMaxStaleness());
}
tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb());
return tablePb;
}

Expand Down Expand Up @@ -594,11 +594,6 @@ public static ExternalTableDefinition of(String sourceUri, FormatOptions format)
static ExternalTableDefinition fromPb(Table tablePb) {
Builder builder = newBuilder().table(tablePb);

if (tablePb.getMaxStaleness() != null) {
// This can be moved to TableDefinition once maxStaleness available for all table types.
builder.setMaxStaleness(tablePb.getMaxStaleness());
}

com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration =
tablePb.getExternalDataConfiguration();
if (externalDataConfiguration != null) {
Expand Down Expand Up @@ -653,6 +648,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (externalDataConfiguration.getMetadataCacheMode() != null) {
builder.setMetadataCacheMode(externalDataConfiguration.getMetadataCacheMode());
}
if (tablePb.getMaxStaleness() != null) {
builder.setMaxStaleness(tablePb.getMaxStaleness());
}
}
return builder.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,6 @@ Table toPb() {
Table tablePb = super.toPb();
com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition =
new com.google.api.services.bigquery.model.MaterializedViewDefinition();
// TODO(NOW)
materializedViewDefinition.setMaxStaleness("INTERVAL 15 MINUTE");
if (getQuery() != null) {
materializedViewDefinition.setQuery(getQuery());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,6 @@ public Dataset create(Dataset dataset, Map<Option, ?> options) {
@Override
public Table create(Table table, Map<Option, ?> options) {
try {
System.out.println("CHUONGPH: out going table value: " + table);
validateRPC();
// unset the type, as it is output only
table.setType(null);
Expand Down Expand Up @@ -320,7 +319,6 @@ public Table getTable(
.setFields(Option.FIELDS.getString(options))
.setView(getTableMetadataOption(options))
.execute();
System.out.println("CHUONGPH: getTableRPC ans: " + ans);
return ans;
} catch (IOException ex) {
BigQueryException serviceException = translate(ex);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1118,11 +1118,10 @@ public static void beforeClass() throws InterruptedException, IOException {
@AfterClass
public static void afterClass() throws ExecutionException, InterruptedException {
if (bigquery != null) {
// TODO(NOW)
// RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
// RemoteBigQueryHelper.forceDelete(bigquery, UK_DATASET);
// RemoteBigQueryHelper.forceDelete(bigquery, MODEL_DATASET);
// RemoteBigQueryHelper.forceDelete(bigquery, ROUTINE_DATASET);
RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
RemoteBigQueryHelper.forceDelete(bigquery, UK_DATASET);
RemoteBigQueryHelper.forceDelete(bigquery, MODEL_DATASET);
RemoteBigQueryHelper.forceDelete(bigquery, ROUTINE_DATASET);
}
if (storage != null) {
boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 10, TimeUnit.SECONDS);
Expand Down Expand Up @@ -2109,22 +2108,20 @@ public void testCreateAndGetTableWithSelectedField() {

@Test
public void testCreateExternalTable() throws InterruptedException {
// TODO(NOW)
String tableName = "test_create_external_table";
TableId tableId = TableId.of(DATASET, tableName);
System.out.println("CHUONGPH: TableId: " + tableId);

ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.of(
"gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json()).toBuilder().setMaxStaleness("INTERVAL '1' DAY")
"gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json()).toBuilder()
.setMaxStaleness("INTERVAL 15 MINUTE")
.build();
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
Table remoteTable = bigquery.getTable(DATASET, tableName);
System.out.println("CHUONGPH: " + remoteTable);
assertNotNull(remoteTable);
assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
Expand Down Expand Up @@ -2163,7 +2160,7 @@ public void testCreateExternalTable() throws InterruptedException {
rowCount++;
}
assertEquals(4, rowCount);
// assertTrue(remoteTable.delete());
assertTrue(remoteTable.delete());
}

@Test
Expand Down Expand Up @@ -2277,10 +2274,8 @@ public void testCreateViewTable() throws InterruptedException {

@Test
public void testCreateMaterializedViewTable() {
// TODO(NOW)
String tableName = "test_materialized_view_table";
TableId tableId = TableId.of(DATASET, tableName);
System.out.println("CHUONGPH: tableId: " + tableId);
MaterializedViewDefinition viewDefinition =
MaterializedViewDefinition.newBuilder(
String.format(
Expand All @@ -2298,7 +2293,7 @@ public void testCreateMaterializedViewTable() {
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition);
assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema());
// assertTrue(remoteTable.delete());
assertTrue(remoteTable.delete());
}

@Test
Expand Down Expand Up @@ -5419,8 +5414,6 @@ public void testCopyJob() throws InterruptedException, TimeoutException {

@Test
public void testCopyJobStatistics() throws InterruptedException, TimeoutException {
// TODO(NOW)
System.out.println("CHUONGPH: Dataset: " + DATASET);
String sourceTableName = "test_copy_job_statistics_source_table";
String destinationTableName = "test_copy_job_statistics_destination_table";

Expand Down

0 comments on commit 53107c4

Please sign in to comment.