From 53107c45333f4d9c01914b1bc039f84197067b8b Mon Sep 17 00:00:00 2001 From: PhongChuong Date: Thu, 26 Sep 2024 15:52:10 -0400 Subject: [PATCH] test: add max staleness to ExternalTableDefinition --- .../google/cloud/bigquery/BigQueryImpl.java | 3 --- .../bigquery/ExternalTableDefinition.java | 10 ++++---- .../bigquery/MaterializedViewDefinition.java | 2 -- .../bigquery/spi/v2/HttpBigQueryRpc.java | 2 -- .../cloud/bigquery/it/ITBigQueryTest.java | 23 +++++++------------ 5 files changed, 12 insertions(+), 28 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index c2e87faa8..576083215 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -803,7 +803,6 @@ public Table getTable(final String datasetId, final String tableId, TableOption. @Override public Table getTable(TableId tableId, TableOption... options) { - System.out.println("CHUONGPH: Ran get table"); // More context about why this: // https://github.com/googleapis/google-cloud-java/issues/3808 final TableId completeTableId = @@ -831,8 +830,6 @@ public com.google.api.services.bigquery.model.Table call() { if (getOptions().getThrowNotFound() && answer == null) { throw new BigQueryException(HTTP_NOT_FOUND, "Table not found"); } - // TODO(NOW) - System.out.println("CHUONGPH: table: " + answer); return answer == null ? null : Table.fromPb(this, answer); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index cf5307bb0..d9d03dece 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -379,10 +379,10 @@ public HivePartitioningOptions getHivePartitioningOptions() { @Override com.google.api.services.bigquery.model.Table toPb() { Table tablePb = super.toPb(); + tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb()); if (getMaxStaleness() != null) { tablePb.setMaxStaleness(getMaxStaleness()); } - tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb()); return tablePb; } @@ -594,11 +594,6 @@ public static ExternalTableDefinition of(String sourceUri, FormatOptions format) static ExternalTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); - if (tablePb.getMaxStaleness() != null) { - // This can be moved to TableDefinition once maxStaleness available for all table types. - builder.setMaxStaleness(tablePb.getMaxStaleness()); - } - com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration = tablePb.getExternalDataConfiguration(); if (externalDataConfiguration != null) { @@ -653,6 +648,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getMetadataCacheMode() != null) { builder.setMetadataCacheMode(externalDataConfiguration.getMetadataCacheMode()); } + if (tablePb.getMaxStaleness() != null) { + builder.setMaxStaleness(tablePb.getMaxStaleness()); + } } return builder.build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java index fef0a9974..69712bb3d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java @@ -134,8 +134,6 @@ Table toPb() { Table tablePb = super.toPb(); com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition = new com.google.api.services.bigquery.model.MaterializedViewDefinition(); - // TODO(NOW) - materializedViewDefinition.setMaxStaleness("INTERVAL 15 MINUTE"); if (getQuery() != null) { materializedViewDefinition.setQuery(getQuery()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index c311ee63a..37b2be1db 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -188,7 +188,6 @@ public Dataset create(Dataset dataset, Map options) { @Override public Table create(Table table, Map options) { try { - System.out.println("CHUONGPH: out going table value: " + table); validateRPC(); // unset the type, as it is output only table.setType(null); @@ -320,7 +319,6 @@ public Table getTable( .setFields(Option.FIELDS.getString(options)) .setView(getTableMetadataOption(options)) .execute(); - System.out.println("CHUONGPH: getTableRPC ans: " + ans); return ans; } catch (IOException ex) { BigQueryException serviceException = translate(ex); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 40f4fa3c0..ec1f64796 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1118,11 +1118,10 @@ public static void beforeClass() throws InterruptedException, IOException { @AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (bigquery != null) { - // TODO(NOW) - // RemoteBigQueryHelper.forceDelete(bigquery, DATASET); - // RemoteBigQueryHelper.forceDelete(bigquery, UK_DATASET); - // RemoteBigQueryHelper.forceDelete(bigquery, MODEL_DATASET); - // RemoteBigQueryHelper.forceDelete(bigquery, ROUTINE_DATASET); + RemoteBigQueryHelper.forceDelete(bigquery, DATASET); + RemoteBigQueryHelper.forceDelete(bigquery, UK_DATASET); + RemoteBigQueryHelper.forceDelete(bigquery, MODEL_DATASET); + RemoteBigQueryHelper.forceDelete(bigquery, ROUTINE_DATASET); } if (storage != null) { boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 10, TimeUnit.SECONDS); @@ -2109,14 +2108,13 @@ public void testCreateAndGetTableWithSelectedField() { @Test public void testCreateExternalTable() throws InterruptedException { - // TODO(NOW) String tableName = "test_create_external_table"; TableId tableId = TableId.of(DATASET, tableName); - System.out.println("CHUONGPH: TableId: " + tableId); ExternalTableDefinition externalTableDefinition = ExternalTableDefinition.of( - "gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json()).toBuilder().setMaxStaleness("INTERVAL '1' DAY") + "gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json()).toBuilder() + .setMaxStaleness("INTERVAL 15 MINUTE") .build(); TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); Table createdTable = bigquery.create(tableInfo); @@ -2124,7 +2122,6 @@ public void testCreateExternalTable() throws InterruptedException { assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(tableName, createdTable.getTableId().getTable()); Table remoteTable = bigquery.getTable(DATASET, tableName); - System.out.println("CHUONGPH: " + remoteTable); assertNotNull(remoteTable); assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); @@ -2163,7 +2160,7 @@ public void testCreateExternalTable() throws InterruptedException { rowCount++; } assertEquals(4, rowCount); - // assertTrue(remoteTable.delete()); + assertTrue(remoteTable.delete()); } @Test @@ -2277,10 +2274,8 @@ public void testCreateViewTable() throws InterruptedException { @Test public void testCreateMaterializedViewTable() { - // TODO(NOW) String tableName = "test_materialized_view_table"; TableId tableId = TableId.of(DATASET, tableName); - System.out.println("CHUONGPH: tableId: " + tableId); MaterializedViewDefinition viewDefinition = MaterializedViewDefinition.newBuilder( String.format( @@ -2298,7 +2293,7 @@ public void testCreateMaterializedViewTable() { assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition); assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema()); - // assertTrue(remoteTable.delete()); + assertTrue(remoteTable.delete()); } @Test @@ -5419,8 +5414,6 @@ public void testCopyJob() throws InterruptedException, TimeoutException { @Test public void testCopyJobStatistics() throws InterruptedException, TimeoutException { - // TODO(NOW) - System.out.println("CHUONGPH: Dataset: " + DATASET); String sourceTableName = "test_copy_job_statistics_source_table"; String destinationTableName = "test_copy_job_statistics_destination_table";