diff --git a/core/src/main/java/google/registry/export/ExportDomainListsAction.java b/core/src/main/java/google/registry/export/ExportDomainListsAction.java index 2911f14823a..6726630aa2d 100644 --- a/core/src/main/java/google/registry/export/ExportDomainListsAction.java +++ b/core/src/main/java/google/registry/export/ExportDomainListsAction.java @@ -17,7 +17,7 @@ import static com.google.common.base.Verify.verifyNotNull; import static google.registry.model.tld.Tlds.getTldsOfType; import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm; import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; @@ -28,6 +28,9 @@ import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; +import google.registry.model.common.FeatureFlag; +import google.registry.model.domain.rgp.GracePeriodStatus; +import google.registry.model.eppcommon.StatusValue; import google.registry.model.tld.Tld; import google.registry.model.tld.Tld.TldType; import google.registry.request.Action; @@ -38,8 +41,13 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.time.Instant; import java.util.List; import javax.inject.Inject; +import org.hibernate.query.NativeQuery; +import org.hibernate.query.TupleTransformer; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; /** * An action that exports the list of active domains on all real TLDs to Google Drive and GCS. @@ -55,7 +63,16 @@ public class ExportDomainListsAction implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt"; + private static final String SELECT_DOMAINS_STATEMENT = + "SELECT domainName FROM Domain WHERE tld = :tld AND deletionTime > :now ORDER by domainName"; + private static final String SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT = + "SELECT d.domain_name, d.deletion_time, d.statuses, gp.type FROM \"Domain\" d LEFT JOIN" + + " (SELECT * FROM \"GracePeriod\" WHERE type = 'REDEMPTION') AS gp ON d.repo_id =" + + " gp.domain_repo_id WHERE d.tld = :tld AND d.deletion_time > CAST(:now AS timestamptz)" + + " ORDER BY d.domain_name"; + + static final String REGISTERED_DOMAINS_TXT_FILENAME = "registered_domains.txt"; + static final String REGISTERED_DOMAINS_CSV_FILENAME = "registered_domains.csv"; @Inject Clock clock; @Inject DriveConnection driveConnection; @@ -68,49 +85,53 @@ public class ExportDomainListsAction implements Runnable { public void run() { ImmutableSet realTlds = getTldsOfType(TldType.REAL); logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds); + + boolean includeDeletionTimes = replicaTm().transact(this::retrieveIncludeDeletionTimesFlag); realTlds.forEach( tld -> { - List domains = - tm().transact( + List domainsList = + replicaTm() + .transact( TRANSACTION_REPEATABLE_READ, - () -> - // Note that if we had "creationTime <= :now" in the condition (not - // necessary as there is no pending creation, the order of deletionTime - // and creationTime in the query would have been significant and it - // should come after deletionTime. When Hibernate substitutes "now" it - // will first validate that the **first** field that is to be compared - // with it (deletionTime) is assignable from the substituted Java object - // (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it - // comes first, we will need to substitute "now" with - // CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit - // strange as the Java object type is clearly incompatible between the - // two fields deletionTime (DateTime) and creationTime, yet they are - // compared with the same "now". It is actually OK because in the end - // Hibernate converts everything to SQL types (and Java field names to - // SQL column names) to run the query. Both CreateAutoTimestamp and - // DateTime are persisted as timestamp_z in SQL. It is only the - // validation that compares the Java types, and only with the first - // field that compares with the substituted value. - tm().query( - "SELECT domainName FROM Domain " - + "WHERE tld = :tld " - + "AND deletionTime > :now " - + "ORDER by domainName ASC", - String.class) + () -> { + if (includeDeletionTimes) { + // We want to include deletion times, but only for domains in the 5-day + // PENDING_DELETE period after the REDEMPTION grace period. In order to + // accomplish this without loading the entire list of domains, we use a + // native query to join against the GracePeriod table to find + // PENDING_DELETE domains that don't have a REDEMPTION grace period. + return replicaTm() + .getEntityManager() + .createNativeQuery(SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT) + .unwrap(NativeQuery.class) + .setTupleTransformer(new DomainResultTransformer()) + .setParameter("tld", tld) + .setParameter("now", replicaTm().getTransactionTime().toString()) + .getResultList(); + } else { + return replicaTm() + .query(SELECT_DOMAINS_STATEMENT, String.class) .setParameter("tld", tld) - .setParameter("now", clock.nowUtc()) - .getResultList()); - String domainsList = Joiner.on("\n").join(domains); + .setParameter("now", replicaTm().getTransactionTime()) + .getResultList(); + } + }); logger.atInfo().log( - "Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld); - exportToGcs(tld, domainsList, gcsBucket, gcsUtils); - exportToDrive(tld, domainsList, driveConnection); + "Exporting %d domains for TLD %s to GCS and Drive.", domainsList.size(), tld); + String domainsListOutput = Joiner.on('\n').join(domainsList); + exportToGcs(tld, domainsListOutput, gcsBucket, gcsUtils, includeDeletionTimes); + exportToDrive(tld, domainsListOutput, driveConnection, includeDeletionTimes); }); } - protected static boolean exportToDrive( - String tldStr, String domains, DriveConnection driveConnection) { + protected static void exportToDrive( + String tldStr, + String domains, + DriveConnection driveConnection, + boolean includeDeletionTimes) { verifyNotNull(driveConnection, "Expecting non-null driveConnection"); + String filename = + includeDeletionTimes ? REGISTERED_DOMAINS_CSV_FILENAME : REGISTERED_DOMAINS_TXT_FILENAME; try { Tld tld = Tld.get(tldStr); if (tld.getDriveFolderId() == null) { @@ -120,10 +141,7 @@ protected static boolean exportToDrive( } else { String resultMsg = driveConnection.createOrUpdateFile( - REGISTERED_DOMAINS_FILENAME, - MediaType.PLAIN_TEXT_UTF_8, - tld.getDriveFolderId(), - domains.getBytes(UTF_8)); + filename, MediaType.CSV_UTF_8, tld.getDriveFolderId(), domains.getBytes(UTF_8)); logger.atInfo().log( "Exporting registered domains succeeded for TLD %s, response was: %s", tldStr, resultMsg); @@ -131,22 +149,49 @@ protected static boolean exportToDrive( } catch (Throwable e) { logger.atSevere().withCause(e).log( "Error exporting registered domains for TLD %s to Drive, skipping...", tldStr); - return false; } - return true; } - protected static boolean exportToGcs( - String tld, String domains, String gcsBucket, GcsUtils gcsUtils) { - BlobId blobId = BlobId.of(gcsBucket, tld + ".txt"); + protected static void exportToGcs( + String tld, + String domains, + String gcsBucket, + GcsUtils gcsUtils, + boolean includeDeletionTimes) { + String extension = includeDeletionTimes ? ".csv" : ".txt"; + BlobId blobId = BlobId.of(gcsBucket, tld + extension); try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId); Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) { osWriter.write(domains); } catch (Throwable e) { logger.atSevere().withCause(e).log( "Error exporting registered domains for TLD %s to GCS, skipping...", tld); + } + } + + /** If we should include deletion times in a CSV file, or use the standard txt file. */ + private boolean retrieveIncludeDeletionTimesFlag() { + try { + return FeatureFlag.isActiveNow( + FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS); + } catch (FeatureFlag.FeatureFlagNotFoundException e) { return false; } - return true; + } + + /** Transforms the multiple columns selected from SQL into the output line. */ + private static class DomainResultTransformer implements TupleTransformer { + @Override + public String transformTuple(Object[] domainResult, String[] strings) { + String domainName = (String) domainResult[0]; + Instant deletionInstant = (Instant) domainResult[1]; + DateTime deletionTime = new DateTime(deletionInstant.toEpochMilli(), DateTimeZone.UTC); + String[] domainStatuses = (String[]) domainResult[2]; + String gracePeriodType = (String) domainResult[3]; + boolean inPendingDelete = + ImmutableSet.copyOf(domainStatuses).contains(StatusValue.PENDING_DELETE.toString()) + && !GracePeriodStatus.REDEMPTION.toString().equals(gracePeriodType); + return String.format("%s,%s", domainName, inPendingDelete ? deletionTime : ""); + } } } diff --git a/core/src/main/java/google/registry/model/common/FeatureFlag.java b/core/src/main/java/google/registry/model/common/FeatureFlag.java index bd682348b8c..01d567d2175 100644 --- a/core/src/main/java/google/registry/model/common/FeatureFlag.java +++ b/core/src/main/java/google/registry/model/common/FeatureFlag.java @@ -66,6 +66,7 @@ public enum FeatureName { TEST_FEATURE, MINIMUM_DATASET_CONTACTS_OPTIONAL, MINIMUM_DATASET_CONTACTS_PROHIBITED, + INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS } /** The name of the flag/feature. */ diff --git a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java index b1332472ff4..8a585f4804a 100644 --- a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java +++ b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java @@ -15,11 +15,16 @@ package google.registry.export; import static com.google.common.truth.Truth.assertThat; -import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME; +import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_CSV_FILENAME; +import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_TXT_FILENAME; +import static google.registry.model.common.FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS; +import static google.registry.model.common.FeatureFlag.FeatureStatus.ACTIVE; +import static google.registry.model.common.FeatureFlag.FeatureStatus.INACTIVE; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; import static google.registry.testing.DatabaseHelper.persistResource; +import static google.registry.util.DateTimeUtils.START_OF_TIME; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; @@ -31,8 +36,14 @@ import com.google.cloud.storage.StorageException; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedMap; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; +import google.registry.model.common.FeatureFlag; +import google.registry.model.domain.Domain; +import google.registry.model.domain.GracePeriod; +import google.registry.model.domain.rgp.GracePeriodStatus; +import google.registry.model.eppcommon.StatusValue; import google.registry.model.tld.Tld; import google.registry.model.tld.Tld.TldType; import google.registry.persistence.transaction.JpaTestExtensions; @@ -56,7 +67,7 @@ class ExportDomainListsActionTest { @RegisterExtension final JpaIntegrationTestExtension jpa = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); + new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension(); @BeforeEach void beforeEach() { @@ -70,20 +81,21 @@ void beforeEach() { action.gcsUtils = gcsUtils; action.clock = clock; action.driveConnection = driveConnection; + persistFeatureFlag(INACTIVE); } - private void verifyExportedToDrive(String folderId, String domains) throws Exception { + private void verifyExportedToDrive(String folderId, String domains, boolean includeDeletionTimes) + throws Exception { + String filename = + includeDeletionTimes ? REGISTERED_DOMAINS_CSV_FILENAME : REGISTERED_DOMAINS_TXT_FILENAME; verify(driveConnection) .createOrUpdateFile( - eq(REGISTERED_DOMAINS_FILENAME), - eq(MediaType.PLAIN_TEXT_UTF_8), - eq(folderId), - bytesExportedToDrive.capture()); + eq(filename), eq(MediaType.CSV_UTF_8), eq(folderId), bytesExportedToDrive.capture()); assertThat(new String(bytesExportedToDrive.getValue(), UTF_8)).isEqualTo(domains); } @Test - void test_outputsOnlyActiveDomains() throws Exception { + void test_outputsOnlyActiveDomains_txt() throws Exception { persistActiveDomain("onetwo.tld"); persistActiveDomain("rudnitzky.tld"); persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z")); @@ -92,12 +104,27 @@ void test_outputsOnlyActiveDomains() throws Exception { String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8); // Check that it only contains the active domains, not the dead one. assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld"); - verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld"); + verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld", false); verifyNoMoreInteractions(driveConnection); } @Test - void test_outputsOnlyDomainsOnRealTlds() throws Exception { + void test_outputsOnlyActiveDomains_csv() throws Exception { + persistFeatureFlag(ACTIVE); + persistActiveDomain("onetwo.tld"); + persistActiveDomain("rudnitzky.tld"); + persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z")); + action.run(); + BlobId existingFile = BlobId.of("outputbucket", "tld.csv"); + String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8); + // Check that it only contains the active domains, not the dead one. + assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,"); + verifyExportedToDrive("brouhaha", "onetwo.tld,\nrudnitzky.tld,", true); + verifyNoMoreInteractions(driveConnection); + } + + @Test + void test_outputsOnlyDomainsOnRealTlds_txt() throws Exception { persistActiveDomain("onetwo.tld"); persistActiveDomain("rudnitzky.tld"); persistActiveDomain("wontgo.testtld"); @@ -111,12 +138,65 @@ void test_outputsOnlyDomainsOnRealTlds() throws Exception { assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile)); ImmutableList ls = gcsUtils.listFolderObjects("outputbucket", ""); assertThat(ls).containsExactly("tld.txt"); - verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld"); + verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld", false); + verifyNoMoreInteractions(driveConnection); + } + + @Test + void test_outputsOnlyDomainsOnRealTlds_csv() throws Exception { + persistFeatureFlag(ACTIVE); + persistActiveDomain("onetwo.tld"); + persistActiveDomain("rudnitzky.tld"); + persistActiveDomain("wontgo.testtld"); + action.run(); + BlobId existingFile = BlobId.of("outputbucket", "tld.csv"); + String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8).trim(); + // Check that it only contains the domains on the real TLD, and not the test one. + assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,"); + // Make sure that the test TLD file wasn't written out. + BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.csv"); + assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile)); + ImmutableList ls = gcsUtils.listFolderObjects("outputbucket", ""); + assertThat(ls).containsExactly("tld.csv"); + verifyExportedToDrive("brouhaha", "onetwo.tld,\nrudnitzky.tld,", true); verifyNoMoreInteractions(driveConnection); } @Test - void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception { + void test_outputIncludesDeletionTimes_forPendingDeletes_notRdemption() throws Exception { + persistFeatureFlag(ACTIVE); + // Domains pending delete (meaning the 5 day period, not counting the 30 day redemption period) + // should include their pending deletion date + persistActiveDomain("active.tld"); + Domain redemption = persistActiveDomain("redemption.tld"); + persistResource( + redemption + .asBuilder() + .addStatusValue(StatusValue.PENDING_DELETE) + .addGracePeriod( + GracePeriod.createWithoutBillingEvent( + GracePeriodStatus.REDEMPTION, + redemption.getRepoId(), + clock.nowUtc().plusDays(20), + redemption.getCurrentSponsorRegistrarId())) + .build()); + persistResource( + persistActiveDomain("pendingdelete.tld") + .asBuilder() + .addStatusValue(StatusValue.PENDING_DELETE) + .setDeletionTime(clock.nowUtc().plusDays(3)) + .build()); + + action.run(); + + verifyExportedToDrive( + "brouhaha", + "active.tld,\npendingdelete.tld,2020-02-05T02:02:02.000Z\nredemption.tld,", + true); + } + + @Test + void test_outputsDomainsFromDifferentTldsToMultipleFiles_txt() throws Exception { createTld("tldtwo"); persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build()); @@ -138,9 +218,48 @@ void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception { BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.txt"); String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim(); assertThat(evenMoreTlds).isEqualTo("cupid.tldthree"); - verifyExportedToDrive("brouhaha", "dasher.tld\nprancer.tld"); - verifyExportedToDrive("hooray", "buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo"); + verifyExportedToDrive("brouhaha", "dasher.tld\nprancer.tld", false); + verifyExportedToDrive("hooray", "buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo", false); // tldthree does not have a drive id, so no export to drive is performed. verifyNoMoreInteractions(driveConnection); } + + @Test + void test_outputsDomainsFromDifferentTldsToMultipleFiles_csv() throws Exception { + persistFeatureFlag(ACTIVE); + createTld("tldtwo"); + persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build()); + + createTld("tldthree"); + // You'd think this test was written around Christmas, but it wasn't. + persistActiveDomain("dasher.tld"); + persistActiveDomain("prancer.tld"); + persistActiveDomain("rudolph.tldtwo"); + persistActiveDomain("santa.tldtwo"); + persistActiveDomain("buddy.tldtwo"); + persistActiveDomain("cupid.tldthree"); + action.run(); + BlobId firstTldFile = BlobId.of("outputbucket", "tld.csv"); + String tlds = new String(gcsUtils.readBytesFrom(firstTldFile), UTF_8).trim(); + assertThat(tlds).isEqualTo("dasher.tld,\nprancer.tld,"); + BlobId secondTldFile = BlobId.of("outputbucket", "tldtwo.csv"); + String moreTlds = new String(gcsUtils.readBytesFrom(secondTldFile), UTF_8).trim(); + assertThat(moreTlds).isEqualTo("buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,"); + BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.csv"); + String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim(); + assertThat(evenMoreTlds).isEqualTo("cupid.tldthree,"); + verifyExportedToDrive("brouhaha", "dasher.tld,\nprancer.tld,", true); + verifyExportedToDrive("hooray", "buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,", true); + // tldthree does not have a drive id, so no export to drive is performed. + verifyNoMoreInteractions(driveConnection); + } + + private void persistFeatureFlag(FeatureFlag.FeatureStatus status) { + persistResource( + new FeatureFlag() + .asBuilder() + .setFeatureName(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS) + .setStatusMap(ImmutableSortedMap.of(START_OF_TIME, status)) + .build()); + } } diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index deb0ba709e6..d97b158fab2 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -470,7 +470,7 @@ ); create table "FeatureFlag" ( - feature_name text not null check (feature_name in ('TEST_FEATURE','MINIMUM_DATASET_CONTACTS_OPTIONAL','MINIMUM_DATASET_CONTACTS_PROHIBITED')), + feature_name text not null check (feature_name in ('TEST_FEATURE','MINIMUM_DATASET_CONTACTS_OPTIONAL','MINIMUM_DATASET_CONTACTS_PROHIBITED','INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS')), status hstore not null, primary key (feature_name) );