Merge pull request #24 from wwong-snow/wwong/tool

Added tool to sync partitions

pom.xml

@@ -102,12 +102,12 @@
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-metastore</artifactId>
-            <version>3.1.1</version>
+            <version>2.3.5</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>3.1.1</version>
+            <version>2.7.7</version>
         </dependency>
         <dependency>
             <groupId>net.snowflake</groupId>

scripts/sync_hive_to_snowflake.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+HIVE_CLASSPATH=$(hive -e "set env:CLASSPATH;" | grep env:CLASSPATH= | sed -e "s/^env:CLASSPATH=//")
+java -cp $HIVE_CLASSPATH:. net.snowflake.hivemetastoreconnector.core.HiveSyncTool

src/main/java/net/snowflake/hivemetastoreconnector/SnowflakeConf.java

@@ -39,6 +39,8 @@ public enum ConfVars
       "The role to use to connect to Snowflake."),
     SNOWFLAKE_JDBC_DB("snowflake.jdbc.db", "db",
       "The database to use to connect to Snowflake."),
+    SNOWFLAKE_JDBC_WAREHOUSE("snowflake.jdbc.warehouse", "warehouse",
+      "The warehouse to use with Snowflake, if necessary."),
     SNOWFLAKE_JDBC_SCHEMA("snowflake.jdbc.schema", "schema",
       "The schema to use to connect to Snowflake."),
     SNOWFLAKE_JDBC_SSL("snowflake.jdbc.ssl", "ssl",

src/main/java/net/snowflake/hivemetastoreconnector/commands/AddPartition.java

@@ -73,11 +73,11 @@ public AddPartition(AlterPartitionEvent alterPartitionEvent,
    * @param isCompact internal marker to check if this command was generated
    *                  by compaction
    */
-  protected AddPartition(Table hiveTable,
-                         Iterator<Partition> partitionsIterator,
-                         SnowflakeConf snowflakeConf,
-                         Configuration hiveConf,
-                         boolean isCompact)
+  public AddPartition(Table hiveTable,
+                      Iterator<Partition> partitionsIterator,
+                      SnowflakeConf snowflakeConf,
+                      Configuration hiveConf,
+                      boolean isCompact)
   {
     super(hiveTable);
     this.hiveTable = Preconditions.checkNotNull(hiveTable);

src/main/java/net/snowflake/hivemetastoreconnector/commands/CreateExternalTable.java

@@ -50,10 +50,10 @@ public CreateExternalTable(CreateTableEvent createTableEvent,
    * @param hiveConf The Hive configuration
    * @param canReplace Whether to replace existing resources or not
    */
-  protected CreateExternalTable(Table hiveTable,
-                                SnowflakeConf snowflakeConf,
-                                Configuration hiveConf,
-                                boolean canReplace)
+  public CreateExternalTable(Table hiveTable,
+                             SnowflakeConf snowflakeConf,
+                             Configuration hiveConf,
+                             boolean canReplace)
   {
     super(hiveTable);
     this.hiveTable = Preconditions.checkNotNull(hiveTable);

src/main/java/net/snowflake/hivemetastoreconnector/commands/DropPartition.java

@@ -4,6 +4,7 @@
 package net.snowflake.hivemetastoreconnector.commands;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
 import net.snowflake.hivemetastoreconnector.util.StringUtil;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
@@ -12,7 +13,6 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.function.Supplier;
 
 /**
  * A class for the DropPartition command
@@ -28,26 +28,45 @@ public DropPartition(DropPartitionEvent dropPartitionEvent)
   {
     super(Preconditions.checkNotNull(dropPartitionEvent).getTable());
     this.hiveTable = Preconditions.checkNotNull(dropPartitionEvent.getTable());
-    this.getPartititonsIterator = dropPartitionEvent::getPartitionIterator;
+    // Avoid 'this' constructor due to usage of hiveTable below
+    this.partititonLocationsIterator = Iterators.transform(
+        dropPartitionEvent.getPartitionIterator(),
+        partition -> StringUtil.relativizePartitionURI(
+            hiveTable,
+            Preconditions.checkNotNull(partition)));
+  }
+
+  /**
+   * Creates a DropPartition command
+   * @param hiveTable The Hive table to generate a command from
+   * @param partititonLocationsIterator iterator of the locations of the
+   *                                    partitions to drop
+   */
+  public DropPartition(Table hiveTable, Iterator<String> partititonLocationsIterator)
+  {
+    super(hiveTable);
+    this.hiveTable = Preconditions.checkNotNull(hiveTable);
+    this.partititonLocationsIterator =
+        Preconditions.checkNotNull(partititonLocationsIterator);
   }
 
   /**
    * Generates the command for drop partitions.
    * Note: Unlike Hive, Snowflake partitions are dropped using locations.
-   * @param partition Partition object to generate a command from
+   * @param partitionLocation Partition location to generate a command from
    * @return The Snowflake command generated, for example:
    *         ALTER EXTERNAL TABLE t1 DROP PARTITION LOCATION 'location'
    *         /* TABLE LOCATION = 's3n://bucketname/path/to/table' * /;
    */
-  private String generateDropPartitionCommand(Partition partition)
+  private String generateDropPartitionCommand(String partitionLocation)
   {
     return String.format(
         "ALTER EXTERNAL TABLE %1$s " +
             "DROP PARTITION " +
             "LOCATION '%2$s' " +
             "/* TABLE LOCATION = '%3$s' */;",
         StringUtil.escapeSqlIdentifier(hiveTable.getTableName()),
-        StringUtil.escapeSqlText(StringUtil.relativizePartitionURI(hiveTable, partition)),
+        StringUtil.escapeSqlText(partitionLocation),
         StringUtil.escapeSqlComment(hiveTable.getSd().getLocation()));
   }
 
@@ -59,17 +78,16 @@ public List<String> generateSqlQueries()
   {
     List<String> queryList = new ArrayList<>();
 
-    Iterator<Partition> partitionIterator = this.getPartititonsIterator.get();
-    while (partitionIterator.hasNext())
+    while (partititonLocationsIterator.hasNext())
     {
-      Partition partition = partitionIterator.next();
-      queryList.add(this.generateDropPartitionCommand(partition));
+      queryList.add(
+          this.generateDropPartitionCommand(partititonLocationsIterator.next()));
     }
 
     return queryList;
   }
 
   private final Table hiveTable;
 
-  private final Supplier<Iterator<Partition>> getPartititonsIterator;
+  private final Iterator<String> partititonLocationsIterator;
 }