Use transformations for type mapping removal

Signed-off-by: Peter Nied <[email protected]>
peternied · Jun 27, 2024 · c1cda29 · c1cda29
1 parent 863cb31
commit c1cda29
Show file tree

Hide file tree

Showing 14 changed files with 68 additions and 48 deletions.
diff --git a/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java b/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java
@@ -273,11 +273,11 @@ public static void main(String[] args) throws InterruptedException {
                 OpenSearchClient targetClient = new OpenSearchClient(targetConnection);
                 if (sourceVersion == ClusterVersion.ES_6_8) {
                     GlobalMetadataCreator_OS_2_11 metadataCreator = new GlobalMetadataCreator_OS_2_11(targetClient, templateWhitelist, componentTemplateWhitelist, List.of());
-                    ObjectNode transformedRoot = transformer.transformGlobalMetadata(globalMetadata);
+                    var transformedRoot = transformer.transformGlobalMetadata(globalMetadata);
                     metadataCreator.create(transformedRoot);
                 } else if (sourceVersion == ClusterVersion.ES_7_10) {
                     GlobalMetadataCreator_OS_2_11 metadataCreator = new GlobalMetadataCreator_OS_2_11(targetClient, List.of(), componentTemplateWhitelist, templateWhitelist);
-                    ObjectNode transformedRoot = transformer.transformGlobalMetadata(globalMetadata);
+                    var transformedRoot = transformer.transformGlobalMetadata(globalMetadata);
                     metadataCreator.create(transformedRoot);
                 }
             }
@@ -312,7 +312,7 @@ public static void main(String[] args) throws InterruptedException {
                     String reindexName = indexMetadata.getName() + indexSuffix;
                     logger.info("Recreating index " + indexMetadata.getName() + " as " + reindexName + " on target...");
 
-                    ObjectNode transformedRoot = transformer.transformIndexMetadata(indexMetadata);
+                    var transformedRoot = transformer.transformIndexMetadata(indexMetadata);
                     indexCreator.create(transformedRoot, reindexName, indexMetadata.getId());
                 }
             }

diff --git a/RFS/src/main/java/com/rfs/models/IndexMetadata.java b/RFS/src/main/java/com/rfs/models/IndexMetadata.java
@@ -1,6 +1,7 @@
 package com.rfs.models;
 
 import org.apache.lucene.codecs.CodecUtil;
+import org.opensearch.migrations.transformation.entity.Index;
 
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
@@ -15,7 +16,7 @@
 import com.rfs.common.RfsException;
 import com.rfs.common.SnapshotRepo;
 
-public interface IndexMetadata {
+public interface IndexMetadata extends Index {
 
     /*
     * Defines the behavior expected of an object that will surface the metadata of an index stored in a snapshot
@@ -28,7 +29,7 @@ public interface IndexMetadata {
     public String getName();
     public int getNumberOfShards();
     public ObjectNode getSettings();
-    public ObjectNode toObjectNode();
+    public IndexMetadata deepCopy();
 
     /**
     * Defines the behavior required to read a snapshot's index metadata as JSON and convert it into a Data object

diff --git a/RFS/src/main/java/com/rfs/transformers/Transformer.java b/RFS/src/main/java/com/rfs/transformers/Transformer.java
@@ -1,6 +1,5 @@
 package com.rfs.transformers;
 
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.rfs.models.GlobalMetadata;
 import com.rfs.models.IndexMetadata;
 
@@ -12,10 +11,10 @@ public interface Transformer {
     /**
      * Takes the raw JSON representing the Global Metadata of one version and returns a new, transformed copy of the JSON
      */
-    public ObjectNode transformGlobalMetadata(GlobalMetadata globalData);
+    public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData);
 
     /**
      * Takes the raw JSON representing the Index Metadata of one version and returns a new, transformed copy of the JSON
      */
-    public ObjectNode transformIndexMetadata(IndexMetadata indexData);    
+    public IndexMetadata transformIndexMetadata(IndexMetadata indexData);    
 }
diff --git a/RFS/src/main/java/com/rfs/transformers/Transformer_ES_6_8_to_OS_2_11.java b/RFS/src/main/java/com/rfs/transformers/Transformer_ES_6_8_to_OS_2_11.java
@@ -1,24 +1,33 @@
 package com.rfs.transformers;
 
+import java.util.List;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.rfs.models.GlobalMetadata;
 import com.rfs.models.IndexMetadata;
+import com.rfs.version_os_2_11.GlobalMetadataData_OS_2_11;
+
+import org.opensearch.migrations.transformation.TransformationRule;
+import org.opensearch.migrations.transformation.entity.Index;
+import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval;
 
 public class Transformer_ES_6_8_to_OS_2_11 implements Transformer {
     private static final Logger logger = LogManager.getLogger(Transformer_ES_6_8_to_OS_2_11.class);
     private static final ObjectMapper mapper = new ObjectMapper();
+
+    private final List<TransformationRule<Index>> indexTransformations = List.of(new IndexMappingTypeRemoval());
     private int awarenessAttributeDimensionality;
 
     public Transformer_ES_6_8_to_OS_2_11(int awarenessAttributeDimensionality) {
         this.awarenessAttributeDimensionality = awarenessAttributeDimensionality;
     }
 
     @Override
-    public ObjectNode transformGlobalMetadata(GlobalMetadata globalData) {
+    public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) {
         var root = globalData.toObjectNode();
         ObjectNode newRoot = mapper.createObjectNode();
 
@@ -50,22 +59,24 @@ public ObjectNode transformGlobalMetadata(GlobalMetadata globalData) {
         componentTemplatesRoot.set("component_template", componentTemplatesSubRoot);
         newRoot.set("component_template", componentTemplatesRoot);
 
-        return newRoot;
+        return new GlobalMetadataData_OS_2_11(newRoot);
     }
 
     @Override
-    public ObjectNode transformIndexMetadata(IndexMetadata index){
-        var root = index.toObjectNode();
-        ObjectNode newRoot = root.deepCopy();
+    public IndexMetadata transformIndexMetadata(IndexMetadata index) {
+        logger.debug("Original Object: " + index.raw().toString());
+        var copy = index.deepCopy();
+        var newRoot = copy.raw();
+
+        indexTransformations.forEach(transformer -> transformer.applyTransformation(index));
 
         TransformFunctions.removeIntermediateMappingsLevels(newRoot);
 
         newRoot.set("settings", TransformFunctions.convertFlatSettingsToTree((ObjectNode) newRoot.get("settings")));
         TransformFunctions.removeIntermediateIndexSettingsLevel(newRoot); // run before fixNumberOfReplicas
         TransformFunctions.fixReplicasForDimensionality(newRoot, awarenessAttributeDimensionality);
 
-        logger.debug("Original Object: " + root.toString());
         logger.debug("Transformed Object: " + newRoot.toString());
-        return newRoot;
+        return copy;
     }    
 }
diff --git a/RFS/src/main/java/com/rfs/transformers/Transformer_ES_7_10_OS_2_11.java b/RFS/src/main/java/com/rfs/transformers/Transformer_ES_7_10_OS_2_11.java
@@ -3,10 +3,12 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.rfs.models.GlobalMetadata;
 import com.rfs.models.IndexMetadata;
+import com.rfs.version_os_2_11.GlobalMetadataData_OS_2_11;
 
 public class Transformer_ES_7_10_OS_2_11 implements Transformer {
     private static final Logger logger = LogManager.getLogger(Transformer_ES_7_10_OS_2_11.class);
@@ -18,8 +20,8 @@ public Transformer_ES_7_10_OS_2_11(int awarenessAttributeDimensionality) {
     }
 
     @Override
-    public ObjectNode transformGlobalMetadata(GlobalMetadata metaData){
-        ObjectNode root = metaData.toObjectNode();
+    public GlobalMetadata transformGlobalMetadata(GlobalMetadata metaData){
+        ObjectNode root = metaData.toObjectNode().deepCopy();
 
         // Transform the legacy templates
         if (root.get("templates") != null) {
@@ -79,22 +81,22 @@ public ObjectNode transformGlobalMetadata(GlobalMetadata metaData){
             root.set("component_template", componentTemplatesRoot);
         }
 
-        return root;
+        return new GlobalMetadataData_OS_2_11(root);
     }
 
     @Override
-    public ObjectNode transformIndexMetadata(IndexMetadata indexData){
+    public IndexMetadata transformIndexMetadata(IndexMetadata indexData){
+        logger.debug("Original Object: " + indexData.raw().toString());
+        var copy = indexData.deepCopy();
+        var newRoot = copy.raw();
 
-        ObjectNode newRoot = indexData.toObjectNode().deepCopy();
-
         TransformFunctions.removeIntermediateMappingsLevels(newRoot);
 
         newRoot.set("settings", TransformFunctions.convertFlatSettingsToTree((ObjectNode) newRoot.get("settings")));
         TransformFunctions.removeIntermediateIndexSettingsLevel(newRoot); // run before fixNumberOfReplicas
         TransformFunctions.fixReplicasForDimensionality(newRoot, awarenessAttributeDimensionality);
 
-        logger.debug("Original Object: " + indexData.toObjectNode().toString());
         logger.debug("Transformed Object: " + newRoot.toString());
-        return newRoot;
+        return indexData;
     }
 }
diff --git a/RFS/src/main/java/com/rfs/version_es_6_8/GlobalMetadataData_ES_6_8.java b/RFS/src/main/java/com/rfs/version_es_6_8/GlobalMetadataData_ES_6_8.java
@@ -1,6 +1,7 @@
 package com.rfs.version_es_6_8;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.rfs.models.GlobalMetadata;
 
 public class GlobalMetadataData_ES_6_8 implements com.rfs.models.GlobalMetadata {
     private final ObjectNode root;
@@ -17,5 +18,4 @@ public ObjectNode toObjectNode() {
     public ObjectNode getTemplates() {
         return (ObjectNode) root.get("templates");
     }
-
 }
diff --git a/RFS/src/main/java/com/rfs/version_es_6_8/IndexMetadataData_ES_6_8.java b/RFS/src/main/java/com/rfs/version_es_6_8/IndexMetadataData_ES_6_8.java
@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.rfs.models.IndexMetadata;
 import com.rfs.transformers.TransformFunctions;
 
 public class IndexMetadataData_ES_6_8 implements com.rfs.models.IndexMetadata {
@@ -68,7 +69,12 @@ public ObjectNode getSettings() {
     }
 
     @Override
-    public ObjectNode toObjectNode() {
+    public ObjectNode raw() {
         return root;
     }
+
+    @Override
+    public IndexMetadata deepCopy() {
+        return new IndexMetadataData_ES_6_8(root.deepCopy(), indexId, indexName);
+    }
 }
diff --git a/RFS/src/main/java/com/rfs/version_es_7_10/IndexMetadataData_ES_7_10.java b/RFS/src/main/java/com/rfs/version_es_7_10/IndexMetadataData_ES_7_10.java
@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.rfs.models.IndexMetadata;
 import com.rfs.transformers.TransformFunctions;
 
 public class IndexMetadataData_ES_7_10 implements com.rfs.models.IndexMetadata {
@@ -68,7 +69,12 @@ public ObjectNode getSettings() {
     }
 
     @Override
-    public ObjectNode toObjectNode() {
+    public ObjectNode raw() {
         return root;
     }
+
+    @Override
+    public IndexMetadata deepCopy() {
+        return new IndexMetadataData_ES_7_10(root.deepCopy(), indexId, indexName);
+    }
 }
diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java
@@ -8,6 +8,7 @@
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.rfs.common.OpenSearchClient;
+import com.rfs.models.GlobalMetadata;
 
 public class GlobalMetadataCreator_OS_2_11 {
     private static final Logger logger = LogManager.getLogger(GlobalMetadataCreator_OS_2_11.class);
@@ -24,10 +25,10 @@ public GlobalMetadataCreator_OS_2_11(OpenSearchClient client, List<String> legac
         this.indexTemplateAllowlist = indexTemplateAllowlist;
     }
 
-    public void create(ObjectNode root) {
+    public void create(GlobalMetadata root) {
         logger.info("Setting Global Metadata");
 
-        GlobalMetadataData_OS_2_11 globalMetadata = new GlobalMetadataData_OS_2_11(root);
+        GlobalMetadataData_OS_2_11 globalMetadata = new GlobalMetadataData_OS_2_11(root.toObjectNode());
         createLegacyTemplates(globalMetadata, client, legacyTemplateAllowlist);
         createComponentTemplates(globalMetadata, client, componentTemplateAllowlist);
         createIndexTemplates(globalMetadata, client, indexTemplateAllowlist);

diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataData_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataData_OS_2_11.java
@@ -1,8 +1,9 @@
 package com.rfs.version_os_2_11;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.rfs.models.GlobalMetadata;
 
-public class GlobalMetadataData_OS_2_11 implements com.rfs.models.GlobalMetadata {
+public class GlobalMetadataData_OS_2_11 implements GlobalMetadata {
     private final ObjectNode root;
 
     public GlobalMetadataData_OS_2_11(ObjectNode root) {

diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/IndexCreator_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/IndexCreator_OS_2_11.java
@@ -5,6 +5,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.rfs.common.OpenSearchClient;
+import com.rfs.models.IndexMetadata;
 
 public class IndexCreator_OS_2_11 {
     private static final ObjectMapper mapper = new ObjectMapper();
@@ -14,8 +15,8 @@ public IndexCreator_OS_2_11 (OpenSearchClient client) {
         this.client = client;
     }
 
-    public Optional<ObjectNode> create(ObjectNode root, String indexName, String indexId) {
-        IndexMetadataData_OS_2_11 indexMetadata = new IndexMetadataData_OS_2_11(root, indexId, indexName);
+    public Optional<ObjectNode> create(IndexMetadata index, String indexName, String indexId) {
+        IndexMetadataData_OS_2_11 indexMetadata = new IndexMetadataData_OS_2_11(index.raw(), indexId, indexName);
 
         // Remove some settings which will cause errors if you try to pass them to the API
         ObjectNode settings = indexMetadata.getSettings();

diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/IndexMetadataData_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/IndexMetadataData_OS_2_11.java
@@ -45,7 +45,11 @@ public ObjectNode getSettings() {
     }
 
     @Override
-    public ObjectNode toObjectNode() {
+    public ObjectNode raw() {
         return root;
     }
+    @Override
+    public IndexMetadata deepCopy() {
+        return new IndexMetadataData_OS_2_11(root.deepCopy(), indexId, indexName);
+    }
 }
diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/transformation/entity/Entity.java b/coreUtilities/src/main/java/org/opensearch/migrations/transformation/entity/Entity.java
@@ -1,21 +1,13 @@
 package org.opensearch.migrations.transformation.entity;
 
-import org.opensearch.migrations.transformation.Version;
-
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
 /**
  * Entities that are used to perform transformations
  */
 public interface Entity {
-    default boolean isImmutable() {
-        return true;
-    }
-
-    /** TODO: Circle back to update this  */
-    default ObjectNode raw() {
-        return null;
-    }
-
-    Version getSourceVersion();
+    /**
+     * Gets the underlying entity as an ObjectNode
+     */
+    ObjectNode raw();
 }
diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/transformation/entity/Index.java b/coreUtilities/src/main/java/org/opensearch/migrations/transformation/entity/Index.java
@@ -1,8 +1,4 @@
 package org.opensearch.migrations.transformation.entity;
 
-import java.util.List;
-
 public interface Index extends Entity {
-    Cluster getImmutableCluster();
-    List<Doc> getImmutableDocs();
 }