Adds a reindex param

The reindex param is used to indicate the need for reindexing. This fixes an issue where the reindexing procedure would reset the field to processing, removing the stale but valid data that already existed in the column. It also makes it easier to find all the assets that require reindexing by having an indexed field to search for.
rpcpool · Jul 25, 2023 · f5b468b · f5b468b
1 parent 838a706
commit f5b468b
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,11 @@ test-ledger
 programs
 *.iml
 skaffold-state.json
-test-programs
+test-programs
+
+# Rust build dirs
+/das_api/target
+/migration/target
+/tests/*/target
+/*/target
+/target
diff --git a/digital_asset_types/src/dao/generated/asset_data.rs b/digital_asset_types/src/dao/generated/asset_data.rs
@@ -23,6 +23,7 @@ pub struct Model {
     pub metadata_mutability: Mutability,
     pub metadata: Json,
     pub slot_updated: i64,
+    pub reindex: Option<bool>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
@@ -34,6 +35,7 @@ pub enum Column {
     MetadataMutability,
     Metadata,
     SlotUpdated,
+    Reindex,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
@@ -64,6 +66,7 @@ impl ColumnTrait for Column {
             Self::MetadataMutability => Mutability::db_type(),
             Self::Metadata => ColumnType::JsonBinary.def(),
             Self::SlotUpdated => ColumnType::BigInteger.def(),
+            Self::Reindex => ColumnType::Boolean.def(),
         }
     }
 }

diff --git a/migration/src/lib.rs b/migration/src/lib.rs
@@ -16,6 +16,7 @@ mod m20230203_205959_improve_upsert_perf;
 mod m20230224_093722_performance_improvements;
 mod m20230310_162227_add_indexes_to_bg;
 mod m20230317_121944_remove_indexes_for_perf;
+mod m20230508_142103_add_offchain_reindex;
 
 pub struct Migrator;
 
@@ -39,6 +40,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230224_093722_performance_improvements::Migration),
             Box::new(m20230310_162227_add_indexes_to_bg::Migration),
             Box::new(m20230317_121944_remove_indexes_for_perf::Migration),
+            Box::new(m20230508_142103_add_offchain_reindex::Migration),
         ]
     }
 }
diff --git a/migration/src/m20230508_142103_add_offchain_reindex.rs b/migration/src/m20230508_142103_add_offchain_reindex.rs
@@ -0,0 +1,36 @@
+use digital_asset_types::dao::asset_data;
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+          .alter_table(
+            sea_query::Table::alter()
+              .table(asset_data::Entity)
+              .add_column(
+                ColumnDef::new(Alias::new("reindex"))
+                  .boolean()
+                  .default(false)
+              )
+              .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+      manager
+        .alter_table(
+          sea_query::Table::alter()
+            .table(asset_data::Entity)
+            .drop_column(Alias::new("reindex"))
+            .to_owned(),
+          )
+          .await?;
+      Ok(())
+    }
+}
diff --git a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs
@@ -92,6 +92,7 @@ where
                     metadata: Set(JsonValue::String("processing".to_string())),
                     metadata_mutability: Set(Mutability::Mutable),
                     slot_updated: Set(slot_i),
+                    reindex: Set(Some(true)),
                     ..Default::default()
                 };
 
@@ -102,9 +103,9 @@ where
                                 asset_data::Column::ChainDataMutability,
                                 asset_data::Column::ChainData,
                                 asset_data::Column::MetadataUrl,
-                                asset_data::Column::Metadata,
                                 asset_data::Column::MetadataMutability,
                                 asset_data::Column::SlotUpdated,
+                                asset_data::Column::Reindex,
                             ])
                             .to_owned(),
                     )

diff --git a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs
@@ -196,6 +196,7 @@ pub async fn save_v1_asset<T: ConnectionTrait + TransactionTrait>(
         metadata: Set(JsonValue::String("processing".to_string())),
         metadata_mutability: Set(Mutability::Mutable),
         slot_updated: Set(slot_i),
+        reindex: Set(Some(true)),
         id: Set(id.to_vec()),
     };
     let txn = conn.begin().await?;
@@ -206,9 +207,9 @@ pub async fn save_v1_asset<T: ConnectionTrait + TransactionTrait>(
                     asset_data::Column::ChainDataMutability,
                     asset_data::Column::ChainData,
                     asset_data::Column::MetadataUrl,
-                    asset_data::Column::Metadata,
                     asset_data::Column::MetadataMutability,
                     asset_data::Column::SlotUpdated,
+                    asset_data::Column::Reindex,
                 ])
                 .to_owned(),
         )

diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs
@@ -102,6 +102,7 @@ impl BgTask for DownloadMetadataTask {
         let model = asset_data::ActiveModel {
             id: Unchanged(download_metadata.asset_data_id.clone()),
             metadata: Set(body),
+            reindex: Set(Some(false)),
             ..Default::default()
         };
         debug!(