From 63e8c45d491b1babedd782ba50995187ce422f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Proch=C3=A1zka?= Date: Thu, 26 Jan 2023 23:09:41 +0100 Subject: [PATCH 1/2] [PDI-19718] - Regression of test * Changed load/save test which reflects behaviour of fieldUpdate and returnType property. --- .../DimensionLookupMetaTest.java | 101 ++++++++++-------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/engine/src/test/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMetaTest.java b/engine/src/test/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMetaTest.java index 6ff3432d04b6..622971a08443 100644 --- a/engine/src/test/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMetaTest.java +++ b/engine/src/test/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMetaTest.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -22,25 +22,7 @@ package org.pentaho.di.trans.steps.dimensionlookup; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - import org.apache.commons.lang.StringUtils; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -51,7 +33,6 @@ import org.pentaho.di.core.SQLStatement; import org.pentaho.di.core.database.Database; import org.pentaho.di.core.database.DatabaseMeta; -import org.pentaho.di.core.exception.KettleDatabaseException; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.logging.KettleLogStore; import org.pentaho.di.core.logging.LogChannelInterface; @@ -78,10 +59,28 @@ import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator; import org.pentaho.metastore.api.IMetaStore; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + public class DimensionLookupMetaTest implements InitializerInterface { LoadSaveTester loadSaveTester; + UpdateLoadSaveValidator updateValidator; + Class testMetaClass = DimensionLookupMeta.class; - private ThreadLocal holdTestingMeta = new ThreadLocal(); + private final ThreadLocal holdTestingMeta = new ThreadLocal<>(); @ClassRule public static RestorePDIEngineEnvironment env = new RestorePDIEngineEnvironment(); @BeforeClass @@ -95,19 +94,19 @@ public void setUpLoadSave() throws Exception { Arrays.asList( "schemaName", "tableName", "update", "dateField", "dateFrom", "dateTo", "keyField", "keyRename", "autoIncrement", "versionField", "commitSize", "useBatchUpdate", "minYear", "maxYear", "techKeyCreation", "cacheSize", "usingStartDateAlternative", "startDateAlternative", "startDateFieldName", "preloadingCache", "keyStream", - "keyLookup", "fieldStream", "fieldLookup", "fieldUpdate", "databaseMeta", "sequenceName" ); + "keyLookup", "fieldStream", "fieldLookup", "fieldUpdate", "returnType", "databaseMeta", "sequenceName" ); Map getterMap = new HashMap() { { put( "useBatchUpdate", "useBatchUpdate" ); } }; - Map setterMap = new HashMap(); + Map setterMap = new HashMap<>(); FieldLoadSaveValidator stringArrayLoadSaveValidator = - new ArrayLoadSaveValidator( new StringLoadSaveValidator(), 5 ); + new ArrayLoadSaveValidator<>( new StringLoadSaveValidator(), 5 ); - Map> attrValidatorMap = new HashMap>(); + Map> attrValidatorMap = new HashMap<>(); attrValidatorMap.put( "keyStream", stringArrayLoadSaveValidator ); attrValidatorMap.put( "keyLookup", stringArrayLoadSaveValidator ); attrValidatorMap.put( "fieldStream", stringArrayLoadSaveValidator ); @@ -117,13 +116,15 @@ public void setUpLoadSave() throws Exception { // ValueMetaInterface.TYPE_STRING. This happens about once out of every 3 or so runs of // the test which made it a bit difficult to track down. // MB - 5/2016 - attrValidatorMap.put( "fieldUpdate", new FieldUpdateIntArrayLoadSaveValidator( new NonZeroIntLoadSaveValidator( + attrValidatorMap.put( "update", updateValidator = new UpdateLoadSaveValidator() ); + attrValidatorMap.put( "fieldUpdate", new PrimitiveIntArrayLoadSaveValidator( new NonZeroIntLoadSaveValidator( DimensionLookupMeta.typeDesc.length ), 5 ) ); + attrValidatorMap.put( "returnType", new ReturnTypeLoadSaveValidator() ); attrValidatorMap.put( "databaseMeta", new DatabaseMetaLoadSaveValidator() ); attrValidatorMap.put( "startDateAlternative", new IntLoadSaveValidator( DimensionLookupMeta.getStartDateAlternativeCodes().length ) ); attrValidatorMap.put( "sequenceName", new SequenceNameLoadSaveValidator() ); - Map> typeValidatorMap = new HashMap>(); + Map> typeValidatorMap = new HashMap<>(); loadSaveTester = new LoadSaveTester( testMetaClass, attributes, new ArrayList(), new ArrayList(), @@ -149,6 +150,9 @@ public void modify( StepMetaInterface someMeta ) { @Test public void testSerialization() throws KettleException { + updateValidator.value = true; + loadSaveTester.testSerialization(); + updateValidator.value = false; loadSaveTester.testSerialization(); } @@ -182,7 +186,7 @@ public void testGetFields() throws Exception { meta.setFieldLookup( new String[] { "field1" } ); meta.setFieldStream( new String[] { "" } ); meta.setDatabaseMeta( dbMeta ); - doReturn( extraFields ).when( meta ).getDatabaseTableFields( (Database) anyObject(), anyString(), anyString() ); + doReturn( extraFields ).when( meta ).getDatabaseTableFields( anyObject(), anyString(), anyString() ); doReturn( mock( LogChannelInterface.class ) ).when( meta ).getLog(); RowMeta row = new RowMeta(); @@ -237,7 +241,7 @@ public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, bo } @Test - public void testProvidesModelerMeta() throws Exception { + public void testProvidesModelerMeta() { final RowMeta rowMeta = Mockito.mock( RowMeta.class ); final DimensionLookupMeta dimensionLookupMeta = new DimensionLookupMeta() { @@ -245,8 +249,7 @@ public void testProvidesModelerMeta() throws Exception { return mock( Database.class ); } - @Override protected RowMetaInterface getDatabaseTableFields( Database db, String schemaName, String tableName ) - throws KettleDatabaseException { + @Override protected RowMetaInterface getDatabaseTableFields( Database db, String schemaName, String tableName ) { assertEquals( "aSchema", schemaName ); assertEquals( "aDimTable", tableName ); return rowMeta; @@ -284,7 +287,6 @@ public void testProvidesModelerMeta() throws Exception { // this special load/save handler for sequenceName. // MB - 5/2016 public class SequenceNameLoadSaveValidator implements FieldLoadSaveValidator { - final Random rand = new Random(); @Override public String getTestObject() { DimensionLookupMeta dlm = holdTestingMeta.get(); // get the currently-being tested meta @@ -307,30 +309,37 @@ public boolean validateTestObject( String testObject, Object actual ) { } } - public class FieldUpdateIntArrayLoadSaveValidator extends PrimitiveIntArrayLoadSaveValidator { + /** + * This validator simulates behavior in {@link DimensionLookupMeta#actualizeWithInjectedValues()}. + */ + public class ReturnTypeLoadSaveValidator implements FieldLoadSaveValidator { + @Override + public int[] getTestObject() { + final DimensionLookupMeta meta = holdTestingMeta.get(); + return meta.isUpdate() ? new int[meta.getFieldUpdate().length] : meta.getFieldUpdate(); + } - public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator fieldValidator ) { - this( fieldValidator, null ); + @Override + public boolean validateTestObject( int[] testObject, Object actual ) { + return Arrays.equals( testObject, (int[]) actual ); } + } - public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator fieldValidator, Integer elements ) { - super( fieldValidator, elements ); + public static class UpdateLoadSaveValidator implements FieldLoadSaveValidator { + boolean value; + @Override + public Boolean getTestObject() { + return value; } @Override - public int[] getTestObject() { - DimensionLookupMeta dlm = holdTestingMeta.get(); - int[] testObject = super.getTestObject(); - if ( !dlm.isUpdate() ) { - dlm.setReturnType( testObject ); - } - return testObject; + public boolean validateTestObject( Boolean testObject, Object actual ) { + return testObject.equals( actual ); } } - @Test - public void testPDI16559() throws Exception { + public void testPDI16559() { DimensionLookupMeta dimensionLookup = new DimensionLookupMeta(); dimensionLookup.setKeyStream( new String[] { "test_field" } ); dimensionLookup.setKeyLookup( new String[] {} ); From 7c48d00623ae200aef5df01c006996d058dcf984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Proch=C3=A1zka?= Date: Thu, 26 Jan 2023 23:12:48 +0100 Subject: [PATCH 2/2] [PDI-19718] - Fixed lookup field data type --- .../dimensionlookup/DimensionLookupMeta.java | 39 ++++++++++++------- .../DimensionLookupDialog.java | 18 ++++----- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMeta.java b/engine/src/main/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMeta.java index 1ebdd9719d43..3e6c8960287b 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMeta.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -22,17 +22,9 @@ package org.pentaho.di.trans.steps.dimensionlookup; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.List; - import org.pentaho.di.core.CheckResult; import org.pentaho.di.core.CheckResultInterface; import org.pentaho.di.core.Const; -import org.pentaho.di.core.injection.AfterInjection; -import org.pentaho.di.core.util.Utils; import org.pentaho.di.core.ProvidesModelerMeta; import org.pentaho.di.core.SQLStatement; import org.pentaho.di.core.database.Database; @@ -42,6 +34,7 @@ import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleValueException; import org.pentaho.di.core.exception.KettleXMLException; +import org.pentaho.di.core.injection.AfterInjection; import org.pentaho.di.core.injection.Injection; import org.pentaho.di.core.injection.InjectionSupported; import org.pentaho.di.core.injection.InjectionTypeConverter; @@ -52,6 +45,7 @@ import org.pentaho.di.core.row.value.ValueMetaDate; import org.pentaho.di.core.row.value.ValueMetaFactory; import org.pentaho.di.core.row.value.ValueMetaInteger; +import org.pentaho.di.core.util.Utils; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.xml.XMLHandler; import org.pentaho.di.i18n.BaseMessages; @@ -65,6 +59,12 @@ import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + /** * @author Matt * @since on 14-may-2003 @@ -430,6 +430,20 @@ public void setFieldUpdate( int[] fieldUpdate ) { this.fieldUpdate = fieldUpdate; } + /** + * @param position + * Index to array of property fieldUpdate. + * @param desc + * Update type code or value meta type name. + * + * @see #getUpdateType(boolean, String) + */ + public void setFieldUpdateValue(int position, String desc) { + fieldUpdate[position] = getUpdateType( update, desc ); + if ( !update ) { + returnType[position] = fieldUpdate[position]; + } + } /** * @return Returns the returnType. */ @@ -937,7 +951,7 @@ private void readData( Node stepnode, List data fieldStream[i] = XMLHandler.getTagValue( fnode, "name" ); fieldLookup[i] = XMLHandler.getTagValue( fnode, "lookup" ); upd = XMLHandler.getTagValue( fnode, "update" ); - fieldUpdate[i] = getUpdateType( update, upd ); + setFieldUpdateValue( i, upd ); } if ( update ) { @@ -998,10 +1012,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis for ( int i = 0; i < nrfields; i++ ) { fieldStream[i] = rep.getStepAttributeString( id_step, i, "field_name" ); fieldLookup[i] = rep.getStepAttributeString( id_step, i, "field_lookup" ); - fieldUpdate[i] = getUpdateType( update, rep.getStepAttributeString( id_step, i, "field_update" ) ); - if ( !update ) { - returnType[i] = fieldUpdate[i]; - } + setFieldUpdateValue( i, rep.getStepAttributeString( id_step, i, "field_update" ) ); } keyField = rep.getStepAttributeString( id_step, "return_name" ); diff --git a/ui/src/main/java/org/pentaho/di/ui/trans/steps/dimensionlookup/DimensionLookupDialog.java b/ui/src/main/java/org/pentaho/di/ui/trans/steps/dimensionlookup/DimensionLookupDialog.java index 98254908a70c..e26a54c1652a 100644 --- a/ui/src/main/java/org/pentaho/di/ui/trans/steps/dimensionlookup/DimensionLookupDialog.java +++ b/ui/src/main/java/org/pentaho/di/ui/trans/steps/dimensionlookup/DimensionLookupDialog.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -22,12 +22,6 @@ package org.pentaho.di.ui.trans.steps.dimensionlookup; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import org.eclipse.swt.SWT; import org.eclipse.swt.custom.CCombo; import org.eclipse.swt.custom.CTabFolder; @@ -63,7 +57,6 @@ import org.eclipse.swt.widgets.TableItem; import org.eclipse.swt.widgets.Text; import org.pentaho.di.core.Const; -import org.pentaho.di.core.util.Utils; import org.pentaho.di.core.Props; import org.pentaho.di.core.SQLStatement; import org.pentaho.di.core.database.Database; @@ -72,6 +65,7 @@ import org.pentaho.di.core.plugins.PluginInterface; import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.util.Utils; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.BaseStepMeta; @@ -89,6 +83,12 @@ import org.pentaho.di.ui.trans.step.TableItemInsertListener; import org.pentaho.di.ui.util.HelpUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * Dialog for the Dimension Lookup/Update step. */ @@ -1454,7 +1454,7 @@ private void getInfo( DimensionLookupMeta in ) { TableItem item = wUpIns.getNonEmpty( i ); in.getFieldLookup()[i] = item.getText( 1 ); in.getFieldStream()[i] = item.getText( 2 ); - in.getFieldUpdate()[i] = DimensionLookupMeta.getUpdateType( in.isUpdate(), item.getText( 3 ) ); + in.setFieldUpdateValue( i, item.getText( 3 ) ); } in.setSchemaName( wSchema.getText() );