Skip to content

Commit

Permalink
branch-3.0: [enhance](regression-test) fix hudi incremental query bug…
Browse files Browse the repository at this point in the history
… and add hudi p2 cases #44003 (#44457)

Cherry-picked from #44003

Co-authored-by: Socrates <[email protected]>
  • Loading branch information
github-actions[bot] and suxiaogang223 authored Nov 29, 2024
1 parent 2891514 commit 16f36aa
Show file tree
Hide file tree
Showing 13 changed files with 929 additions and 197 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
if (!metaClient.getTableConfig().populateMetaFields()) {
throw new HoodieException("Incremental queries are not supported when meta fields are disabled");
}
HoodieInstant lastInstant = commitTimeline.lastInstant().get();

String startInstantTime = optParams.get("hoodie.datasource.read.begin.instanttime");
if (startInstantTime == null) {
throw new HoodieException("Specify the begin instant time to pull from using "
Expand All @@ -89,16 +89,18 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
startInstantTime = "000";
}
String endInstantTime = optParams.getOrDefault("hoodie.datasource.read.end.instanttime",
lastInstant.getTimestamp());
hollowCommitHandling == HollowCommitHandling.USE_TRANSITION_TIME
? commitTimeline.lastInstant().get().getStateTransitionTime()
: commitTimeline.lastInstant().get().getTimestamp());
startInstantArchived = commitTimeline.isBeforeTimelineStarts(startInstantTime);
endInstantArchived = commitTimeline.isBeforeTimelineStarts(endInstantTime);

HoodieTimeline commitsTimelineToReturn;
if (hollowCommitHandling == HollowCommitHandling.USE_TRANSITION_TIME) {
commitsTimelineToReturn = commitTimeline.findInstantsInRangeByStateTransitionTime(startInstantTime,
lastInstant.getStateTransitionTime());
endInstantTime);
} else {
commitsTimelineToReturn = commitTimeline.findInstantsInRange(startInstantTime, lastInstant.getTimestamp());
commitsTimelineToReturn = commitTimeline.findInstantsInRange(startInstantTime, endInstantTime);
}
List<HoodieInstant> commitsToReturn = commitsTimelineToReturn.getInstants();

Expand Down
3 changes: 3 additions & 0 deletions regression-test/conf/regression-conf.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,9 @@ extEsPort = 9200
extEsUser = "*******"
extEsPassword = "***********"

enableExternalHudiTest = false
hudiEmrCatalog = "***********"

enableObjStorageTest=false
enableMaxComputeTest=false
aliYunAk="***********"
Expand Down
349 changes: 349 additions & 0 deletions regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,349 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

Loading

0 comments on commit 16f36aa

Please sign in to comment.