From d85b144bba418a4f466f6325e5594f27ff14d8d8 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 22 Nov 2024 17:48:21 +0800 Subject: [PATCH] test new hudi scanner --- .../hudi/test_hudi_incremental.out | 348 ++++++++++++++++++ .../hudi/test_hudi_schema_evolution.out | 32 ++ .../hudi/test_hudi_snapshot.out | Bin 348526 -> 697561 bytes .../hudi/test_hudi_incremental.groovy | 13 +- .../hudi/test_hudi_schema_evolution.groovy | 13 + .../hudi/test_hudi_snapshot.groovy | 11 +- .../hudi/test_hudi_timetravel.groovy | 7 +- 7 files changed, 416 insertions(+), 8 deletions(-) diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out index b1bdad85013bfce..852aebe48961b5c 100644 --- a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out +++ b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out @@ -347,3 +347,351 @@ -- !incremental_9_10 -- 1000 +-- !incremental_1_end -- +9000 + +-- !incremental_earliest_1 -- +1000 + +-- !incremental_2_end -- +8000 + +-- !incremental_earliest_2 -- +2000 + +-- !incremental_1_2 -- +1000 + +-- !incremental_3_end -- +7000 + +-- !incremental_earliest_3 -- +3000 + +-- !incremental_2_3 -- +1000 + +-- !incremental_4_end -- +6000 + +-- !incremental_earliest_4 -- +4000 + +-- !incremental_3_4 -- +1000 + +-- !incremental_5_end -- +5000 + +-- !incremental_earliest_5 -- +5000 + +-- !incremental_4_5 -- +1000 + +-- !incremental_6_end -- +4000 + +-- !incremental_earliest_6 -- +6000 + +-- !incremental_5_6 -- +1000 + +-- !incremental_7_end -- +3000 + +-- !incremental_earliest_7 -- +7000 + +-- !incremental_6_7 -- +1000 + +-- !incremental_8_end -- +2000 + +-- !incremental_earliest_8 -- +8000 + +-- !incremental_7_8 -- +1000 + +-- !incremental_9_end -- +1000 + +-- !incremental_earliest_9 -- +9000 + +-- !incremental_8_9 -- +1000 + +-- !incremental_10_end -- +0 + +-- !incremental_earliest_10 -- +10000 + +-- !incremental_9_10 -- +1000 + +-- !incremental_1_end -- +9000 + +-- !incremental_earliest_1 -- +1000 + +-- !incremental_2_end -- +8000 + +-- !incremental_earliest_2 -- +2000 + +-- !incremental_1_2 -- +1000 + +-- !incremental_3_end -- +7000 + +-- !incremental_earliest_3 -- +3000 + +-- !incremental_2_3 -- +1000 + +-- !incremental_4_end -- +6000 + +-- !incremental_earliest_4 -- +4000 + +-- !incremental_3_4 -- +1000 + +-- !incremental_5_end -- +5000 + +-- !incremental_earliest_5 -- +5000 + +-- !incremental_4_5 -- +1000 + +-- !incremental_6_end -- +4000 + +-- !incremental_earliest_6 -- +6000 + +-- !incremental_5_6 -- +1000 + +-- !incremental_7_end -- +3000 + +-- !incremental_earliest_7 -- +7000 + +-- !incremental_6_7 -- +1000 + +-- !incremental_8_end -- +2000 + +-- !incremental_earliest_8 -- +8000 + +-- !incremental_7_8 -- +1000 + +-- !incremental_9_end -- +1000 + +-- !incremental_earliest_9 -- +9000 + +-- !incremental_8_9 -- +1000 + +-- !incremental_10_end -- +0 + +-- !incremental_earliest_10 -- +10000 + +-- !incremental_9_10 -- +1000 + +-- !incremental_1_end -- +9000 + +-- !incremental_earliest_1 -- +1000 + +-- !incremental_2_end -- +8000 + +-- !incremental_earliest_2 -- +2000 + +-- !incremental_1_2 -- +1000 + +-- !incremental_3_end -- +7000 + +-- !incremental_earliest_3 -- +3000 + +-- !incremental_2_3 -- +1000 + +-- !incremental_4_end -- +6000 + +-- !incremental_earliest_4 -- +4000 + +-- !incremental_3_4 -- +1000 + +-- !incremental_5_end -- +5000 + +-- !incremental_earliest_5 -- +5000 + +-- !incremental_4_5 -- +1000 + +-- !incremental_6_end -- +4000 + +-- !incremental_earliest_6 -- +6000 + +-- !incremental_5_6 -- +1000 + +-- !incremental_7_end -- +3000 + +-- !incremental_earliest_7 -- +7000 + +-- !incremental_6_7 -- +1000 + +-- !incremental_8_end -- +2000 + +-- !incremental_earliest_8 -- +8000 + +-- !incremental_7_8 -- +1000 + +-- !incremental_9_end -- +1000 + +-- !incremental_earliest_9 -- +9000 + +-- !incremental_8_9 -- +1000 + +-- !incremental_10_end -- +0 + +-- !incremental_earliest_10 -- +10000 + +-- !incremental_9_10 -- +1000 + +-- !incremental_1_end -- +9000 + +-- !incremental_earliest_1 -- +1000 + +-- !incremental_2_end -- +8000 + +-- !incremental_earliest_2 -- +2000 + +-- !incremental_1_2 -- +1000 + +-- !incremental_3_end -- +7000 + +-- !incremental_earliest_3 -- +3000 + +-- !incremental_2_3 -- +1000 + +-- !incremental_4_end -- +6000 + +-- !incremental_earliest_4 -- +4000 + +-- !incremental_3_4 -- +1000 + +-- !incremental_5_end -- +5000 + +-- !incremental_earliest_5 -- +5000 + +-- !incremental_4_5 -- +1000 + +-- !incremental_6_end -- +4000 + +-- !incremental_earliest_6 -- +6000 + +-- !incremental_5_6 -- +1000 + +-- !incremental_7_end -- +3000 + +-- !incremental_earliest_7 -- +7000 + +-- !incremental_6_7 -- +1000 + +-- !incremental_8_end -- +2000 + +-- !incremental_earliest_8 -- +8000 + +-- !incremental_7_8 -- +1000 + +-- !incremental_9_end -- +1000 + +-- !incremental_earliest_9 -- +9000 + +-- !incremental_8_9 -- +1000 + +-- !incremental_10_end -- +0 + +-- !incremental_earliest_10 -- +10000 + +-- !incremental_9_10 -- +1000 + diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out index 12dd0cf086d3f04..da7273d4c14ef95 100644 --- a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out +++ b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out @@ -31,3 +31,35 @@ 20241118012149007 20241118012149007_0_4 5 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 5 Eva {"age":31.5, "address":"Chengdu"} 20241118012149007 20241118012149007_0_5 6 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 6 Frank {"age":29.2, "address":"Wuhan"} +-- !adding_simple_columns_table -- +20241118012126237 20241118012126237_0_1 1 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 1 Alice \N +20241118012126237 20241118012126237_0_0 2 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 2 Bob \N +20241118012126237 20241118012126237_0_2 3 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 3 Cathy \N +20241118012132306 20241118012132306_0_3 4 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 4 David 25 +20241118012132306 20241118012132306_0_4 5 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 5 Eva 30 +20241118012132306 20241118012132306_0_5 6 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 6 Frank 28 + +-- !altering_simple_columns_table -- +20241118012136512 20241118012136512_0_0 1 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 1 Alice 25.0 +20241118012136512 20241118012136512_0_2 2 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 2 Bob 30.0 +20241118012136512 20241118012136512_0_1 3 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 3 Cathy 28.0 +20241118012138287 20241118012138287_0_3 4 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 4 David 26.0 +20241118012138287 20241118012138287_0_4 5 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 5 Eva 31.5 +20241118012138287 20241118012138287_0_5 6 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 6 Frank 29.2 + +-- !adding_complex_columns_table -- +20241118012144831 20241118012144831_0_1 1 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 1 Alice {"age":25, "address":"Guangzhou", "email":null} +20241118012144831 20241118012144831_0_0 2 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 2 Bob {"age":30, "address":"Shanghai", "email":null} +20241118012144831 20241118012144831_0_2 3 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 3 Cathy {"age":28, "address":"Beijing", "email":null} +20241118012146150 20241118012146150_0_3 4 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 4 David {"age":25, "address":"Shenzhen", "email":"david@example.com"} +20241118012146150 20241118012146150_0_4 5 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 5 Eva {"age":30, "address":"Chengdu", "email":"eva@example.com"} +20241118012146150 20241118012146150_0_5 6 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 6 Frank {"age":28, "address":"Wuhan", "email":"frank@example.com"} + +-- !altering_complex_columns_table -- +20241118012147879 20241118012147879_0_0 1 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 1 Alice {"age":25, "address":"Guangzhou"} +20241118012147879 20241118012147879_0_2 2 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 2 Bob {"age":30, "address":"Shanghai"} +20241118012147879 20241118012147879_0_1 3 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 3 Cathy {"age":28, "address":"Beijing"} +20241118012149007 20241118012149007_0_3 4 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 4 David {"age":26, "address":"Shenzhen"} +20241118012149007 20241118012149007_0_4 5 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 5 Eva {"age":31.5, "address":"Chengdu"} +20241118012149007 20241118012149007_0_5 6 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 6 Frank {"age":29.2, "address":"Wuhan"} + diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out index efad67ffbfa8c407fa42d0dd28f569f492bd1d12..41f34255f40a5249cc790acd1c805f7917621a7b 100644 GIT binary patch delta 37933 zcmeHQdyL%1c?Xv~%WI{IrPJ$l5AE&Adf8mfki+5dEJsjfyS4m~PnKWR#g`#DyjyE; zcg5Y)6U5kkr<0VxVb_A}tcYuL42Q zCI$MDTq!Flh1WJDrSbrrA(zKn4u{|T=9}MskGKC|#rMCn`SCx~CYIc0A>?=t$y>-G z7-NE+&Y3ARMVU+T93jsR{Lpl9z)j-1p6S`PXEM&P?ReP76t%To_q2MrE>CRAJTFgB zdOs6m6O%m%joa=x@=DCM9m~o$<2jKhC5H0j+nje!9M85}?t3&lD0NOf`#age&I9Sb zm0Y*J@Aj?EKl}w8ZGf^23MxP~4rR*(RIq&#N_KZdO^fFD7R6`l6<)3uKb`;0qAzC3 z)v!2ToS!LIMDeCVvC*81A~?~Qi$(Fb;F6%+T!7EF*BVU`*K2VzyGlNA#3&ey5iEJ6 zjf|Oul!%KNVeloDT6MbIoC`&9+{Dx^Q3pX!$Z!9VG2nW{atIDg>XFEV{try%1j3B` zhy^S{#KJ7OL%Attr>cAwzM@!5U&8RV)WAa~7>I9Ry77N7Sr?gfb|?5xdH<$HJK zR^~j?-hW_Jo)G$o@Iwo^u5XIS@=f9}W^$Kf)AoFu3mBP}mA=((NtqQq6QS2-9{}e&-}y~k%zWwTjZzi(DSQy#6s*#uI+T!Jz~CaH;?7fZ)vM_;^Fr8 zm&fGeKhc-AcMJ}1pq7&lk~w-E z>!92XWio#o)X9ag?MP6gVQn7GK2F_qEgWouBA~1t_OsI?1rrV)gVJZ9>=e}ebJ<}B zXlxU@WpA-sOGgT?rXv;20_Zf)0#CCxUrkU(91zuRnE4GJ%IADt5<#YiIwo4QdM;?CNSo4VQMiKX2Kst*#CJ93fg3JOl;EwNM#v(ho zNPMG8trn+mylY6$Rm5sK zf;M0<*K#Fak}I~4{LhdOt`7nL^9Zvo6Cf}&S>#cZxj}?o581*?CA3?Zc_S0Vu4Xl7 z;@W(2!}(fV3A;tvwQvc`!#j;Nnl1O^9D3%j3}ciR?whFrdUmz(HV)(ezNVAK z`Hxl^aDO84ZF(UYcf;#^At(YIG=tE`07}d^JxVAbITt>LVd(^@jlQ;FQ!_5tlW&g2 z*?ECiW?`P_ZkI)Ii6+Rjs(t!1*T_S+=1TdUJlH=cD&?wNnm3jkzs@Tax&K{bm5yvn zUQcoxW!=1sJD2Rs`}q*5!m`yJiK#Bb_n(0J11Jlw`3nzPgZR> zS`Q^)P$goxf)c0~&(%lv!ak9`3%2bKCBpdqFFyL%JnW6^ui?3il8NW0&eXRo5%KyR z6Whi*^AI+Xo!?6VpMo05*uhMwqH<*xWGbpvDuAV3D(-`zDVF(cqgDl>T98l0xs8VF zQY`T{ZDiE8N|Yc*?F(eep(UGH)B(NgW__R)f=cOzzR7HWO+sv8GAu-*3R+mWcIfT{6(t&buQDGS0KM(DdG7Mt^(BYeO473*|Ydlp_(EG(ym1ACxW61(+of@odHe zK^RR}JUv{Ow_v)au)NAvSm}xfW{T+br&JYcj;>j80LEf3rKbm$iY&;azP2$rVZQ`MImU*HZvVd&_p>Ursu$G z4lzg~yo|uMXmXG4IW$akv!u~ove=yAO{p<`r4AE%qB9S)k9_YM`5*g1wSTUqEm{qF zeYe`2Y8m(NY9Qa7(w6J+)_;GruORDR(S{lEBM#G{fYAwBGx2C>a>sT}${g2riN_;K zE^L!1H)rIT-!}SJ*5>*7z)At`sXX(9zDD!p#H5Z#`TV|eHN(^NIkFY(eXsCxPpt-c*v0pXq*1xb4>H-h$rw&_I?EYK{+aUy(Sd&F$LQdF%a^M1JefwPCsIuz}?deyA<#djQ0^(LVE!n{~^vrS?O8?Dw{< zfl0)1e2ft^CDCkbgiX%^9oDizgu>AEUH2l*bruw;wCB4dsLtzIJDNZ_?CWj`B_&Yb z6U~*%{p&RJiC^RKL0;HVj)RJjhu8KkmtR=Z*KdGu(6dssNS=IM|CD5ha_HE~E!ydI z@~@ZVZqSIFu>01723$K~tk*1gW+0b-8ipN~Cx51|e)8lV?byjZxqn?M4?LDzbt546 z6hKlhw2wUeu+}f%KBMhv^>Xq9#XfN2b|Z~6!`lZ9>wD_+;~feu7dm@tz<0djy+$#>?4 zTklIVT}t{W(c?iix?=RvEH5egNYO`E6@9c!KJu0}a3c)yDQxvbA<3h94i-x^FMQ)eBQ~2^pwNb-8j3)ctn? ziYW%_BQQ`zzBZB@*lc4qMS3Ek~`X(Jcbv5 zYpHzrhQ5IhZU}YFK|>irDMKjb1(!J~ek@s^W%8}RF$O-QArwxi=MY>C8MrQx2!na6 z=Wvr*t{-8C`VM6sLnsE$T(-wtPcGeK47J|h;qH=JLS-flSI!b@i*lq>j&xU-Bi%At z`fuaISwb-sfOjVtK~c*CQ+WUy3hz^gm8gvwyV1Mis8lA$c2lqJ+T z$EgqJi$}8A??YHZaZEx76R?DGBAENY{K;hCTMK^Ej1W#xNQl+3gi3^#pv1#D6>vxTTO&{103W4Jj2Tvm~go3CO1k)t`gyn#*k8ei|hZZ;ib_}5yID~@( z4sqpUF9UcgL#Qk24X6yElrx}m2JE<$riM`8>xJ}6T>#sEa{us)rw&XR@~idSb>Ig* zd}wvwvDJN=msjzjRD7tSiVvmYLuKK_RD39KmQnGcRD38EA40+{#fMVyp)LpuuHr+TjSjBjL#g24o!IRkOS32yA1Vn7d|AVbtN2hVy!h1? zUR=e8y8IE!ReY$Qhm-!WR#7TG)MbuwuHr-e!Wic&KGfw7lCI)IsTk?qq2yJJ^eY)7 zUB!n=LKbHU^Hj+6G;fW{mzU-VRQd8Ur^U;cFRx#!_)scip30c_anG2i;zNDVoOz0o z`sj?5iVp>f@CV77rx>OGgHclPp+J(dt;fW9iW5?tkm7`XS@Pzo_)tmmVpM=C$(g)m zBEzyz!mg7w7LtckNct&03%P#F)ie*WFCb@OlD~5v@_4414I9n5C`uA~&c)~D^Sm8$ zb&9yI458{5rQwdtkovSMHzoxCX4)4wzY4`$BSUMa#QYQZJ-PO$u`P_}-mEzUtayQ6R9 zQ2W`>FKRC%zmT_5@u9M1UMt(J&UU-^!T!~6J_NOyooQw}yL9R~IGTboot-&XE7I$H caSm(q${A2O1O9S5115%0J;aC7@paezFRByvL;wH) delta 18799 zcmeHvYj7Obb*ASD3}s0q0g`wS;NZch)WdtbZ@(Z@B1F>EgOmu8Hl>gfb-VjEFydeu zni-ISWgDWrij!T%7Cp9Z?QU&K%+{9kV!Gk0#^bIy0Z^PPF?>)YP`oi$JYL2BX_`P)zD z2ewe^6oh3m7I>a!FyGf0aj53nRA__=W)W%#R~OCkmHX$Hb78@yHZ#luHl659qf{gM zVPcF?cZ`yf>AD6n3ys>WD6o?27W&_L{Z+cx%D?#~oM7w2ejAd^~Ufcg{Ox7 zXnI=I{4%c$j}6ynCnp8|fGYz&J2}-5HN09Ayi%D{FV0kWHGaVt)!xSN$JGBXH|Frt zu4uN_sEWCIIS}&Pe@qXjDU(B=&KHYj$Dyb$)I0=6|=khX>Q;YPsI1@kTkSs*%;p z(=(MhIX;u>%GjvVYQr-Dk-xu||y!u?OyC-`b(!@kCa*k{CmnmaS1XH(eB%B>xBT>-*<9v6J}rmxE4nkx zV)DWp+0o|8gZ(3+>(J0NHJ6&M#tg^Pcwn0vu>$7lI&(P<<;&m7Z!Ff;Tvp@zGmsh`(T9?M{JG&`8{K6yI zfxu^$O>EepZs5Se(AOLygywk;WlliwC4;4-s0e)C(lMNhYA4idUL}fxWmgbUa7a;M$0+ixc=H1P9DzUaO<^K9=>-3=hG)zFSK0--kiZ@)I~gD0cc*!Nq&bF-*mj2e%{eyb~^&RTPUC~KVPu$K!d|EBM%gtXw38A5e(l7T(Wc=^%Or2OnKqiPO|O@CU&m);WAGQZh!fNq|>>cjLs~%>Xy`y{L0xCgMndE z$8rKq*9{LNV;Gq)a0bVTs02$1+0v1fr}<>L8oPf#VULr_1TaFs!t3?q@N9YJ?^6Tv z`5h~=vg?bv?Sth>{Qr9KK!lZ%oWEUDvo(K8uKM5U-ZazY^FQhsmM=e@8aPT2?;OEG z*egOPzv?I@o*AD^rOlmD|K|cIq^^Za%M$3E5G_|XX*X- z@p@Fnjh&v3s`BiC&aUk2WQEJied%>+c)pchYU2x9M^^su8|k%X=)wp+>TAB?I~WY{ zV4Td<1o3R*=uSujZ|US!<%Qo&cMVp=NfE~l*#A^=vJ68nG0AtN`{X-8iuN6;sfAcP zG}F5M?J%S4LB9A*dTkbwo=7^99hN)0GOL=Ox?^CAWwC3VAmvHhV_8r)_zVrJ0t9uWv`Y6u->Qgv;2s^e2`;!(9R(_*8#olwQ@L@c(wn03E zE)E~TzUt%aaGd4p@NpbE?!*DxH(qW%-lQ&UZI925+c@jer?>1>kJumT#o=Sv-}?P) zaEB=0RTI7_pG4>b8krW;UW{2h5hT4*9PIDsHBt3u5tFd?>kX_nII#U(^6HA79J5^J z$g6*o-%wPzj8V8LT$VsA)YeNBo>!NbU(5H-+r;${n@!CNT|{Z&*qQ^_p!uH1S>XG` z<=1(N#@x(x#&~K9138I7E0+;_Q=9;DEn5B-h3BOB0~Qw|eO(-vZ@-+nS^jusiuR9J z%E}auh(-7Y&_dMZnZM1h%@UWg=4o?87Hg@#5uE|WxPoXl2@Ti+)*AP14tU|gzX*T~ zOFnTMa1>&JJpDg&U85&NJY`{21FBKOEH|bw;H1lSo8;XM@^79?ZSOxg$E(E;18e~e zs;d)*&^)lVKjQ+)o%?)ts9F5!z^D$tYB4wnT^R88oyn4`Yrcr z7?!;B?T$O$AOQT-74CFw+)8dcn#&PQclodE&X;jX3OHrbkRkaGlqu&C$9na6T2EEuk1+E&4$ZC3yYWgVvW7@=djQ6i>c zP&<)%94o-I?zay%-g*uv@58=bnQ_b1C_&l+Q;Wies17%;obBGO?gbJJ_0X8DRb%S| z@Cst;Wqi&lOVL2%-sj-erqStWx|O0P3H z4XCu7*H|RRn9&TpxhhH!1TBH$)1dtM^K-q7&tYKP9SiVq2B_ z5sB61B3l~Qv%T`MyRv!tiGS=Y4eaMNm_ys#5+i*;oGR{?XLFrE5w84ECcbz%pewcpnP4pttY zxBSoyGMBPnp5d(&XeO$c8w-&js;UfZ_|#OnB9f6{KcA~BWUA(BNesxi4wH0EX2S{% zt{(QGy9n5Cc6Q|m;;I3(*wH&`c`mR$^EAisRdPl!aVA58Xot2fC=r(7$%&_P-7y$T zF1wjp6wLYy39Bb6Ww`ytx0lGv)14|6IsIoTGPo;>W<;%cU;)9k{MQ3O;8Vrjyg9MI zJA-g9FI~-#O7ewFpC<@1G!h6+1c*kA0^m5raW%`)eV@6mK#*Fx0&8NLm*W&VIaYtR z0;^s$I+Mak72N4?WYyA3TX&+W_@xCaKt5$b(Wf&44H{0+k-Ru2{Ny7CQ|BK!nEqd@ zr1?APb=wT$9(N16L6M!fzJ`<apraSGHcl z@uAfhfAa%)T&q%LY{v>?t&>~uFoAub^>jNPyuP*<0*7b83suTt?5C%ZTTZuqH#|xV zdEGcc`PN{2YMgbzeDye~ z#+zKdu&7c@JzqR?)$jjflKV8~zZ|xKaKb0WH^Uw;iv2E_cZJLvpwjg(B zG2Z)fX2%E@p~(X*QA?<4b`fK_W8$f9_#PrI>`=U@|F2sIg{_q%Uz#OmGTFG zq{=h$(FfCO3k$BRAy1b(K(_#d%ykaEa3s|&*N>!o=P7XzQB2d+E#y$hixrQ}Ra^_e zrMn0)0H2obxS~)mPgZAVjt5ABceF|*$Ti%80UP%%d@I5o5H=n`=%k3X)IwwgZ9;*y z;0XXNTM-7Fzj0jse`FC%{_EFLT`-8B|LKgFz%>Q)J))J^syw3uxauXHA>pu+q2?ws z39Gj&IH*$1(eL~Qj_lYm9I?G`KMo(nzA-d*V`2wP!D1+Xp0uA9R6$oG_HNZytOGFWy@1-@W{0LFn5Yd36Z+OT(QRDP!K=Hg_k)L zV`AK>Vi~Mg07fKmcSrBxqFLPf<}s>>!t6>cX^ zYZIu^F=VkPJ&>YuJeMc=7B?<0{3O>U_gze{&$uo~#t$l^eYm54)aJzSthnl;BlmE@ zOFF*mXs}m6GFLZkVPC8Mjx2!esi@In=6w*mi;Y3@lbJrb@*hBx>?)*7&F8me*Dfsp zAB-@Ajkt!9bsdnBd?thOY)|(ccZDrN1{R0oqfvE)GeiBrN6x47(BRG+2+#Y&?!)tplV=n7%M|`RaqlA2wgLEZd#U$auJwQ{(4~z?644q#Pb8+ zLs<*B5cwJE3zTpT(;F}(3kZS;C&v|6pz<|vZg9cr_4CX5kb*-%fSB*ussS0oVV#aK z$Q&qE9!OP^wAIed`}$ARU=$0+4eacwNCqgNRut-H^;w7#hHu{=b2QbU%>bMN?~KJC z8wXhb2-V3kplTnQ1NHwBl7-gNNEpq7m3s7~Wx+WzYVN>E>nJn2zH5141fR@p%z%rM&%Zz46Z6{_ zIsxR2<`Z{$_iVp7<274;jU05cH9t^Aciond#=smPh~!c&j3X|us!f zx70eH%6{_Jm(u5NeJSG?48LFjUpcrY>{4ZCG)aXe@$QxoL zWn(wQNWXGor2qD#%%&|4xYOl`j3m8~-hV@mbVH7GLyoj4*ZEbHBX!Alb{6`Vtw(CP zj^!wqeu0W4)N+2Qilm2My`e}_IOB#Q>HjN5(s$BZw=7qU)P@kt&@J7NBeewbH{?i3 zIgRyygfzafBqiMe)8Rp=jC(iwU4T&-~V36J?B9vIV1C+wL-M+1P}xns^sK?*mF^2 zMR9=m;BX<2P>ht}1eSxI3;a>ZdrZ{=hkg_-44`bJij?;RF0-J@AO-}gffF;(cwZiJ z=pSL7DD?V}Ifb}Px2?b?0oDB7t{?aC;Ef3+zg%@P7)q*QtLr?26QT!DP($mA!9gI# zxeu*~ZJ`y9TV@z*q0T(I?)`cUxgmD1v+uT`B~`Y!oZ9`j<4$>NdzhUG)Z70Tjsw z*9C{L1Zrt;PD?suMv%1P-4pD%3;B^w zRbL89Nh2Xg>xsvq$$>5&Vo02EloK?a3R|~OKy*S$F{vvo=dMiU=SbxVS= zR~6^H{DX(kT(Y+Ybw+U)R9(=(pZi(1TOQn#Tb(g5J>zzINM1S6(IF4NQP_->={SlT zKfm`&UV5!#tp)WFsznj1{b4W{N-o~c=_;#}`G>KunJu{jGVhbYaI5Dx`w&Pn1ca;aBLu?DP z)SI`0=h(37@wkUi&$N^RHl?;4>Pc-F!fYdkKrgx^b_2wrn+7N^7Hf@$0k^n?YNsU- zahQOJkwbH%g$O;6svA%VQ2E^M)aKh&K^`^s7-_uMFYy1+3Zq}DnYz;ZnSaPPe|n)b zI$ocGqzql>t%jDi4ylzZ?Bc42tc=!|ujTsF=%%@FDKpfZnI4$;^$`CcUAJAVMCgia zz)JL5K+}Lm1``dq_H~Wtkb%adODZn}w6wLTs?@8k+fa|DMWc+axb{t}HqZ)~fC1>b zN7sk!rs%NQIowG{R3}i>P%WJp=wwo&(fSVK)k%3}f2uocyU-z@{>P3TBT$B-?J5Kg zLa#8Qy8_Xk50Ro75J6#+fz!OSrz@+Jz;b9szN>3-vx3Xq63}Is?Q0HPp&Raj-dgpY z*Nb~A=)6*kt(9;ci^a_c zK>TRpNBtTHgUjMSr@8^`m5wk#x!i-+QVT*3#fvZ?Yo+iSP+XG`_y~9kCLQZ7+o;5l zPwZN;e*Wiwj01fo{$WUVEVIP>Kp|otreuuKF@uySl%Uj1Dnyi!A6pvxhd%cG-@8j$ z@X40>Eyy0RdZz25a(JPyd*Syi>XBa-@lR4jGuL`;D(C=I0{lfkS#jSiZ>g=5Hg#{n zry@CYv2zVX2})EoOrZvWjv9*&v!6i1fbq0sxv;a=FIc#S0aY_JH4NQeShg0{fQ37c zrVfL18+vD(HZQK-JTO_V#j3B#T6C(hsG}H=xg`OH_+%Qo!nePI7Qss$neGf)SLMVl z`864I2FK0npSi2g;S}P6KnZs&^@{`O8iu|W_e@DQEbjS`j9+Ics<~fvn|1faeQos! zKcA5_0n5hnI~U~ou&f|ME*?bFv+AHnf2oowC05{GB{mJ@)qhT}P3w+4^KNRRym%#_ zO+6yZH|I8PwG@UepdUQs+)(y_yZ{0XsFZ>bDWmE}ff@qc0rJf4*=~p*4GPsSEQv#a z2(=4RF@lB}j&uW4>V)WnHZh<0ddH#3Y6mEM?#}G``DJ{HUN9K+^60n57RM(mR{xvm zh8#0li9x4`Ur$L$3VX2VHeo+j-LeA@Dl}oUS`RqBCC72xgmi6Q{Xt#+(Ze_>>ETfO zGvaTBmVmg?Ub*g>cI(=-2;k@A`Q&I|pNx~Y$oJ7Zj*w7A<9__NX*46jv*DLn(8idE z3HLw~)9w|(EoqMa^=Q_FcaalrYH$GEu#UOZ`J+b`xjKA*>Y>hFOr#rx=u_07eSsbf zLK(DD;A?-Y~tXW6c;C#un@iU?ZWxN}G{(H1P=_vo$)sv znysTH_;3xq2J-4xvzz3_uXSeS$Y^F*e)_M{{iEpUA(TV!2hlnV#()QMl-x=W(Ba|f z!Z!s3x91C8a>z@sTe(P$ODUvkzn)qzj}4`1^TFryJ-ZNWQXn^H6bvU{7WMFvg;cnIBSCS;T*etN`nuIolqze}LpFk=70&zs=4 zXUY}y{Uq$jbn!DC@`-G2ee>dHx1*B}Kk{&QEsB!kibeE?U|ks4nzYGe>iziwA1vj*JTmVD$O8>bj<9T)kpN4AJ|ASfo*Qu2)&+j31r!QeO&xB7 zjByP}uNiJJI1jWm6XwJ6C{$4Bm`B$cnn8R4UA*?ga)Nf7 4.5 ORDER BY rating DESC LIMIT 5;""" + qt_q10 """SELECT * FROM ${table_name} WHERE rating > 4.5 ORDER BY event_time DESC LIMIT 5;""" // Query all users' signup dates and limit output qt_q11 """SELECT user_id, signup_date FROM ${table_name} ORDER BY signup_date DESC LIMIT 10;""" @@ -79,9 +79,16 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu qt_q14 """SELECT * FROM ${table_name} WHERE signup_date = '2024-01-15' ORDER BY user_id LIMIT 5;""" // Query the total count of purchases for each user and limit output - qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY purchase_count DESC LIMIT 5;""" + qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY user_id LIMIT 5;""" } + sql """set force_jni_scanner=true;""" + test_hudi_snapshot_querys("user_activity_log_cow_non_partition") + test_hudi_snapshot_querys("user_activity_log_cow_partition") + test_hudi_snapshot_querys("user_activity_log_mor_non_partition") + test_hudi_snapshot_querys("user_activity_log_mor_partition") + + sql """set force_jni_scanner=false;""" test_hudi_snapshot_querys("user_activity_log_cow_non_partition") test_hudi_snapshot_querys("user_activity_log_cow_partition") test_hudi_snapshot_querys("user_activity_log_mor_non_partition") diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy index 4d458dc4381dcf7..111b3d528a42e13 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy @@ -54,7 +54,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_ "20241114152009764", "20241114152011901", ] - test_hudi_timetravel_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition) // spark-sql "select distinct _hoodie_commit_time from user_activity_log_cow_partition order by _hoodie_commit_time;" def timestamps_cow_partition = [ @@ -69,7 +68,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_ "20241114152147114", "20241114152156417", ] - test_hudi_timetravel_querys("user_activity_log_cow_partition", timestamps_cow_partition) // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_non_partition order by _hoodie_commit_time;" def timestamps_mor_non_partition = [ @@ -84,7 +82,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_ "20241114152028770", "20241114152030746", ] - test_hudi_timetravel_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition) // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_partition order by _hoodie_commit_time;" def timestamps_mor_partition = [ @@ -99,6 +96,10 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_ "20241114152323587", "20241114152334111", ] + + test_hudi_timetravel_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition) + test_hudi_timetravel_querys("user_activity_log_cow_partition", timestamps_cow_partition) + test_hudi_timetravel_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition) test_hudi_timetravel_querys("user_activity_log_mor_partition", timestamps_mor_partition) sql """drop catalog if exists ${catalog_name};"""