diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out index 82d041526b307c..12dd0cf086d3f0 100644 --- a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out +++ b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out @@ -1,53 +1,33 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !adding_simple_columns_table -- -20241118011808250 20241118011808250_0_1 1 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 1 Alice \N -20241118011808250 20241118011808250_0_0 2 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 2 Bob \N -20241118011808250 20241118011808250_0_2 3 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 3 Cathy \N -20241118011814200 20241118011814200_0_3 4 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 4 David 25 -20241118011814200 20241118011814200_0_4 5 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 5 Eva 30 -20241118011814200 20241118011814200_0_5 6 52471a9f-e8e0-46ae-b492-f090a881fbec-0_0-35-121_20241118011814200.parquet 6 Frank 28 +20241118012126237 20241118012126237_0_1 1 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 1 Alice \N +20241118012126237 20241118012126237_0_0 2 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 2 Bob \N +20241118012126237 20241118012126237_0_2 3 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 3 Cathy \N +20241118012132306 20241118012132306_0_3 4 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 4 David 25 +20241118012132306 20241118012132306_0_4 5 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 5 Eva 30 +20241118012132306 20241118012132306_0_5 6 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 6 Frank 28 -- !altering_simple_columns_table -- -20241118011819566 20241118011819566_0_0 1 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 1 Alice 25.0 -20241118011819566 20241118011819566_0_2 2 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 2 Bob 30.0 -20241118011819566 20241118011819566_0_1 3 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 3 Cathy 28.0 -20241118011821101 20241118011821101_0_3 4 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 4 David 26.0 -20241118011821101 20241118011821101_0_4 5 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 5 Eva 31.5 -20241118011821101 20241118011821101_0_5 6 d5178aff-9318-4f4d-8c1e-9d17fff92980-0_0-78-246_20241118011821101.parquet 6 Frank 29.2 - --- !deleting_simple_columns_table -- -20241118011823668 20241118011823668_0_1 1 9665ca27-ea45-49a3-b044-465db4471500-0_0-100-309_20241118011823668.parquet 1 Alice 25 -20241118011823668 20241118011823668_0_0 2 9665ca27-ea45-49a3-b044-465db4471500-0_0-100-309_20241118011823668.parquet 2 Bob 30 -20241118011823668 20241118011823668_0_2 3 9665ca27-ea45-49a3-b044-465db4471500-0_0-100-309_20241118011823668.parquet 3 Cathy 28 - --- !renaming_simple_columns_table -- -20241118011825614 20241118011825614_0_0 1 673d9ad3-dc3f-4f6a-80f8-dc72e2305dc8-0_0-122-372_20241118011825614.parquet 1 Alice -20241118011825614 20241118011825614_0_2 2 673d9ad3-dc3f-4f6a-80f8-dc72e2305dc8-0_0-122-372_20241118011825614.parquet 2 Bob -20241118011825614 20241118011825614_0_1 3 673d9ad3-dc3f-4f6a-80f8-dc72e2305dc8-0_0-122-372_20241118011825614.parquet 3 Cathy +20241118012136512 20241118012136512_0_0 1 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 1 Alice 25.0 +20241118012136512 20241118012136512_0_2 2 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 2 Bob 30.0 +20241118012136512 20241118012136512_0_1 3 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 3 Cathy 28.0 +20241118012138287 20241118012138287_0_3 4 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 4 David 26.0 +20241118012138287 20241118012138287_0_4 5 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 5 Eva 31.5 +20241118012138287 20241118012138287_0_5 6 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 6 Frank 29.2 -- !adding_complex_columns_table -- -20241118011827479 20241118011827479_0_0 1 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 1 Alice {"age":25, "address":"Guangzhou", "email":null} -20241118011827479 20241118011827479_0_2 2 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 2 Bob {"age":30, "address":"Shanghai", "email":null} -20241118011827479 20241118011827479_0_1 3 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 3 Cathy {"age":28, "address":"Beijing", "email":null} -20241118011829143 20241118011829143_0_3 4 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 4 David {"age":25, "address":"Shenzhen", "email":"david@example.com"} -20241118011829143 20241118011829143_0_4 5 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 5 Eva {"age":30, "address":"Chengdu", "email":"eva@example.com"} -20241118011829143 20241118011829143_0_5 6 1b8d6739-acf0-45c9-a1ee-e66577ccb4a6-0_0-165-497_20241118011829143.parquet 6 Frank {"age":28, "address":"Wuhan", "email":"frank@example.com"} +20241118012144831 20241118012144831_0_1 1 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 1 Alice {"age":25, "address":"Guangzhou", "email":null} +20241118012144831 20241118012144831_0_0 2 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 2 Bob {"age":30, "address":"Shanghai", "email":null} +20241118012144831 20241118012144831_0_2 3 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 3 Cathy {"age":28, "address":"Beijing", "email":null} +20241118012146150 20241118012146150_0_3 4 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 4 David {"age":25, "address":"Shenzhen", "email":"david@example.com"} +20241118012146150 20241118012146150_0_4 5 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 5 Eva {"age":30, "address":"Chengdu", "email":"eva@example.com"} +20241118012146150 20241118012146150_0_5 6 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 6 Frank {"age":28, "address":"Wuhan", "email":"frank@example.com"} -- !altering_complex_columns_table -- -20241118011830794 20241118011830794_0_0 1 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 1 Alice {"age":25, "address":"Guangzhou"} -20241118011830794 20241118011830794_0_2 2 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 2 Bob {"age":30, "address":"Shanghai"} -20241118011830794 20241118011830794_0_1 3 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 3 Cathy {"age":28, "address":"Beijing"} -20241118011831837 20241118011831837_0_3 4 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 4 David {"age":26, "address":"Shenzhen"} -20241118011831837 20241118011831837_0_4 5 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 5 Eva {"age":31.5, "address":"Chengdu"} -20241118011831837 20241118011831837_0_5 6 8a5af5f5-7601-432a-a55b-ca77669efe08-0_0-208-622_20241118011831837.parquet 6 Frank {"age":29.2, "address":"Wuhan"} - --- !deleting_complex_columns_table -- -20241118011834881 20241118011834881_0_1 1 9bdc363e-f445-4904-8001-7f5a76b32a97-0_0-230-685_20241118011834881.parquet 1 Alice {"age":25, "address":"Guangzhou"} -20241118011834881 20241118011834881_0_0 2 9bdc363e-f445-4904-8001-7f5a76b32a97-0_0-230-685_20241118011834881.parquet 2 Bob {"age":30, "address":"Shanghai"} -20241118011834881 20241118011834881_0_2 3 9bdc363e-f445-4904-8001-7f5a76b32a97-0_0-230-685_20241118011834881.parquet 3 Cathy {"age":28, "address":"Beijing"} - --- !renaming_complex_columns_table -- -20241118011836377 20241118011836377_0_1 1 58382ae1-20ac-430d-a500-933817aa23ad-0_0-252-748_20241118011836377.parquet 1 Alice {"age":25, "location":"Guangzhou"} -20241118011836377 20241118011836377_0_0 2 58382ae1-20ac-430d-a500-933817aa23ad-0_0-252-748_20241118011836377.parquet 2 Bob {"age":30, "location":"Shanghai"} -20241118011836377 20241118011836377_0_2 3 58382ae1-20ac-430d-a500-933817aa23ad-0_0-252-748_20241118011836377.parquet 3 Cathy {"age":28, "location":"Beijing"} +20241118012147879 20241118012147879_0_0 1 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 1 Alice {"age":25, "address":"Guangzhou"} +20241118012147879 20241118012147879_0_2 2 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 2 Bob {"age":30, "address":"Shanghai"} +20241118012147879 20241118012147879_0_1 3 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 3 Cathy {"age":28, "address":"Beijing"} +20241118012149007 20241118012149007_0_3 4 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 4 David {"age":26, "address":"Shenzhen"} +20241118012149007 20241118012149007_0_4 5 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 5 Eva {"age":31.5, "address":"Chengdu"} +20241118012149007 20241118012149007_0_5 6 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 6 Frank {"age":29.2, "address":"Wuhan"} diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out index 0d18f413b27ade..efad67ffbfa8c4 100644 Binary files a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out and b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out differ diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy index f66bdc6da56a72..b247aaf492400d 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy @@ -36,13 +36,13 @@ suite("test_hudi_schema_evolution", "p2,external,hudi,external_remote,external_r qt_adding_simple_columns_table """ select * from adding_simple_columns_table order by id """ qt_altering_simple_columns_table """ select * from altering_simple_columns_table order by id """ - qt_deleting_simple_columns_table """ select * from deleting_simple_columns_table order by id """ - qt_renaming_simple_columns_table """ select * from renaming_simple_columns_table order by id """ + // qt_deleting_simple_columns_table """ select * from deleting_simple_columns_table order by id """ + // qt_renaming_simple_columns_table """ select * from renaming_simple_columns_table order by id """ qt_adding_complex_columns_table """ select * from adding_complex_columns_table order by id """ qt_altering_complex_columns_table """ select * from altering_complex_columns_table order by id """ - qt_deleting_complex_columns_table """ select * from deleting_complex_columns_table order by id """ - qt_renaming_complex_columns_table """ select * from renaming_complex_columns_table order by id """ + // qt_deleting_complex_columns_table """ select * from deleting_complex_columns_table order by id """ + // qt_renaming_complex_columns_table """ select * from renaming_complex_columns_table order by id """ sql """drop catalog if exists ${catalog_name};""" } \ No newline at end of file diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy index 59608710fa1de9..53c09e6d5a9031 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy @@ -43,7 +43,7 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu qt_q02 """SELECT * FROM ${table_name} WHERE is_active = TRUE ORDER BY event_time LIMIT 10;""" // Query specific user's activity records and limit output - qt_q03 """SELECT * FROM ${table_name} WHERE user_id = '6ced17b6-17ee-4e7c-b7fa-02b087628fdf' ORDER BY event_time LIMIT 5;""" + qt_q03 """SELECT * FROM ${table_name} WHERE user_id = '62785e0e-ad44-4321-8b20-9ee4c4daca4a' ORDER BY event_time LIMIT 5;""" // Query events within a specific time range and limit output qt_q04 """SELECT * FROM ${table_name} WHERE event_time BETWEEN '2024-01-01 00:00:00' AND '2024-12-31 23:59:59' ORDER BY event_time LIMIT 10;""" @@ -58,7 +58,7 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu qt_q07 """SELECT * FROM ${table_name} WHERE array_contains(tags, 'others') ORDER BY event_time LIMIT 5;""" // Query users living in a specific city and limit output - qt_q08 """SELECT * FROM ${table_name} WHERE struct_element(address, 'city') = 'Brooksstad' ORDER BY event_time LIMIT 5;""" + qt_q08 """SELECT * FROM ${table_name} WHERE struct_element(address, 'city') = 'North Rachelview' ORDER BY event_time LIMIT 5;""" // Query users within a specific coordinate range and limit output qt_q09 """SELECT * FROM ${table_name} WHERE struct_element(struct_element(address, 'coordinates'), 'latitude') BETWEEN 0 AND 100 AND struct_element(struct_element(address, 'coordinates'), 'longitude') BETWEEN 0 AND 100 ORDER BY event_time LIMIT 5;""" @@ -70,19 +70,16 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu qt_q11 """SELECT user_id, signup_date FROM ${table_name} ORDER BY signup_date DESC LIMIT 10;""" // Query users with a specific postal code and limit output - qt_q12 """SELECT * FROM ${table_name} WHERE struct_element(address, 'postal_code') = '02663' ORDER BY event_time LIMIT 5;""" + qt_q12 """SELECT * FROM ${table_name} WHERE struct_element(address, 'postal_code') = '80312' ORDER BY event_time LIMIT 5;""" // Query users with profile pictures and limit output qt_q13 """SELECT user_id, profile_picture FROM ${table_name} WHERE profile_picture IS NOT NULL ORDER BY user_id LIMIT 5;""" - // Query users with specific event_id and limit output - qt_q14 """SELECT * FROM ${table_name} WHERE event_id = 151385 ORDER BY event_time LIMIT 5;""" - // Query users by signup date and limit output - qt_q15 """SELECT * FROM ${table_name} WHERE signup_date = '2024-01-15' ORDER BY user_id LIMIT 5;""" + qt_q14 """SELECT * FROM ${table_name} WHERE signup_date = '2024-01-15' ORDER BY user_id LIMIT 5;""" // Query the total count of purchases for each user and limit output - qt_q16 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY purchase_count DESC LIMIT 5;""" + qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY purchase_count DESC LIMIT 5;""" } test_hudi_snapshot_querys("user_activity_log_cow_non_partition")