From 937f4f512141bfba4393e9a0d4d4bb34b5ae704e Mon Sep 17 00:00:00 2001 From: Jianmin Zhao Date: Thu, 12 Sep 2024 17:52:25 -0700 Subject: [PATCH 1/2] CBL-6207: Test multi-level unnest w/o index. The tests come from the API Specification. Note that "group-by" does not work with unnest without index. --- C/tests/c4QueryTest.cc | 12 ++ LiteCore/tests/QueryTest.cc | 280 ++++++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+) diff --git a/C/tests/c4QueryTest.cc b/C/tests/c4QueryTest.cc index 29b35843f..70a92fea8 100644 --- a/C/tests/c4QueryTest.cc +++ b/C/tests/c4QueryTest.cc @@ -864,6 +864,18 @@ N_WAY_TEST_CASE_METHOD(NestedQueryTest, "C4Query UNNEST objects", "[Query][C]") WHERE: ['=', ['concat()', ['.shape.color'], ['to_string()',['.shape.size']]], 'red3']}")); checkExplanation(withIndex); CHECK(run() == (vector{"3"})); + + compileSelect(json5("{WHAT: [['.shape.color'], ['min()', ['.shape.size']]], \ + FROM: [{as: 'doc'}, \ + {as: 'shape', unnest: ['.doc.shapes']}],\ + GROUP_BY: [['.shape.color']]}")); + checkExplanation(false); // even with index, this must do a scan + if ( withIndex ) + CHECK(run2() == (vector{"blue, 10", "cyan, 3", "green, 2", "red, 3", "white, 1", "yellow, 5"})); + else + CHECK(run2() + == (vector{"MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING", + "MISSING, MISSING", "MISSING, MISSING"})); } } diff --git a/LiteCore/tests/QueryTest.cc b/LiteCore/tests/QueryTest.cc index 25cd7c69e..3af1a16c9 100644 --- a/LiteCore/tests/QueryTest.cc +++ b/LiteCore/tests/QueryTest.cc @@ -2991,3 +2991,283 @@ TEST_CASE_METHOD(QueryTest, "Invalid collection names", "[Query]") { } } } + +namespace { + void AddUnnestDoc(KeyStore* store, slice docID, slice fname, slice lname) { + ExclusiveTransaction t(store->dataFile()); + DataFileTestFixture::writeDoc(*store, docID, DocumentFlags::kNone, t, [=](Encoder& enc) { + enc.writeKey("name"); + enc.beginDictionary(); + enc.writeKey("first"); + enc.writeString(fname); + enc.writeKey("last"); + enc.writeString(lname); + enc.endDictionary(); + + enc.writeKey("contacts"); + enc.beginArray(); + + // contacts[0] + enc.beginDictionary(); + enc.writeKey("type"); + enc.writeString("primary"); + enc.writeKey("address"); + enc.beginDictionary(); + enc.writeKey("street"); + enc.writeString("1 St"); + enc.writeKey("city"); + enc.writeString("San Pedro"); + enc.writeKey("state"); + enc.writeString("CA"); + enc.endDictionary(); + enc.writeKey("phones"); + enc.beginArray(); + enc.beginDictionary(); + enc.writeKey("type"); + enc.writeString("home"); + enc.writeKey("numbers"); + enc.beginArray(); + enc.writeString("310-123-4567"); + enc.writeString("310-123-4568"); + enc.endArray(); // numbers + enc.endDictionary(); //phones[0] + enc.beginDictionary(); + enc.writeKey("type"); + enc.writeString("mobile"); + enc.writeKey("numbers"); + enc.beginArray(); + enc.writeString("310-123-6789"); + enc.writeString("310-222-1234"); + enc.endArray(); + enc.endDictionary(); // phones[1] + enc.endArray(); // phones + enc.writeKey("emails"); + enc.beginArray(); + enc.writeString("lue@email.com"); + enc.writeString("laserna@email.com"); + enc.endArray(); // emails + enc.endDictionary(); // contacts[0] + + // contacts[1] + enc.beginDictionary(); + enc.writeKey("type"); + enc.writeString("secondary"); + enc.writeKey("address"); + enc.beginDictionary(); + enc.writeKey("street"); + enc.writeString("5 St"); + enc.writeKey("city"); + enc.writeString("Santa Clara"); + enc.writeKey("state"); + enc.writeString("CA"); + enc.endDictionary(); // address + enc.writeKey("phones"); + enc.beginArray(); + enc.beginDictionary(); + enc.writeKey("type"); + enc.writeString("home"); + enc.writeKey("numbers"); + enc.beginArray(); + enc.writeString("650-123-4567"); + enc.writeString("650-123-2120"); + enc.endArray(); // numbers + enc.endDictionary(); //phones[0] + enc.beginDictionary(); //phones[1] + enc.writeKey("type"); + enc.writeString("mobile"); + enc.writeKey("numbers"); + enc.beginArray(); + enc.writeString("650-123-6789"); + enc.endArray(); + enc.endDictionary(); // phones[1] + enc.endArray(); // phones + enc.writeKey("emails"); + enc.beginArray(); + enc.writeString("eul@email.com"); + enc.writeString("laser@email.com"); + enc.endArray(); // emails + enc.endDictionary(); // contacts[1] + + enc.endArray(); // contacts + + enc.writeKey("likes"); + enc.beginArray(); + enc.writeString("Soccer"); + enc.writeString("Cat"); + enc.endArray(); + }); + t.commit(); + } +} // anonymous namespace + +N_WAY_TEST_CASE_METHOD(QueryTest, "Query with Unnest", "[Query]") { + AddUnnestDoc(store, "doc01"_sl, "Lue"_sl, "Laserna"_sl); + + // Case 1, Single level UNNEST + // --------------------------- + // SELECT name.first as name, c.address.city as city + // FROM profiles + // UNNEST contacts AS c + + string queryStr = "{WHAT: [['AS', ['."s + collectionName + ".name.first'], 'name']," + + "['AS', ['.c.address.city'], 'city']], " + "FROM: [{COLLECTION: '" + collectionName + "'}, " + + "{UNNEST: ['." + collectionName + ".contacts'], AS: 'c'}]}"; + Retained query{store->compileQuery(json5(queryStr), QueryLanguage::kJSON)}; + Retained e{query->createEnumerator()}; + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(!e->next()); + + // Case 2, Multiple Single level UNNESTs + // ------------------------------------- + // SELECT name.first as name, c.address.city as city, `like` + // FROM profiles + // UNNEST contacts AS c + // UNNEST likes AS `like` + + queryStr = "{WHAT: [['AS', ['."s + collectionName + ".name.first'], 'name']," + + "['AS', ['.c.address.city'], 'city'], ['.like']], " + "FROM: [{COLLECTION: '" + collectionName + "'}, " + + "{UNNEST: ['." + collectionName + ".contacts'], AS: 'c'}, " + "{UNNEST: ['." + collectionName + + ".likes'], AS: 'like'}]}"; + query = store->compileQuery(json5(queryStr), QueryLanguage::kJSON); + e = query->createEnumerator(); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "Soccer"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "Cat"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "Soccer"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "Cat"_sl); + CHECK(!e->next()); + + // Case 3, 2-Level UNNEST + // ---------------------- + // SELECT name.first as name, c.address.city as city, p.numbers as phone + // FROM profiles + // UNNEST contacts AS c + // UNNEST c.phones AS p + + queryStr = "{WHAT: [['AS', ['."s + collectionName + ".name.first'], 'name']," + + "['AS', ['.c.address.city'], 'city'], ['AS', ['.p.numbers'], 'phone']], " + "FROM: [{COLLECTION: '" + + collectionName + "'}, " + "{UNNEST: ['." + collectionName + ".contacts'], AS: 'c'}, " + + "{UNNEST: ['.c.phones'], AS: 'p'}]}"; + query = store->compileQuery(json5(queryStr), QueryLanguage::kJSON); + e = query->createEnumerator(); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->toJSONString() == "[\"310-123-4567\",\"310-123-4568\"]"); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->toJSONString() == "[\"310-123-6789\",\"310-222-1234\"]"); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->toJSONString() == "[\"650-123-4567\",\"650-123-2120\"]"); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->toJSONString() == "[\"650-123-6789\"]"); + CHECK(!e->next()); + + // Case 4, 3-level UNNEST + // ---------------------- + // SELECT name.first as name, c.address.city as city, p.type as `phone-type`, number + // FROM profiles + // UNNEST contacts AS c + // UNNEST c.phones AS p + // UNNEST p.numbers as number + + queryStr = "{WHAT: [['AS', ['."s + collectionName + ".name.first'], 'name'], " + + "['AS', ['.c.address.city'], 'city'], ['AS', ['.p.type'], 'phone-type'], " + "['.number']], " + + "FROM: [{COLLECTION: '" + collectionName + "'}, " + "{UNNEST: ['." + collectionName + + ".contacts'], AS: 'c'}, " + "{UNNEST: ['.c.phones'], AS: 'p'}, " + + "{UNNEST: ['.p.numbers'], AS: 'number'}]}"; + query = store->compileQuery(json5(queryStr), QueryLanguage::kJSON); + e = query->createEnumerator(); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "home"_sl); + CHECK(e->columns()[3]->asString() == "310-123-4567"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "home"_sl); + CHECK(e->columns()[3]->asString() == "310-123-4568"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "mobile"_sl); + CHECK(e->columns()[3]->asString() == "310-123-6789"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "mobile"_sl); + CHECK(e->columns()[3]->asString() == "310-222-1234"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "home"_sl); + CHECK(e->columns()[3]->asString() == "650-123-4567"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "home"_sl); + CHECK(e->columns()[3]->asString() == "650-123-2120"_sl); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "mobile"_sl); + CHECK(e->columns()[3]->asString() == "650-123-6789"_sl); + CHECK(!e->next()); + + // Test 5, Unnest with Where & order by + // ------------------------- + // SELECT name.first as name, c.address.city as city, p.numbers[0] as phone + // FROM profiles + // UNNEST contacts AS c + // UNNEST c.phones AS p + // WHERE p.type = "mobile" + // ORDER BY c.address.city DESC + + queryStr = "{WHAT: [['AS', ['."s + collectionName + ".name.first'], 'name']," + + "['AS', ['.c.address.city'], 'city'], ['AS', ['.p.numbers[0]'], 'phone']], " + "FROM: [{COLLECTION: '" + + collectionName + "'}, " + "{UNNEST: ['." + collectionName + ".contacts'], AS: 'c'}, " + + "{UNNEST: ['.c.phones'], AS: 'p'}], " + "WHERE: ['=', ['.p.type'], 'mobile'], " + + "ORDER_BY: [['DESC', ['.c.address.city']]]}"; + query = store->compileQuery(json5(queryStr), QueryLanguage::kJSON); + e = query->createEnumerator(); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "Santa Clara"_sl); + CHECK(e->columns()[2]->asString() == "650-123-6789"); + CHECK(e->next()); + CHECK(e->columns()[0]->asString() == "Lue"_sl); + CHECK(e->columns()[1]->asString() == "San Pedro"_sl); + CHECK(e->columns()[2]->asString() == "310-123-6789"); + CHECK(!e->next()); + + // Test 6, Unnest with Group-by + // ---------------------------- + // SELECT DISTINCT c.address.state as state, count(c.address.city) as num + // FROM profiles + // UNNEST contacts AS c + // GROUP BY c.address.state, c.address.city + // + // "Group By" does not work with pure virtual table, fl_each. c.f. test "C4Query UNNEST objects" in C4QueryTest.cc +} From 7404ccd9148dfb81f9f8aeda3d6d7cee1c927973 Mon Sep 17 00:00:00 2001 From: Jianmin Zhao Date: Tue, 17 Sep 2024 16:14:40 -0700 Subject: [PATCH 2/2] Add docs by JSON. --- C/tests/c4QueryTest.cc | 2 +- LiteCore/tests/QueryTest.cc | 148 +++++++++++------------------------- 2 files changed, 44 insertions(+), 106 deletions(-) diff --git a/C/tests/c4QueryTest.cc b/C/tests/c4QueryTest.cc index 70a92fea8..517b6e4b9 100644 --- a/C/tests/c4QueryTest.cc +++ b/C/tests/c4QueryTest.cc @@ -872,7 +872,7 @@ N_WAY_TEST_CASE_METHOD(NestedQueryTest, "C4Query UNNEST objects", "[Query][C]") checkExplanation(false); // even with index, this must do a scan if ( withIndex ) CHECK(run2() == (vector{"blue, 10", "cyan, 3", "green, 2", "red, 3", "white, 1", "yellow, 5"})); - else + else // Unnest without index is not working yet with group_by. CHECK(run2() == (vector{"MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING", "MISSING, MISSING"})); diff --git a/LiteCore/tests/QueryTest.cc b/LiteCore/tests/QueryTest.cc index 3af1a16c9..5f055e108 100644 --- a/LiteCore/tests/QueryTest.cc +++ b/LiteCore/tests/QueryTest.cc @@ -2993,115 +2993,52 @@ TEST_CASE_METHOD(QueryTest, "Invalid collection names", "[Query]") { } namespace { - void AddUnnestDoc(KeyStore* store, slice docID, slice fname, slice lname) { - ExclusiveTransaction t(store->dataFile()); - DataFileTestFixture::writeDoc(*store, docID, DocumentFlags::kNone, t, [=](Encoder& enc) { - enc.writeKey("name"); - enc.beginDictionary(); - enc.writeKey("first"); - enc.writeString(fname); - enc.writeKey("last"); - enc.writeString(lname); - enc.endDictionary(); - - enc.writeKey("contacts"); - enc.beginArray(); - - // contacts[0] - enc.beginDictionary(); - enc.writeKey("type"); - enc.writeString("primary"); - enc.writeKey("address"); - enc.beginDictionary(); - enc.writeKey("street"); - enc.writeString("1 St"); - enc.writeKey("city"); - enc.writeString("San Pedro"); - enc.writeKey("state"); - enc.writeString("CA"); - enc.endDictionary(); - enc.writeKey("phones"); - enc.beginArray(); - enc.beginDictionary(); - enc.writeKey("type"); - enc.writeString("home"); - enc.writeKey("numbers"); - enc.beginArray(); - enc.writeString("310-123-4567"); - enc.writeString("310-123-4568"); - enc.endArray(); // numbers - enc.endDictionary(); //phones[0] - enc.beginDictionary(); - enc.writeKey("type"); - enc.writeString("mobile"); - enc.writeKey("numbers"); - enc.beginArray(); - enc.writeString("310-123-6789"); - enc.writeString("310-222-1234"); - enc.endArray(); - enc.endDictionary(); // phones[1] - enc.endArray(); // phones - enc.writeKey("emails"); - enc.beginArray(); - enc.writeString("lue@email.com"); - enc.writeString("laserna@email.com"); - enc.endArray(); // emails - enc.endDictionary(); // contacts[0] - - // contacts[1] - enc.beginDictionary(); - enc.writeKey("type"); - enc.writeString("secondary"); - enc.writeKey("address"); - enc.beginDictionary(); - enc.writeKey("street"); - enc.writeString("5 St"); - enc.writeKey("city"); - enc.writeString("Santa Clara"); - enc.writeKey("state"); - enc.writeString("CA"); - enc.endDictionary(); // address - enc.writeKey("phones"); - enc.beginArray(); - enc.beginDictionary(); - enc.writeKey("type"); - enc.writeString("home"); - enc.writeKey("numbers"); - enc.beginArray(); - enc.writeString("650-123-4567"); - enc.writeString("650-123-2120"); - enc.endArray(); // numbers - enc.endDictionary(); //phones[0] - enc.beginDictionary(); //phones[1] - enc.writeKey("type"); - enc.writeString("mobile"); - enc.writeKey("numbers"); - enc.beginArray(); - enc.writeString("650-123-6789"); - enc.endArray(); - enc.endDictionary(); // phones[1] - enc.endArray(); // phones - enc.writeKey("emails"); - enc.beginArray(); - enc.writeString("eul@email.com"); - enc.writeString("laser@email.com"); - enc.endArray(); // emails - enc.endDictionary(); // contacts[1] - - enc.endArray(); // contacts - - enc.writeKey("likes"); - enc.beginArray(); - enc.writeString("Soccer"); - enc.writeString("Cat"); - enc.endArray(); - }); + void AddDocWithJSON(KeyStore* store, slice docID, ExclusiveTransaction& t, const char* jsonBody) { + DataFileTestFixture::writeDoc( + *store, docID, DocumentFlags::kNone, t, + [=](Encoder& enc) { + impl::JSONConverter jc(enc); + if ( !jc.encodeJSON(slice(jsonBody)) ) { + enc.reset(); + error(error::Fleece, jc.errorCode(), jc.errorMessage())._throw(); + } + }, + false); t.commit(); } } // anonymous namespace N_WAY_TEST_CASE_METHOD(QueryTest, "Query with Unnest", "[Query]") { - AddUnnestDoc(store, "doc01"_sl, "Lue"_sl, "Laserna"_sl); + const char* json = R"==( +{ + "name":{"first":"Lue","last":"Laserna"}, + "contacts":[ + { + "type":"primary", + "address":{"street":"1 St","city":"San Pedro","state":"CA"}, + "phones":[ + {"type":"home","numbers":["310-123-4567","310-123-4568"]}, + {"type":"mobile","numbers":["310-123-6789","310-222-1234"]} + ], + "emails":["lue@email.com","laserna@email.com"] + }, + { + "type":"secondary", + "address":{"street":"5 St","city":"Santa Clara","state":"CA"}, + "phones":[ + {"type":"home","numbers":["650-123-4567","650-123-2120"]}, + {"type":"mobile","numbers":["650-123-6789"]} + ], + "emails":["eul@email.com","laser@email.com"] + } + ], + "likes":["Soccer","Cat"] +} +)=="; + { + ExclusiveTransaction t(store->dataFile()); + AddDocWithJSON(store, "doc01"_sl, t, json); + } // Case 1, Single level UNNEST // --------------------------- @@ -3269,5 +3206,6 @@ N_WAY_TEST_CASE_METHOD(QueryTest, "Query with Unnest", "[Query]") { // UNNEST contacts AS c // GROUP BY c.address.state, c.address.city // - // "Group By" does not work with pure virtual table, fl_each. c.f. test "C4Query UNNEST objects" in C4QueryTest.cc + // "Group By" does not work yet with pure virtual table, fl_each. + // c.f. test "C4Query UNNEST objects" in C4QueryTest.cc }