CBL-6737: Merge branch 'release/3.2' to the master branch

C/c4Index.cc

@@ -37,13 +37,14 @@ struct C4IndexImpl final : public C4Index {
     slice getExpression() const noexcept { return _spec.expression; }
 
     bool getOptions(C4IndexOptions& opts) const noexcept {
-        opts = {};
+        opts            = {};
+        bool hasOptions = false;
         if ( auto ftsOpts = _spec.ftsOptions() ) {
             opts.language         = ftsOpts->language;
             opts.ignoreDiacritics = ftsOpts->ignoreDiacritics;
             opts.disableStemming  = ftsOpts->disableStemming;
             opts.stopWords        = ftsOpts->stopWords;
-            return true;
+            hasOptions            = true;
 
 #ifdef COUCHBASE_ENTERPRISE
         } else if ( auto vecOpts = _spec.vectorOptions() ) {
@@ -87,14 +88,18 @@ struct C4IndexImpl final : public C4Index {
             if ( vecOpts->minTrainingCount ) opts.vector.minTrainingSize = unsigned(*vecOpts->minTrainingCount);
             if ( vecOpts->maxTrainingCount ) opts.vector.maxTrainingSize = unsigned(*vecOpts->maxTrainingCount);
             opts.vector.lazy = vecOpts->lazyEmbedding;
-            return true;
+            hasOptions       = true;
 #endif
         } else if ( auto arrOpts = _spec.arrayOptions() ) {
             opts.unnestPath = (const char*)arrOpts->unnestPath.buf;
-            return true;
-        } else {
-            return false;
+            hasOptions      = true;
         }
+
+        if ( !_spec.whereClause.empty() ) {
+            opts.where = (char*)_spec.whereClause.buf;
+            hasOptions = true;
+        }
+        return hasOptions;
     }
 
 #ifdef COUCHBASE_ENTERPRISE

C/scripts/c4_ee.txt

@@ -1,7 +1,6 @@
 # Couchbase Lite (used for bindings generation)
 # New API should go below the 'C4Tests' comment line below first
 
-c4listener_availableAPIs
 c4listener_start
 c4listener_free
 c4listener_shareDB

C/tests/c4ArrayIndexTest.cc

@@ -364,26 +364,30 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "CRUD Array Index Shared Path", "[C][Arra
     bool deleted = c4coll_deleteIndex(coll, "phones"_sl, ERROR_INFO());
     REQUIRE(deleted);
 
-    // We must re-create the Query objects because we deleted the index - which means the unnest tables may be missing.
-    cityQuery = c4query_new2(
-            db, kC4N1QLQuery,
-            R"(SELECT p.pid, c.address.city, c.address.state FROM profiles AS p UNNEST p.contacts AS c WHERE c.address.state = "CA")"_sl,
-            nullptr, ERROR_INFO());
-    REQUIRE(cityQuery);
+    // cityQuery is not affected by the deletion of index "phones"
+    queryenum = REQUIRED(c4query_run(cityQuery, nullslice, nullptr));
+    validateQuery(queryenum, {
+                                     R"(["p-0001", "San Pedro", "CA"])",
+                                     R"(["p-0001", "San Pedro", "CA"])",
+                             });
+
+    // phoneQuery is affected by the deletion of index "phones"
+    // Following error will be logged,
+    // 2024-10-29T21:14:28.226339 DB ERROR SQLite error (code 1): no such table: 152b9815998e188eb99eb1612aafbb3ee6031535 in "SELECT fl_result(fl_value(prof.body, 'pid')), fl_result(fl_unnested_value(c.body, 'address.city')), fl_result(fl_unnested_value(c.body, 'address.state')), fl_result(fl_unnested_value(p.body, 'type')), fl_result(fl_unnested_value(p.body, 'numbers')) FROM "kv_.profiles" AS prof JOIN bc89db8a20fe759bf161b84adf2294d9bfe0c88d AS c ON c.docid=prof.rowid JOIN "152b9815998e188eb99eb1612aafbb3ee6031535" AS p ON p.docid=c.rowid WHERE fl_unnested_value(p.body, 'type') = 'mobile'". This table is referenced by an array index, which may have been deleted.
+    C4Error error;
+    queryenum = c4query_run(phoneQuery, nullslice, &error);
+    CHECK(!queryenum);  // This query relies on the index that has been deleted.
+    CHECK((error.domain == SQLiteDomain && error.code == 1));
 
+    // Recompile the query
     phoneQuery = c4query_new2(
             db, kC4N1QLQuery,
             R"(SELECT prof.pid, c.address.city, c.address.state, p.type, p.numbers FROM profiles AS prof UNNEST prof.contacts AS c UNNEST c.phones AS p WHERE p.type = "mobile")"_sl,
             nullptr, ERROR_INFO());
     REQUIRE(phoneQuery);
+    queryenum = c4query_run(phoneQuery, nullslice, &error);
+    CHECK(queryenum);
 
-
-    queryenum = REQUIRED(c4query_run(cityQuery, nullslice, nullptr));
-    validateQuery(queryenum, {
-                                     R"(["p-0001", "San Pedro", "CA"])",
-                                     R"(["p-0001", "San Pedro", "CA"])",
-                             });
-    queryenum = REQUIRED(c4query_run(phoneQuery, nullslice, nullptr));
     validateQuery(queryenum, {
                                      R"(["p-0001", "San Pedro", "CA", "mobile", ["310-9601308"]])",
                                      R"(["p-0001", "San Pedro", "CA", "mobile", ["310-4833623"]])",
@@ -669,6 +673,9 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Without Alias", "[C][Unnest]") {
 
 // 7. TestUnnestArrayLiteralNotSupport
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Array Literal Not Supported", "[C][Unnest]") {
+    C4Collection* coll = createCollection(db, {"profiles"_sl, "_default"_sl});
+    importTestData(coll);
+
     C4Error err{};
     c4::ref query = c4query_new2(
             db, kC4N1QLQuery,

C/tests/c4CppUtils.hh

@@ -93,7 +93,9 @@ namespace c4 {
 
         ref(const ref& r) noexcept : _obj(retainRef(r._obj)) {}
 
-        ~ref() noexcept { releaseRef(_obj); }
+        ~ref() noexcept {
+            if ( _obj ) releaseRef(_obj);
+        }
 
         static ref retaining(T* t) { return ref(retainRef(t)); }
 

C/tests/c4DatabaseTest.cc

@@ -1334,6 +1334,33 @@ TEST_CASE("Database Upgrade from 3.1 with peerCheckpoints table", "[Database][Up
     CHECK(db);
 }
 
+TEST_CASE("Database Upgrade to WithIndexesWhereColumn", "[Database][Upgrade][C]") {
+    // This db has table "indexes". This test ensures that the column whereClause is
+    // successfully added to the table.
+    string dbPath = "upgrade_2.8_index.cblite2";
+
+    C4DatabaseFlags withFlags{0};
+    C4Log("---- Opening copy of db %s with flags 0x%x", dbPath.c_str(), withFlags);
+    C4DatabaseConfig2 config = {slice(TempDir()), withFlags};
+    auto              name   = C4Test::copyFixtureDB(kVersionedFixturesSubDir + dbPath);
+    C4Log("---- copy Fixture to: %s/%s", TempDir().c_str(), name.asString().c_str());
+    C4Error             err;
+    c4::ref<C4Database> db = REQUIRED(c4db_openNamed(name, &config, WITH_ERROR(&err)));
+
+    auto           defaultColl = REQUIRED(c4db_getDefaultCollection(db, nullptr));
+    C4IndexOptions options{};
+    options.where = "gender = 'female'";
+    REQUIRE(c4coll_createIndex(defaultColl, C4STR("length"), c4str("length(name.first)"), kC4N1QLQuery, kC4ValueIndex,
+                               &options, WITH_ERROR(&err)));
+
+    // Check we can get the where clause back via c4index_getOptions
+    auto           index = REQUIRED(c4coll_getIndex(defaultColl, C4STR("length"), nullptr));
+    C4IndexOptions outOptions;
+    REQUIRE(c4index_getOptions(index, &outOptions));
+    CHECK(string(outOptions.where) == options.where);
+    c4index_release(index);
+}
+
 static void setRemoteRev(C4Database* db, slice docID, slice revID, C4RemoteID remote) {
     auto        defaultColl = c4db_getDefaultCollection(db, nullptr);
     C4Document* doc         = c4coll_getDoc(defaultColl, docID, true, kDocGetAll, ERROR_INFO());

C/tests/c4QueryTest.cc

@@ -289,6 +289,34 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query expression index", "[Query][C]") {
     CHECK(run() == (vector<string>{"0000015", "0000099"}));
 }
 
+N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query partial value index", "[Query][C]") {
+    C4Error              err;
+    auto                 defaultColl = getCollection(db, kC4DefaultCollectionSpec);
+    const vector<string> result{"0000099"};
+    for ( int withIndex = 0; withIndex < 2; ++withIndex ) {
+        if ( withIndex ) {
+            C4IndexOptions options{};
+            options.where = "gender = 'female'";
+            REQUIRE(c4coll_createIndex(defaultColl, C4STR("length"), c4str("length(name.first)"), kC4N1QLQuery,
+                                       kC4ValueIndex, &options, WITH_ERROR(&err)));
+        }
+        compileSelect("SELECT META().id FROM _ WHERE length(name.first) = 9 AND gender = 'female'", kC4N1QLQuery);
+        REQUIRE(query);
+        checkExplanation(withIndex);
+        CHECK(run() == result);
+
+        if ( withIndex ) {
+            // Logically equivalent query, changing gender = 'female' to gender != 'male'.
+            // Because the condition is not exact as the condition in partial index, it would not use
+            // the index.
+            compileSelect("SELECT META().id FROM _ WHERE length(name.first) = 9 AND gender != 'male'", kC4N1QLQuery);
+            REQUIRE(query);
+            checkExplanation(!withIndex);
+            CHECK(run() == result);
+        }
+    }
+}
+
 static bool lookForIndex(C4Database* db, slice name) {
     bool found       = false;
     auto defaultColl = C4QueryTest::getCollection(db, kC4DefaultCollectionSpec);
@@ -421,9 +449,18 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query dict literal", "[Query][C]") {
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query FTS", "[Query][C][FTS]") {
     C4Error err;
     auto    defaultColl = getCollection(db, kC4DefaultCollectionSpec);
-    REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("[[\".contact.address.street\"]]"), kC4JSONQuery,
-                               kC4FullTextIndex, nullptr, WITH_ERROR(&err)));
-    compile(json5("['MATCH()', 'byStreet', 'Hwy']"));
+
+    bool useJSON = GENERATE(true, false);
+    if ( useJSON ) {
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("[[\".contact.address.street\"]]"),
+                                   kC4JSONQuery, kC4FullTextIndex, nullptr, WITH_ERROR(&err)));
+        compile(json5("['MATCH()', 'byStreet', 'Hwy']"));
+    } else {
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("contact.address.street"), kC4N1QLQuery,
+                                   kC4FullTextIndex, nullptr, WITH_ERROR(&err)));
+        compileSelect("SELECT META().id FROM _ WHERE MATCH(byStreet, 'Hwy')", kC4N1QLQuery);
+    }
+
     auto results = runFTS();
     CHECK(results
           == (vector<vector<C4FullTextMatch>>{{{13, 0, 0, 10, 3}},
@@ -438,6 +475,57 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query FTS", "[Query][C][FTS]") {
     c4slice_free(matched);
 }
 
+N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query FTS (Partial)", "[Query][C][FTS]") {
+    C4Error err;
+    auto    defaultColl = getCollection(db, kC4DefaultCollectionSpec);
+
+    // c.f. above test, "C4Query FTS". The query is the same.
+    // W/o "where" in options, there are five contacts whose street addresses contain "Hwy".
+
+    bool useJSON = GENERATE(false, true);
+
+    C4IndexOptions options{};
+    // With the following "where" option, the query picks 2 Californians.
+    if ( useJSON ) {
+        options.where = R"(["=", [".contact.address.state"], "CA"])";
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("[[\".contact.address.street\"]]"),
+                                   kC4JSONQuery, kC4FullTextIndex, &options, WITH_ERROR(&err)));
+        compile(json5("['MATCH()', 'byStreet', 'Hwy']"));
+    } else {
+        options.where = "contact.address.state = 'CA'";
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("contact.address.street"), kC4N1QLQuery,
+                                   kC4FullTextIndex, &options, WITH_ERROR(&err)));
+        compileSelect("SELECT META().id FROM _ WHERE MATCH(byStreet, 'Hwy')", kC4N1QLQuery);
+    }
+
+    auto results = runFTS();
+    CHECK(results == (vector<vector<C4FullTextMatch>>{{{15, 0, 0, 11, 3}}, {{43, 0, 0, 12, 3}}}));
+
+    {
+        // Check we can get the where clause back via c4index_getOptions
+        auto           index = REQUIRED(c4coll_getIndex(defaultColl, C4STR("byStreet"), nullptr));
+        C4IndexOptions outOptions;
+        REQUIRE(c4index_getOptions(index, &outOptions));
+        CHECK(string(outOptions.where) == options.where);
+        c4index_release(index);
+    }
+
+    // By creating the index with different options.where, the original index will be deleted,
+    // and the index table will be based soly on the new where clause. We get one Texan.
+    if ( useJSON ) {
+        options.where = R"(["=", [".contact.address.state"], "TX"])";
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("[[\".contact.address.street\"]]"),
+                                   kC4JSONQuery, kC4FullTextIndex, &options, WITH_ERROR(&err)));
+    } else {
+        options.where = "contact.address.state = 'TX'";
+        REQUIRE(c4coll_createIndex(defaultColl, C4STR("byStreet"), C4STR("contact.address.street"), kC4N1QLQuery,
+                                   kC4FullTextIndex, &options, WITH_ERROR(&err)));
+    }
+
+    results = runFTS();
+    CHECK(results == (vector<vector<C4FullTextMatch>>{{{44, 0, 0, 12, 3}}}));
+}
+
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query FTS multiple properties", "[Query][C][FTS]") {
     C4Error err;
     auto    defaultColl = c4db_getDefaultCollection(db, nullptr);
@@ -1496,6 +1584,15 @@ TEST_CASE_METHOD(CollectionTest, "C4Query FTS Multiple collections", "[Query][C]
 
     CHECK(run().size() == 50);
     CHECK(runFTS().size() == 50);
+
+    auto deleted = c4coll_deleteIndex(names, C4STR("byStreet"), nullptr);
+    CHECK(deleted);
+    // The query won't run after the index is deleted. We should see following error in the log,
+    // 2024-10-29T20:57:00.896439 DB ERROR SQLite error (code 1): no such table: kv_.namedscope.names::by\Street in "SELECT "namedscope.names".rowid, offsets(fts1."kv_.namedscope.names::by\Street"), offsets(fts2."kv_.wiki::by\Text"), fl_result("namedscope.names".key), fl_result(wiki.key) FROM "kv_.namedscope.names" AS "namedscope.names" INNER JOIN "kv_.wiki" AS wiki ON (fl_value("namedscope.names".body, 'birthday') != fl_value(wiki.body, 'title')) JOIN "kv_.namedscope.names::by\Street" AS fts1 ON fts1.docid = "namedscope.names".rowid JOIN "kv_.wiki::by\Text" AS fts2 ON fts2.docid = wiki.rowid WHERE fts1."kv_.namedscope.names::by\Street" MATCH 'Hwy' AND fts2. This table is referenced by an FTS index, which may have been deleted.
+    C4Error error;
+    auto    qenum = c4query_run(query, c4str(nullptr), &error);
+    CHECK(!qenum);
+    CHECK((error.domain == SQLiteDomain && error.code == 1));
 }
 
 #pragma mark - OBSERVERS:

C/tests/c4QueryTest.hh

@@ -34,11 +34,11 @@ class C4QueryTest : public C4Test {
 
     ~C4QueryTest() { c4query_release(query); }
 
-    void compileSelect(const std::string& queryStr) {
+    void compileSelect(const std::string& queryStr, C4QueryLanguage language = kC4JSONQuery) {
         INFO("Query = " << queryStr);
         C4Error error{};
         c4query_release(query);
-        query = c4query_new2(db, kC4JSONQuery, c4str(queryStr.c_str()), nullptr, ERROR_INFO(error));
+        query = c4query_new2(db, language, c4str(queryStr.c_str()), nullptr, ERROR_INFO(error));
         REQUIRE(query);
     }
 

LiteCore/Database/CollectionImpl.hh

@@ -480,8 +480,16 @@ namespace litecore {
                     error::_throw(error::InvalidParameter, "Invalid index type");
                     break;
             }
-            return keyStore().createIndex(indexName, indexSpec, (QueryLanguage)indexLanguage,
-                                          (IndexSpec::Type)indexType, options);
+            if ( indexOptions ) {
+                constexpr const char* indexTypeNames[] = {"Value", "FullText", "Array", "Predictive", "Vector"};
+                if ( indexOptions->where && !IndexSpec::canPartialIndex((IndexSpec::Type)indexType) )
+                    error::_throw(error::InvalidParameter, "%s index does support partial index.",
+                                  indexTypeNames[indexType]);
+            }
+
+            return keyStore().createIndex({indexName.asString(), (IndexSpec::Type)indexType, indexSpec,
+                                           slice{indexOptions ? indexOptions->where : nullptr},
+                                           (QueryLanguage)indexLanguage, options});
         }
 
         Retained<C4Index> getIndex(slice name) override { return C4Index::getIndex(this, name); }