WIP REFACTOR multiInstance step 1 (pubkey#3870)

* REFACTOR multiInstance step 1

* FIX lokijs multiInstance events

* CHANGE do not reuse the same worker

* CHANGE test dexie-worker instead of loki-worker

* FIX dexie-worker tests

* FIX randomly failing tests

* FIX slow test

* FIX timeout test

* FIX test

* FIX test

* FIX tests

* FIX tests

* FIX tests

* FIX timeouts

* FIX teimouts

* FIX timeouts

* FIX ci

* FIX ci

* FIX timeouts

* FIX pouchdb multi instance support

* FIX timeouts

* FIX timeouts

* FIX update lwt

* FIX

* FIX randomly failing tests

* FIX randomly failing test

* FIX random failing test

* UPDATE node js

* REFACTOR correct re-use of the broadcast channel

* FIX cleanup of broadcast channels

* FIX timeout

* FIX do not use multi instance in rx-storage-replication tests

* FIX stuff

* FIX lokijs tests

* FIX imports

* FIX logs

Author: pubkey

.github/workflows/main.yml

@@ -178,7 +178,7 @@ jobs:
 
   # run the node tests for the LokiJS RxStorage in a different
   # task to run in parallel.
-  node-lokijs-worker:
+  node-dexie-worker:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -205,11 +205,11 @@ jobs:
       - name: build
         run: npm run build
 
-      - name: npm run test:node:lokijs-worker
-        run: npm run test:node:lokijs-worker
+      - name: npm run test:node:dexie-worker
+        run: npm run test:node:dexie-worker
 
-      - name: npm run test:fast:lokijs-worker
-        run: npm run test:fast:lokijs-worker
+      - name: npm run test:fast:dexie-worker
+        run: npm run test:fast:dexie-worker
 
   # run the node tests for the Dexie.js RxStorage in a different
   # task to run in parallel.

.nvmrc

@@ -1 +1 @@
-v18.3.0
+v18.4.0

CHANGELOG.md

@@ -3,6 +3,8 @@
 
 <!-- CHANGELOG NEWEST -->
 
+- The worker RxStorage will no longer reuse the worker so that multiple RxDatabase instances can use different workers for better performance.
+
 <!-- ADD new changes here! -->
 
 <!-- /CHANGELOG NEWEST -->

docs-src/rx-storage-worker.md

@@ -2,10 +2,6 @@
 
 With the worker plugin, you can put the `RxStorage` of your database inside of a WebWorker (in browsers) or a Worker Thread (in node.js). By doing so, you can take CPU load from the main process and move it into the worker's process which can improve the percieved performance of your application. RxDB uses [threads.js](https://github.com/andywer/threads.js/) to create the Worker process an to communicate with it.
 
-In theory you can put any `RxStorage` implementation into a worker. For now this is only tested with the [LokiJS RxStorage](./rx-storage-lokijs.md).
-
-
-
 ## On the worker process
 
 ```ts
@@ -85,6 +81,35 @@ const database = await createRxDatabase({
 ```
 
 
+## One worker per database
+
+Each call to `getRxStorageWorker()` will create a different worker instance so that when you have more then one `RxDatabase`, each database will have its own JavaScript worker process.
+
+To reuse the worker instance in more then one `RxDatabase`, you can store the output of `getRxStorageWorker()` into a variable an use that one. Reusing the worker can decrease the initial page load, but you might get slower database operations.
+
+```ts
+// Call getRxStorageWorker() exactly once
+const workerStorage = getRxStorageWorker({
+    statics: RxStorageLokiStatics,
+    workerInput: 'path/to/worker.js'
+});
+
+// use the same storage for both databases.
+const databaseOne = await createRxDatabase({
+    name: 'database-one',
+    storage: workerStorage
+});
+const databaseTwo = await createRxDatabase({
+    name: 'database-two',
+    storage: workerStorage
+});
+
+```
+
+
+
+
+
 --------------------------------------------------------------------------------
 
 If you are new to RxDB, you should continue [here](./rx-storage-sharding.md)

orga/before-next-major.md

@@ -82,6 +82,11 @@ Having mixed types causes many confusion, for example when the type is `['string
 you could run a query selector like `$gt: 10` where it now is not clear if the string `foobar` is matching or not.
 
 
+
+## Remove RxDatabase.broadcastChannel
+The broadcast channel has been moved out of the RxDatabase and is part of the RxStorage. So we should not longer expose the getter.
+
+
 ## Remove temporary documents
 
 https://github.com/pubkey/rxdb/pull/3777#issuecomment-1120669088

package.json

@@ -36,20 +36,20 @@
     "postinstall": "node scripts/postinstall.js || echo \"ignore\"",
     "test": "npm run test:node && npm run test:browser",
     "// test:fast": "run tests in the fast-mode. Most of them will run in parrallel, skips tests that are known slow",
-    "test:fast": "npm run test:fast:memory && npm run test:fast:pouchdb && npm run test:fast:lokijs && npm run test:fast:lokijs-worker && npm run test:fast:dexie",
+    "test:fast": "npm run test:fast:memory && npm run test:fast:pouchdb && npm run test:fast:lokijs && npm run test:fast:dexie-worker && npm run test:fast:dexie",
     "test:fast:pouchdb": "npm run transpile && rimraf -rf pouch__all_dbs__ && cross-env DEFAULT_STORAGE=pouchdb NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:fast:memory": "npm run transpile && rimraf -rf pouch__all_dbs__ && cross-env DEFAULT_STORAGE=memory NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:fast:lokijs": "npm run transpile && cross-env DEFAULT_STORAGE=lokijs NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:fast:dexie": "npm run transpile && cross-env DEFAULT_STORAGE=dexie NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
-    "test:fast:lokijs-worker": "npm run transpile && npm run build:workers && rimraf -rf pouch__all_dbs__ && cross-env DEFAULT_STORAGE=lokijs-worker NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
+    "test:fast:dexie-worker": "npm run transpile && npm run build:workers && rimraf -rf pouch__all_dbs__ && cross-env DEFAULT_STORAGE=dexie-worker NODE_ENV=fast mocha --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "// test:fast:loop": "runs tests in the fast-mode in a loop. Use this to debug tests that only fail sometimes",
     "test:fast:loop": "npm run test:fast && npm run test:fast:loop",
     "test:fast:loop:lokijs": "npm run test:fast:lokijs && npm run test:fast:loop:lokijs",
-    "test:node": "npm run test:node:pouchdb && npm run test:node:lokijs && npm run test:node:lokijs-worker",
+    "test:node": "npm run test:node:pouchdb && npm run test:node:lokijs && npm run test:node:dexie",
     "test:node:pouchdb": "npm run transpile && cross-env DEFAULT_STORAGE=pouchdb mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:node:memory": "npm run transpile && cross-env DEFAULT_STORAGE=memory mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:node:lokijs": "npm run transpile && cross-env DEFAULT_STORAGE=lokijs mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
-    "test:node:lokijs-worker": "npm run transpile && npm run build:workers && cross-env DEFAULT_STORAGE=lokijs-worker mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
+    "test:node:dexie-worker": "npm run transpile && npm run build:workers && cross-env DEFAULT_STORAGE=dexie-worker mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:node:dexie": "npm run transpile && cross-env DEFAULT_STORAGE=dexie mocha --expose-gc --config ./config/.mocharc.js ./test_tmp/unit.test.js",
     "test:node:pouchdb:loop": "npm run test:node:pouchdb && npm run test:node:pouchdb:loop",
     "test:node:lokijs:loop": "npm run test:node:lokijs && npm run test:node:lokijs:loop",
@@ -72,7 +72,7 @@
     "test:circular": "npm run build && madge --circular ./dist/es/index.js",
     "test:performance:pouchdb": "npm run transpile && cross-env STORAGE=pouchdb mocha --config ./config/.mocharc.js ./test_tmp/performance.test.js --unhandled-rejections=strict --expose-gc",
     "test:performance:lokijs": "npm run transpile && cross-env STORAGE=lokijs mocha --config ./config/.mocharc.js ./test_tmp/performance.test.js --unhandled-rejections=strict --expose-gc",
-    "test:performance:lokijs-worker": "npm run transpile && cross-env STORAGE=lokijs-worker mocha --config ./config/.mocharc.js ./test_tmp/performance.test.js --unhandled-rejections=strict --expose-gc",
+    "test:performance:dexie-worker": "npm run transpile && cross-env STORAGE=dexie-worker mocha --config ./config/.mocharc.js ./test_tmp/performance.test.js --unhandled-rejections=strict --expose-gc",
     "couch:start": "docker run -d -p 5984:5984 --rm --name rxdb-couchdb couchdb:2.1.1",
     "couch:stop": "docker rm -f rxdb-couchdb",
     "test:couchdb": "npm run transpile && cross-env DEFAULT_STORAGE=pouchdb mocha --config ./config/.mocharc.js ./test_tmp/couch-db-integration.test.js",
@@ -264,4 +264,4 @@
     "webpack-bundle-analyzer": "4.5.0",
     "webpack-cli": "4.9.2"
   }
-}
+}

src/index.ts

@@ -80,6 +80,7 @@ export {
 export * from './rx-storage-helper';
 
 export * from './rx-storage-replication';
+export * from './rx-storage-multiinstance';
 
 export * from './custom-index';
 export * from './query-planner';

src/plugins/dev-mode/error-messages.ts

@@ -59,7 +59,7 @@ export const ERROR_MESSAGES = {
     DB8: 'RxDatabase.create(): A RxDatabase with the same name and adapter already exists.\n' +
         'Make sure to use this combination only once or set ignoreDuplicate to true if you do this intentional',
     DB9: 'createRxDatabase(): Adapter not added. Use addPouchPlugin(require(\'pouchdb-adapter-[adaptername]\'));',
-    DB10: 'createRxDatabase(): To use leveldown-adapters, you have to add the leveldb-plugin. Use addRxPlugin(require(\'pouchdb-adapter-leveldb\'));',
+    DB10: 'createRxDatabase(): To use leveldown-adapters, you have to add the leveldb-plugin. Use addPouchPlugin(require(\'pouchdb-adapter-leveldb\'));',
     DB11: 'createRxDatabase(): Invalid db-name, folder-paths must not have an ending slash',
 
     // rx-collection

src/plugins/dexie/rx-storage-dexie.ts

@@ -37,9 +37,6 @@ export const RxStorageDexieStatics: RxStorageStatics = {
         });
     },
     hashKey: 'md5',
-    doesBroadcastChangestream() {
-        return false;
-    },
     prepareQuery<RxDocType>(
         schema: RxJsonSchema<RxDocumentData<RxDocType>>,
         mutateableQuery: FilledMangoQuery<RxDocType>

src/plugins/dexie/rx-storage-instance-dexie.ts

@@ -44,6 +44,7 @@ import {
 import { dexieQuery } from './dexie-query';
 import { getPrimaryFieldOfPrimaryKey } from '../../rx-schema-helper';
 import { getUniqueDeterministicEventKey } from '../../rx-storage-helper';
+import { addRxStorageMultiInstanceSupport } from '../../rx-storage-multiinstance';
 
 let instanceId = now();
 
@@ -419,5 +420,10 @@ export async function createDexieStorageInstance<RxDocType>(
         settings
     );
 
+    addRxStorageMultiInstanceSupport(
+        params,
+        instance
+    );
+
     return instance;
 }

src/plugins/leader-election.ts

@@ -7,19 +7,23 @@ import {
     LeaderElector,
     BroadcastChannel
 } from 'broadcast-channel';
+import {
+    getBroadcastChannelReference,
+    removeBroadcastChannelReference
+} from '../rx-storage-multiinstance';
 
 import type {
     RxDatabase,
     RxPlugin
 } from '../types';
 import {
-    ensureNotFalsy,
     PROMISE_RESOLVE_TRUE
 } from '../util';
 
 const LEADER_ELECTORS_OF_DB: WeakMap<RxDatabase, LeaderElector> = new WeakMap();
 const LEADER_ELECTOR_BY_BROADCAST_CHANNEL: WeakMap<BroadcastChannel, LeaderElector> = new WeakMap();
 
+
 /**
  * Returns the leader elector of a broadcast channel.
  * Used to ensure we reuse the same elector for the channel each time.
@@ -33,9 +37,28 @@ export function getLeaderElectorByBroadcastChannel(broadcastChannel: BroadcastCh
     return elector;
 }
 
-
+/**
+ * @overwrites RxDatabase().leaderElector for caching
+ */
 export function getForDatabase(this: RxDatabase): LeaderElector {
-    const broadcastChannel = ensureNotFalsy(this.broadcastChannel);
+
+
+    const broadcastChannel = getBroadcastChannelReference(
+        this.token,
+        this.name,
+        this
+    );
+
+    /**
+     * Clean up the reference on RxDatabase.destroy()
+     */
+    const oldDestroy = this.destroy.bind(this);
+    this.destroy = function () {
+        removeBroadcastChannelReference(this.token, this);
+        return oldDestroy();
+    }
+
+
     let elector = getLeaderElectorByBroadcastChannel(broadcastChannel);
     if (!elector) {
         elector = getLeaderElectorByBroadcastChannel(broadcastChannel);
@@ -44,6 +67,12 @@ export function getForDatabase(this: RxDatabase): LeaderElector {
             elector
         );
     }
+
+    /**
+     * Overwrite for caching
+     */
+    this.leaderElector = () => elector;
+
     return elector;
 }
 

src/plugins/local-documents/local-documents-helper.ts

@@ -18,6 +18,7 @@ import type {
     RxLocalDocumentData,
     RxStorage
 } from '../../types';
+import { randomCouchString } from '../../util';
 
 const LOCAL_DOC_STATE_BY_PARENT: WeakMap<LocalDocumentParent, Promise<LocalDocumentState>> = new WeakMap();
 
@@ -27,6 +28,7 @@ export function createLocalDocStateByParent(parent: LocalDocumentParent): void {
     const collectionName = parent.database ? parent.name : '';
     const statePromise = (async () => {
         let storageInstance = await createLocalDocumentStorageInstance(
+            database.token,
             database.storage,
             database.name,
             collectionName,
@@ -101,13 +103,15 @@ export function getLocalDocStateByParent(parent: LocalDocumentParent): Promise<L
 
 
 export function createLocalDocumentStorageInstance(
+    databaseInstanceToken: string,
     storage: RxStorage<any, any>,
     databaseName: string,
     collectionName: string,
     instanceCreationOptions: any,
     multiInstance: boolean
 ) {
     return storage.createStorageInstance<RxLocalDocumentData>({
+        databaseInstanceToken,
         databaseName: databaseName,
         /**
          * Use a different collection name for the local documents instance
@@ -134,7 +138,9 @@ export async function removeLocalDocumentsStorageInstance(
     databaseName: string,
     collectionName: string
 ) {
+    const databaseInstanceToken = randomCouchString(10);
     const storageInstance = await createLocalDocumentStorageInstance(
+        databaseInstanceToken,
         storage,
         databaseName,
         collectionName,

src/plugins/lokijs/loki-save-queue.ts

@@ -64,8 +64,7 @@ export class LokiSaveQueue {
                  * Because LokiJS is a in-memory database,
                  * we can just wait until the JavaScript process is idle
                  * via requestIdlePromise(). Then we know that nothing important
-                 * is running at the moment. Also we wait at least wait 100ms
-                 * to ensure it has enough time to free up stuff.
+                 * is running at the moment.
                  */
                 await requestIdlePromise().then(() => requestIdlePromise());