Multitenant duckduckapi

README.md

@@ -101,11 +101,29 @@ To test, run the ts connector and refresh the supergraph project (step 3 onwards
 
 ----------------------------
 
+### Environment variables
+
+The connector supports the following environment variables:
+
+- `DUCKDB_PATH`: Path inside the docker container to store DuckDB database.
+  - On DDN, set this to inside the /etc/connector/persist-data directory to persist data on connector restarts.
+  - DDN scaffolded value: `/etc/connector/persist-data/db`
+  - Default value: `./persist-data/db`
+- `DUCKDB_URL`: Optional. File name of the default DuckDB database. Relative to the DUCKDB_PATH.
+  - Default value: `./duck.db`
+
+DDN recognizes the following additional environment variables:
+
+- `FEATURE_PERSISTENT_DATA`: Optional. Whether to persist data in the connector deployment.
+  - DDN scaffolded value: `true`
+- `FEATURE_MIN_INSTANCES`: Optional. Minimum number of instances to keep running (set to 1 to keep one instance running at all times).
+  - DDN scaffolded value: `1`
+
 ### How to add a custom OAuth2 provider
 
-_TODO:_
+DDN console has built in OAuth provider templates that can be used by end users to connect to external services.
 
-## Single-tenant support
+### Single-tenant support
 
 ```typescript
 const DATABASE_SCHEMA = "create table if not exists foo( ... )";
@@ -116,21 +134,18 @@ const connectorConfig: duckduckapi = {
 };
 ```
 
-## Multi-tenant support
+### Multi-tenant support
 
 ```typescript
 const connectorConfig: duckduckapi = {
   dbSchema: DATABASE_SCHEMA,
   functionsFilePath: path.resolve(__dirname, "./functions.ts"),
   multitenantMode: true,
-  oauthProviderName: "zendesk",
+  headersArgumentName: "headers",
+  getTenantIdFromHeaders: (headers: JSONValue) => string
 };
 ```
 
-A `Tenant` is identified by the tenantToken in the `oauthProviderName` key of the `x-hasura-oauth-services` header forwarded by the engine.
-
-A `Tenant` has a unique `tenantId` and an isolated duckdb database. Multiple tenantTokens can map to the same `Tenant` over multiple logins.
-
 The [Zendesk data connector](https://github.com/hasura/zendesk-data-connector) is an example of a multi-tenant data connector.
 
 ## Duck DB Features

connector-definition/connector-metadata.yaml

@@ -2,17 +2,14 @@ packagingDefinition:
   type: ManagedDockerBuild
 supportedEnvironmentVariables:
   - name: FEATURE_PERSISTENT_DATA
-    description: Persist data in DDN connector deployments
+    description: Persist data in the connector deployment
     defaultValue: true
   - name: FEATURE_MIN_INSTANCES
     description: Minimum number of instances to keep running (set to 1 to keep one instance running at all times)
     defaultValue: 1
   - name: DUCKDB_PATH
     description: Path inside the docker container to store DuckDB database. Do not change this to outside the persist-data directory or data may not be persisted on connector restarts.
     defaultValue: /etc/connector/persist-data/db
-  - name: DUCKDB_URL
-    description: File name of the default DuckDB database.
-    defaultValue: ./duck.db
 commands: {}
 dockerComposeWatch:
   # Rebuild the container if a new package restore is required because package[-lock].json changed

connector-definition/template/.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 duck.db
 duck.db.wal
+persist-data/
 github.index.ts
 github.functions.ts

ndc-duckduckapi/src/consoleTypes.ts

@@ -0,0 +1,42 @@
+export type DDNConnectorEndpointsConfigV1 = {
+  version: 1;
+  jobs: Array<{
+    id: string;
+    title: string;
+    functions: {
+      status: {
+        functionTag: string;
+      };
+    };
+    oauthProviders: Array<{
+      id: string;
+      template: string;
+      oauthCodeLogin: {
+        functionTag: string;
+      };
+      oauthDetails: {
+        clientId: string;
+        scopes: string;
+        pkceRequired?: boolean;
+        authorizationEndpoint?: string;
+      };
+    }>;
+  }>;
+};
+
+export type DDNConfigResponseV1 = {
+  version: 1;
+  config: string;
+};
+
+export type DDNJobStatusV1 = {
+  ok: boolean;
+  message: string;
+};
+
+export type DDNOAuthProviderCodeLoginRequestV1 = {
+  code: string;
+  tokenEndpoint: string;
+  codeVerifier?: string;
+  redirectUri: string;
+};

ndc-duckduckapi/src/duckduckapi.ts

@@ -26,61 +26,47 @@ import { Connection, Database } from "duckdb-async";
 import fs from "fs-extra";
 import path from "path";
 
+const DUCKDB_PATH =
+  (process.env["DUCKDB_PATH"] as string) ?? "./persist-data/db";
+const DUCKDB_URL = (process.env["DUCKDB_URL"] as string) ?? "duck.db";
+
 let DATABASE_SCHEMA = "";
 
 // Single tenant
-const DUCKDB_URL = "duck.db";
 let db: Database;
 export async function getDB() {
   if (!db) {
-    const dbUrl = (process.env["DUCKDB_URL"] as string) ?? DUCKDB_URL;
-    db = await openDatabaseFile(dbUrl);
+    db = await openDatabaseFile(DUCKDB_URL);
   }
   return db;
 }
 
 // Multi tenant
-export interface Tenant {
-  tenantId: string;
-  tenantToken: string | null;
-  db: Database;
-  syncState: string;
-}
+type TenantId = string;
+const tenants = new Map<TenantId, Database>();
 
-export type TenantToken = string;
+export async function getTenantDB(tenantId: TenantId) {
+  let tenantDb = tenants.get(tenantId);
 
-const tenants = new Map<TenantToken, Tenant>();
-
-export function getTenants() {
-  return tenants;
-}
-
-export function getTenantById(tenantId: string): Tenant | null {
-  for (let [_, tenant] of tenants.entries()) {
-    if (tenant.tenantId === tenantId) {
-      return tenant;
-    }
+  if (!tenantDb) {
+    const dbUrl = `duck-${tenantId}.db`;
+    tenantDb = await openDatabaseFile(dbUrl);
+    tenants.set(tenantId, tenantDb);
   }
-  return null;
-}
-
-export async function getTenantDB(tenantId: string) {
-  const tenantDb = getTenantById(tenantId)?.db;
-  if (tenantDb) return tenantDb;
 
-  const dbUrl = `duck-${tenantId}.db`;
-  return await openDatabaseFile(dbUrl);
+  return tenantDb;
 }
 
-export async function transaction(
+export async function transaction<T>(
   db: Database,
-  fn: (conn: Connection) => Promise<void>
-) {
+  fn: (conn: Connection) => Promise<T>
+): Promise<T> {
   const conn = await db.connect();
   await conn.run("begin");
   try {
-    await fn(conn);
+    const result = await fn(conn);
     await conn.run("commit");
+    return result;
   } catch (e) {
     await conn.run("rollback");
     throw e;
@@ -108,12 +94,19 @@ export type Configuration = lambdaSdk.Configuration & {
 
 export type State = lambdaSdk.State;
 
-export interface duckduckapi {
+export type duckduckapi = {
   dbSchema: string;
   functionsFilePath: string;
-  multitenantMode?: boolean;
-  oauthProviderName?: string;
-}
+} & (
+  | {
+      multitenantMode?: undefined | false;
+    }
+  | {
+      multitenantMode: true;
+      getTenantIdFromHeaders: (headers: JSONValue) => string;
+      headersArgumentName: string;
+    }
+);
 
 export async function makeConnector(
   dda: duckduckapi
@@ -189,7 +182,7 @@ export async function makeConnector(
       configuration: Configuration
     ): Promise<SchemaResponse> {
       const schema = await lambdaSdkConnector.getSchema(configuration);
-      return do_get_schema(configuration.duckdbConfig, schema);
+      return do_get_schema(dda, configuration.duckdbConfig, schema);
     },
 
     /**
@@ -240,7 +233,7 @@ export async function makeConnector(
       if (configuration.functionsSchema.functions[request.collection]) {
         return lambdaSdkConnector.query(configuration, state, request);
       } else {
-        const db = selectTenantDatabase(dda, request?.arguments?.headers);
+        const db = await selectTenantDatabase(dda, request?.arguments?.headers);
 
         let query_plans = await plan_queries(configuration, request);
         return await perform_query(db, query_plans);
@@ -296,48 +289,17 @@ export async function makeConnector(
   return Promise.resolve(connector);
 }
 
-export function getOAuthCredentialsFromHeader(
-  headers: JSONValue
-): Record<string, any> {
-  if (!headers) {
-    console.log("Engine header forwarding is disabled");
-    throw new Error("Engine header forwarding is disabled");
-  }
-
-  const oauthServices = headers.value as any;
-
-  try {
-    const decodedServices = Buffer.from(
-      oauthServices["x-hasura-oauth-services"] as string,
-      "base64"
-    ).toString("utf-8");
-    const serviceTokens = JSON.parse(decodedServices);
-    return serviceTokens;
-  } catch (error) {
-    console.log(error);
-    if (error instanceof Error) {
-      console.log(error.stack);
-    }
-    throw error;
-  }
-}
-
-function selectTenantDatabase(dda: duckduckapi, headers: any): Database {
+async function selectTenantDatabase(
+  dda: duckduckapi,
+  headers: any
+): Promise<Database> {
   if (!dda.multitenantMode) {
     return db;
   }
 
-  const token =
-    getOAuthCredentialsFromHeader(headers)?.[dda.oauthProviderName!]
-      ?.access_token;
-
-  const tenantDb = tenants.get(token)?.db;
-
-  if (!tenantDb) {
-    throw new Forbidden("Tenant not found", {});
-  }
+  const tenantId = dda.getTenantIdFromHeaders(headers);
 
-  return tenantDb;
+  return getTenantDB(tenantId);
 }
 
 async function openDatabaseFile(dbUrl: string): Promise<Database> {
@@ -356,10 +318,7 @@ async function openDatabaseFile(dbUrl: string): Promise<Database> {
 }
 
 function getDatabaseFileParts(dbUrl: string) {
-  const dbPath = path.resolve(
-    (process.env["DUCKDB_PATH"] as string) ?? ".",
-    dbUrl
-  );
+  const dbPath = path.resolve(DUCKDB_PATH, dbUrl);
 
   const dirPath = path.dirname(dbPath);
 

ndc-duckduckapi/src/handlers/schema.ts

@@ -4,10 +4,28 @@ import {
   ScalarType,
   ObjectType,
 } from "@hasura/ndc-sdk-typescript";
-import { DuckDBConfigurationSchema } from "../duckduckapi";
+import { DuckDBConfigurationSchema, duckduckapi } from "../duckduckapi";
 import { SCALAR_TYPES } from "../constants";
 
-export function do_get_schema(duckdbconfig: DuckDBConfigurationSchema, functionsNDCSchema: SchemaResponse): SchemaResponse {
+function getHeadersArgument(dda: duckduckapi) {
+  if (dda.multitenantMode) {
+    return {
+      [dda.headersArgumentName]: {
+        type: {
+          type: "named",
+          name: "JSON",
+        },
+      },
+    } as const;
+  }
+  return {};
+}
+
+export function do_get_schema(
+  dda: duckduckapi,
+  duckdbconfig: DuckDBConfigurationSchema,
+  functionsNDCSchema: SchemaResponse
+): SchemaResponse {
   if (!duckdbconfig) {
     throw new Error("Configuration is missing");
   }
@@ -17,11 +35,13 @@ export function do_get_schema(duckdbconfig: DuckDBConfigurationSchema, functions
     if (collection_names.includes(cn)) {
       collection_infos.push({
         name: cn,
-        arguments: {},
+        arguments: {
+          ...getHeadersArgument(dda),
+        },
         type: cn,
         uniqueness_constraints: {},
         foreign_keys: {},
-        description: object_types[cn].description
+        description: object_types[cn].description,
       });
     }
   });