From 33b8c2f5b53a4217bb95d98b59f287fb6816149a Mon Sep 17 00:00:00 2001 From: Akshaya Acharya Date: Sat, 8 Feb 2025 22:34:53 +0530 Subject: [PATCH] Multitenant duckduckapi --- README.md | 31 +++-- connector-definition/connector-metadata.yaml | 5 +- connector-definition/template/.gitignore | 1 + ndc-duckduckapi/src/consoleTypes.ts | 42 +++++++ ndc-duckduckapi/src/duckduckapi.ts | 119 ++++++------------- ndc-duckduckapi/src/handlers/schema.ts | 28 ++++- ndc-duckduckapi/src/oauth.ts | 50 ++++++++ ndc-duckduckapi/src/sdk.ts | 10 +- 8 files changed, 185 insertions(+), 101 deletions(-) create mode 100644 ndc-duckduckapi/src/consoleTypes.ts create mode 100644 ndc-duckduckapi/src/oauth.ts diff --git a/README.md b/README.md index 0382c0c..611ff3e 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,29 @@ To test, run the ts connector and refresh the supergraph project (step 3 onwards ---------------------------- +### Environment variables + +The connector supports the following environment variables: + +- `DUCKDB_PATH`: Path inside the docker container to store DuckDB database. + - On DDN, set this to inside the /etc/connector/persist-data directory to persist data on connector restarts. + - DDN scaffolded value: `/etc/connector/persist-data/db` + - Default value: `./persist-data/db` +- `DUCKDB_URL`: Optional. File name of the default DuckDB database. Relative to the DUCKDB_PATH. + - Default value: `./duck.db` + +DDN recognizes the following additional environment variables: + +- `FEATURE_PERSISTENT_DATA`: Optional. Whether to persist data in the connector deployment. + - DDN scaffolded value: `true` +- `FEATURE_MIN_INSTANCES`: Optional. Minimum number of instances to keep running (set to 1 to keep one instance running at all times). + - DDN scaffolded value: `1` + ### How to add a custom OAuth2 provider -_TODO:_ +DDN console has built in OAuth provider templates that can be used by end users to connect to external services. -## Single-tenant support +### Single-tenant support ```typescript const DATABASE_SCHEMA = "create table if not exists foo( ... )"; @@ -116,21 +134,18 @@ const connectorConfig: duckduckapi = { }; ``` -## Multi-tenant support +### Multi-tenant support ```typescript const connectorConfig: duckduckapi = { dbSchema: DATABASE_SCHEMA, functionsFilePath: path.resolve(__dirname, "./functions.ts"), multitenantMode: true, - oauthProviderName: "zendesk", + headersArgumentName: "headers", + getTenantIdFromHeaders: (headers: JSONValue) => string }; ``` -A `Tenant` is identified by the tenantToken in the `oauthProviderName` key of the `x-hasura-oauth-services` header forwarded by the engine. - -A `Tenant` has a unique `tenantId` and an isolated duckdb database. Multiple tenantTokens can map to the same `Tenant` over multiple logins. - The [Zendesk data connector](https://github.com/hasura/zendesk-data-connector) is an example of a multi-tenant data connector. ## Duck DB Features diff --git a/connector-definition/connector-metadata.yaml b/connector-definition/connector-metadata.yaml index 28b4169..13b1e30 100644 --- a/connector-definition/connector-metadata.yaml +++ b/connector-definition/connector-metadata.yaml @@ -2,7 +2,7 @@ packagingDefinition: type: ManagedDockerBuild supportedEnvironmentVariables: - name: FEATURE_PERSISTENT_DATA - description: Persist data in DDN connector deployments + description: Persist data in the connector deployment defaultValue: true - name: FEATURE_MIN_INSTANCES description: Minimum number of instances to keep running (set to 1 to keep one instance running at all times) @@ -10,9 +10,6 @@ supportedEnvironmentVariables: - name: DUCKDB_PATH description: Path inside the docker container to store DuckDB database. Do not change this to outside the persist-data directory or data may not be persisted on connector restarts. defaultValue: /etc/connector/persist-data/db - - name: DUCKDB_URL - description: File name of the default DuckDB database. - defaultValue: ./duck.db commands: {} dockerComposeWatch: # Rebuild the container if a new package restore is required because package[-lock].json changed diff --git a/connector-definition/template/.gitignore b/connector-definition/template/.gitignore index 23f293a..11c6fad 100644 --- a/connector-definition/template/.gitignore +++ b/connector-definition/template/.gitignore @@ -1,5 +1,6 @@ node_modules/ duck.db duck.db.wal +persist-data/ github.index.ts github.functions.ts diff --git a/ndc-duckduckapi/src/consoleTypes.ts b/ndc-duckduckapi/src/consoleTypes.ts new file mode 100644 index 0000000..bc826c1 --- /dev/null +++ b/ndc-duckduckapi/src/consoleTypes.ts @@ -0,0 +1,42 @@ +export type DDNConnectorEndpointsConfigV1 = { + version: 1; + jobs: Array<{ + id: string; + title: string; + functions: { + status: { + functionTag: string; + }; + }; + oauthProviders: Array<{ + id: string; + template: string; + oauthCodeLogin: { + functionTag: string; + }; + oauthDetails: { + clientId: string; + scopes: string; + pkceRequired?: boolean; + authorizationEndpoint?: string; + }; + }>; + }>; +}; + +export type DDNConfigResponseV1 = { + version: 1; + config: string; +}; + +export type DDNJobStatusV1 = { + ok: boolean; + message: string; +}; + +export type DDNOAuthProviderCodeLoginRequestV1 = { + code: string; + tokenEndpoint: string; + codeVerifier?: string; + redirectUri: string; +}; diff --git a/ndc-duckduckapi/src/duckduckapi.ts b/ndc-duckduckapi/src/duckduckapi.ts index 8f057a1..a4c85cd 100644 --- a/ndc-duckduckapi/src/duckduckapi.ts +++ b/ndc-duckduckapi/src/duckduckapi.ts @@ -26,61 +26,47 @@ import { Connection, Database } from "duckdb-async"; import fs from "fs-extra"; import path from "path"; +const DUCKDB_PATH = + (process.env["DUCKDB_PATH"] as string) ?? "./persist-data/db"; +const DUCKDB_URL = (process.env["DUCKDB_URL"] as string) ?? "duck.db"; + let DATABASE_SCHEMA = ""; // Single tenant -const DUCKDB_URL = "duck.db"; let db: Database; export async function getDB() { if (!db) { - const dbUrl = (process.env["DUCKDB_URL"] as string) ?? DUCKDB_URL; - db = await openDatabaseFile(dbUrl); + db = await openDatabaseFile(DUCKDB_URL); } return db; } // Multi tenant -export interface Tenant { - tenantId: string; - tenantToken: string | null; - db: Database; - syncState: string; -} +type TenantId = string; +const tenants = new Map(); -export type TenantToken = string; +export async function getTenantDB(tenantId: TenantId) { + let tenantDb = tenants.get(tenantId); -const tenants = new Map(); - -export function getTenants() { - return tenants; -} - -export function getTenantById(tenantId: string): Tenant | null { - for (let [_, tenant] of tenants.entries()) { - if (tenant.tenantId === tenantId) { - return tenant; - } + if (!tenantDb) { + const dbUrl = `duck-${tenantId}.db`; + tenantDb = await openDatabaseFile(dbUrl); + tenants.set(tenantId, tenantDb); } - return null; -} - -export async function getTenantDB(tenantId: string) { - const tenantDb = getTenantById(tenantId)?.db; - if (tenantDb) return tenantDb; - const dbUrl = `duck-${tenantId}.db`; - return await openDatabaseFile(dbUrl); + return tenantDb; } -export async function transaction( +export async function transaction( db: Database, - fn: (conn: Connection) => Promise -) { + fn: (conn: Connection) => Promise +): Promise { const conn = await db.connect(); await conn.run("begin"); try { - await fn(conn); + const result = await fn(conn); await conn.run("commit"); + return result; } catch (e) { await conn.run("rollback"); throw e; @@ -108,12 +94,19 @@ export type Configuration = lambdaSdk.Configuration & { export type State = lambdaSdk.State; -export interface duckduckapi { +export type duckduckapi = { dbSchema: string; functionsFilePath: string; - multitenantMode?: boolean; - oauthProviderName?: string; -} +} & ( + | { + multitenantMode?: undefined | false; + } + | { + multitenantMode: true; + getTenantIdFromHeaders: (headers: JSONValue) => string; + headersArgumentName: string; + } +); export async function makeConnector( dda: duckduckapi @@ -189,7 +182,7 @@ export async function makeConnector( configuration: Configuration ): Promise { const schema = await lambdaSdkConnector.getSchema(configuration); - return do_get_schema(configuration.duckdbConfig, schema); + return do_get_schema(dda, configuration.duckdbConfig, schema); }, /** @@ -240,7 +233,7 @@ export async function makeConnector( if (configuration.functionsSchema.functions[request.collection]) { return lambdaSdkConnector.query(configuration, state, request); } else { - const db = selectTenantDatabase(dda, request?.arguments?.headers); + const db = await selectTenantDatabase(dda, request?.arguments?.headers); let query_plans = await plan_queries(configuration, request); return await perform_query(db, query_plans); @@ -296,48 +289,17 @@ export async function makeConnector( return Promise.resolve(connector); } -export function getOAuthCredentialsFromHeader( - headers: JSONValue -): Record { - if (!headers) { - console.log("Engine header forwarding is disabled"); - throw new Error("Engine header forwarding is disabled"); - } - - const oauthServices = headers.value as any; - - try { - const decodedServices = Buffer.from( - oauthServices["x-hasura-oauth-services"] as string, - "base64" - ).toString("utf-8"); - const serviceTokens = JSON.parse(decodedServices); - return serviceTokens; - } catch (error) { - console.log(error); - if (error instanceof Error) { - console.log(error.stack); - } - throw error; - } -} - -function selectTenantDatabase(dda: duckduckapi, headers: any): Database { +async function selectTenantDatabase( + dda: duckduckapi, + headers: any +): Promise { if (!dda.multitenantMode) { return db; } - const token = - getOAuthCredentialsFromHeader(headers)?.[dda.oauthProviderName!] - ?.access_token; - - const tenantDb = tenants.get(token)?.db; - - if (!tenantDb) { - throw new Forbidden("Tenant not found", {}); - } + const tenantId = dda.getTenantIdFromHeaders(headers); - return tenantDb; + return getTenantDB(tenantId); } async function openDatabaseFile(dbUrl: string): Promise { @@ -356,10 +318,7 @@ async function openDatabaseFile(dbUrl: string): Promise { } function getDatabaseFileParts(dbUrl: string) { - const dbPath = path.resolve( - (process.env["DUCKDB_PATH"] as string) ?? ".", - dbUrl - ); + const dbPath = path.resolve(DUCKDB_PATH, dbUrl); const dirPath = path.dirname(dbPath); diff --git a/ndc-duckduckapi/src/handlers/schema.ts b/ndc-duckduckapi/src/handlers/schema.ts index 9926774..53603fb 100644 --- a/ndc-duckduckapi/src/handlers/schema.ts +++ b/ndc-duckduckapi/src/handlers/schema.ts @@ -4,10 +4,28 @@ import { ScalarType, ObjectType, } from "@hasura/ndc-sdk-typescript"; -import { DuckDBConfigurationSchema } from "../duckduckapi"; +import { DuckDBConfigurationSchema, duckduckapi } from "../duckduckapi"; import { SCALAR_TYPES } from "../constants"; -export function do_get_schema(duckdbconfig: DuckDBConfigurationSchema, functionsNDCSchema: SchemaResponse): SchemaResponse { +function getHeadersArgument(dda: duckduckapi) { + if (dda.multitenantMode) { + return { + [dda.headersArgumentName]: { + type: { + type: "named", + name: "JSON", + }, + }, + } as const; + } + return {}; +} + +export function do_get_schema( + dda: duckduckapi, + duckdbconfig: DuckDBConfigurationSchema, + functionsNDCSchema: SchemaResponse +): SchemaResponse { if (!duckdbconfig) { throw new Error("Configuration is missing"); } @@ -17,11 +35,13 @@ export function do_get_schema(duckdbconfig: DuckDBConfigurationSchema, functions if (collection_names.includes(cn)) { collection_infos.push({ name: cn, - arguments: {}, + arguments: { + ...getHeadersArgument(dda), + }, type: cn, uniqueness_constraints: {}, foreign_keys: {}, - description: object_types[cn].description + description: object_types[cn].description, }); } }); diff --git a/ndc-duckduckapi/src/oauth.ts b/ndc-duckduckapi/src/oauth.ts new file mode 100644 index 0000000..c1a2031 --- /dev/null +++ b/ndc-duckduckapi/src/oauth.ts @@ -0,0 +1,50 @@ +export async function exchangeOAuthCodeForToken(req: { + code: string; + tokenEndpoint: string; + clientId: string; + clientSecret?: string; + codeVerifier?: string; + redirectUri: string; +}) { + const params = new URLSearchParams(); + params.append("grant_type", "authorization_code"); + params.append("code", req.code); + params.append("redirect_uri", req.redirectUri); + params.append("client_id", req.clientId); + + if (req.clientSecret) { + params.append("client_secret", req.clientSecret); + } + + if (req.codeVerifier) { + params.append("code_verifier", req.codeVerifier); + } + + try { + if (!req.tokenEndpoint) { + throw new Error("tokenEndpoint is empty"); + } + + const response = await fetch(req.tokenEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: params, + }); + + if (!response.ok) { + throw new Error("Failed to exchange code for token"); + } + + const data: any = await response.json(); + return data; + // return { + // accessToken: data.access_token, + // refreshToken: data?.refresh_token, + // expiresIn: data?.expires_in, + // }; + } catch (error) { + throw error; + } +} diff --git a/ndc-duckduckapi/src/sdk.ts b/ndc-duckduckapi/src/sdk.ts index 0327b96..1d2dc0c 100644 --- a/ndc-duckduckapi/src/sdk.ts +++ b/ndc-duckduckapi/src/sdk.ts @@ -1,14 +1,14 @@ export { start } from "@hasura/ndc-sdk-typescript"; export { Connection, Database } from "duckdb-async"; + export { makeConnector, duckduckapi, getDB, transaction, - getOAuthCredentialsFromHeader, - getTenants, - getTenantById, getTenantDB, - Tenant, - TenantToken, } from "./duckduckapi"; + +export { exchangeOAuthCodeForToken } from "./oauth"; + +export * from "./consoleTypes";