feat: duckdb-wasm query UI [DRAFT]

package.json

@@ -7,6 +7,8 @@
     "serve": "vite preview"
   },
   "dependencies": {
+    "@codemirror/lang-sql": "^0.20.0",
+    "@duckdb/duckdb-wasm": "^1.14.1",
     "@githubocto/flat-ui": "^0.13.5",
     "@octokit/rest": "^18.3.5",
     "@popperjs/core": "^2.9.1",
@@ -15,10 +17,12 @@
     "@types/lodash": "^4.14.168",
     "@types/lodash.debounce": "^4.0.6",
     "@types/lodash.truncate": "^4.0.6",
+    "@uiw/react-codemirror": "^4.7.0",
     "classcat": "^5.0.3",
     "d3-dsv": "^2.0.0",
     "date-fns": "^2.19.0",
     "downshift": "^6.1.0",
+    "duckdb": "^0.3.4",
     "formik": "^2.2.6",
     "lodash": "^4.17.21",
     "lodash.debounce": "^4.0.8",
@@ -36,6 +40,7 @@
     "react-router-dom": "^5.2.0",
     "reakit": "^1.3.8",
     "store2": "^2.12.0",
+    "use-debounce": "^8.0.1",
     "use-query-params": "^1.2.2",
     "vite-plugin-rewrite-all": "^0.1.2",
     "wretch": "^1.7.4",
@@ -44,16 +49,16 @@
   },
   "devDependencies": {
     "@octokit/types": "^6.12.2",
-    "@tailwindcss/forms": "^0.2.1",
+    "@tailwindcss/forms": "^0.5.1",
     "@types/nprogress": "^0.2.0",
     "@types/react": "^17.0.0",
     "@types/react-dom": "^17.0.0",
     "@types/react-portal": "^4.0.2",
     "@types/react-router-dom": "^5.1.7",
-    "@vitejs/plugin-react-refresh": "^1.3.1",
+    "@vitejs/plugin-react": "^1.3.2",
     "autoprefixer": "^10.2.5",
     "postcss": "^8.2.8",
-    "tailwindcss": "^2.1.2",
+    "tailwindcss": "^3.0.24",
     "typescript": "^4.1.2",
     "vite": "^2.0.5"
   }

public/duckdb/duckdb-browser-blocking.d.ts

@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb-browser-blocking';

public/duckdb/duckdb-browser.d.ts

@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb';

public/duckdb/duckdb-node-blocking.d.ts

@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb-node-blocking';

public/duckdb/duckdb-node.d.ts

@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb';

src/api/index.ts

@@ -139,6 +139,41 @@ export function fetchCommits(params: FileParams) {
     });
 }
 
+export async function fetchRawDataFile(params: FileParamsWithSHA) {
+  const { filename, name, owner, sha } = params;
+  if (!filename) return "";
+  const fileType = filename.split(".").pop() || "";
+  const validTypes = [
+    "csv",
+    "tsv",
+    "json",
+    "geojson",
+    "topojson",
+    "yml",
+    "yaml",
+  ];
+  if (!validTypes.includes(fileType)) return "";
+  const text = await wretch(
+    `https://raw.githubusercontent.com/${owner}/${name}/${sha}/${filename}`
+  )
+    .get()
+    .notFound(async () => {
+      if (cachedPat) {
+        const data = await githubWretch
+          .url(`/repos/${owner}/${name}/contents/${filename}`)
+          .get()
+          .json();
+        const content = atob(data.content);
+        return content;
+      } else {
+        throw new Error("Data file not found");
+      }
+    })
+    .text();
+
+  return text;
+}
+
 export async function fetchDataFile(params: FileParamsWithSHA) {
   const { filename, name, owner, sha } = params;
   if (!filename) return [];

src/components/db-explorer.tsx

@@ -0,0 +1,214 @@
+import { sql } from "@codemirror/lang-sql";
+import { Grid } from "@githubocto/flat-ui";
+import * as duckdb from "@duckdb/duckdb-wasm";
+import CodeMirror from "@uiw/react-codemirror";
+import React, { useEffect, useMemo, useRef, useState } from "react";
+import { useQuery } from "react-query";
+import { useDebounce } from "use-debounce";
+import { useRawDataFile } from "../hooks";
+import { LoadingState } from "./loading-state";
+import { ErrorState } from "./error-state";
+import { Spinner } from "./spinner";
+import Bug from "../bug.svg";
+
+interface Props {
+  sha: string;
+  filename: string;
+  owner: string;
+  name: string;
+}
+
+interface DBExplorerInnerProps {
+  content: string;
+  filename: string;
+  extension: string;
+  sha: string;
+}
+
+const VALID_EXTENSIONS = ["csv", "json"];
+
+function DBExplorerInner(props: DBExplorerInnerProps) {
+  const { content, extension, filename, sha } = props;
+  const connectionRef = useRef<duckdb.AsyncDuckDBConnection | null>(null);
+  const [query, setQuery] = useState("");
+  const [debouncedQuery] = useDebounce(query, 500);
+  const [dbStatus, setDbStatus] = useState<"error" | "idle" | "success">(
+    "idle"
+  );
+
+  const execQuery = async (query: string) => {
+    if (!connectionRef.current) return;
+    const queryRes = await connectionRef.current.query(query);
+    const asArray = queryRes.toArray();
+    return {
+      numRows: queryRes.numRows,
+      numCols: queryRes.numCols,
+      results: asArray.map((row) => {
+        return row.toJSON();
+      }),
+    };
+  };
+
+  const { data, status, error } = useQuery(
+    ["query-results", filename, sha, debouncedQuery],
+    () => execQuery(debouncedQuery),
+    {
+      refetchOnWindowFocus: false,
+      retry: false,
+    }
+  );
+
+  useEffect(() => {
+    const initDuckDb = async () => {
+      const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
+        mvp: {
+          mainModule: "/duckdb/duckdb.wasm",
+          mainWorker: "/duckdb/duckdb-browser-mvp.worker.js",
+        },
+        eh: {
+          mainModule: "/duckdb/duckdb-eh.wasm",
+          mainWorker: "/duckdb/duckdb-browser-eh.worker.js",
+        },
+      };
+      const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
+      const worker = new Worker(bundle.mainWorker!);
+      const logger = new duckdb.ConsoleLogger();
+      const db = new duckdb.AsyncDuckDB(logger, worker);
+      await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+
+      const c = await db.connect();
+      connectionRef.current = c;
+
+      try {
+        if (extension === "csv") {
+          await db.registerFileText(`data.csv`, content);
+          await c.insertCSVFromPath(`data.csv`, { name: "data" });
+        } else if (extension === "json") {
+          await db.registerFileText(`data.json`, content);
+          await c.insertJSONFromPath("data.json", { name: "data" });
+        }
+        setDbStatus("success");
+        setQuery("select * from data");
+      } catch {
+        setDbStatus("error");
+      }
+    };
+
+    initDuckDb();
+
+    return () => {
+      if (connectionRef.current) {
+        connectionRef.current.close();
+        connectionRef.current = null;
+        setDbStatus("idle");
+      }
+    };
+  }, [content, sha, filename]);
+
+  const sqlSchema = useMemo(() => {
+    if (!content) return [];
+
+    if (extension === "csv") {
+      const names = content.split("\n")[0].split(",");
+      return names.map((name) => name.replace(/"/g, ""));
+    } else if (extension === "json") {
+      try {
+        return Object.keys(JSON.parse(content)[0]);
+      } catch {
+        return [];
+      }
+    } else {
+      return [];
+    }
+  }, [content]);
+
+  return (
+    <div className="flex-1 flex-shrink-0 overflow-hidden flex flex-col z-0">
+      {dbStatus === "idle" && <LoadingState text="Initializing DuckDB 🦆" />}
+      {dbStatus === "error" && (
+        <ErrorState img={Bug} alt="Database initialization error">
+          Couldn't initialize DuckDB 😕
+        </ErrorState>
+      )}
+      {dbStatus === "success" && (
+        <>
+          <div className="border-b bg-gray-50 sticky top-0 z-20">
+            <CodeMirror
+              value={query}
+              height={"120px"}
+              className="w-full"
+              extensions={[
+                sql({
+                  defaultTable: "data",
+                  schema: {
+                    data: sqlSchema,
+                  },
+                }),
+              ]}
+              onChange={(value) => {
+                setQuery(value);
+              }}
+            />
+          </div>
+          <div className="flex-1 flex flex-col h-full overflow-scroll">
+            {status === "error" && error && (
+              <div className="bg-red-50 border-b border-red-600 p-2 text-sm text-red-600">
+                {(error as Error)?.message || "An unexpected error occurred."}
+              </div>
+            )}
+            <div className="relative flex-1 h-full">
+              {status === "loading" && (
+                <div className="absolute top-4 right-4 z-20">
+                  <Spinner />
+                </div>
+              )}
+              {data && (
+                <Grid
+                  data={data.results}
+                  diffData={undefined}
+                  defaultSort={undefined}
+                  defaultStickyColumnName={undefined}
+                  defaultFilters={{}}
+                  downloadFilename={filename}
+                  onChange={() => {}}
+                />
+              )}
+            </div>
+          </div>
+        </>
+      )}
+    </div>
+  );
+}
+
+export function DBExplorer(props: Props) {
+  const { sha, filename, owner, name } = props;
+  const { data, status } = useRawDataFile(
+    {
+      sha,
+      filename,
+      owner,
+      name,
+    },
+    {
+      refetchOnWindowFocus: false,
+      retry: false,
+    }
+  );
+
+  const extension = filename.split(".").pop() || "";
+
+  return (
+    <>
+      {status === "loading" && <LoadingState />}
+      {status === "success" && data && VALID_EXTENSIONS.includes(extension) && (
+        <DBExplorerInner
+          sha={sha}
+          filename={filename}
+          extension={extension}
+          content={data}
+        />
+      )}
+    </>
+  );
+}

src/components/json-detail-container.tsx

@@ -94,7 +94,7 @@ export function JSONDetail(props: JSONDetailProps) {
         sort: newState.sort.join(","),
         stickyColumnName: newState.stickyColumnName,
         filters: encodeFilterString(newState.filters),
-      }, 
+      },
       "replaceIn"
     );
   };