githubocto · mattrothenberg · May 12, 2022 · May 12, 2022 · May 12, 2022 · May 12, 2022
diff --git a/index.html b/index.html
@@ -14,6 +14,9 @@
   </head>
   <body>
     <div id="root"></div>
+    <script>
+      window.require = () => {};
+    </script>
     <script type="module" src="/src/main.tsx"></script>
   </body>
 </html>
diff --git a/package.json b/package.json
@@ -7,6 +7,8 @@
     "serve": "vite preview"
   },
   "dependencies": {
+    "@codemirror/lang-sql": "^0.20.0",
+    "@duckdb/duckdb-wasm": "^1.14.1",
     "@githubocto/flat-ui": "^0.13.5",
     "@octokit/rest": "^18.3.5",
     "@popperjs/core": "^2.9.1",
@@ -15,10 +17,12 @@
     "@types/lodash": "^4.14.168",
     "@types/lodash.debounce": "^4.0.6",
     "@types/lodash.truncate": "^4.0.6",
+    "@uiw/react-codemirror": "^4.7.0",
     "classcat": "^5.0.3",
     "d3-dsv": "^2.0.0",
     "date-fns": "^2.19.0",
     "downshift": "^6.1.0",
+    "duckdb": "^0.3.4",
     "formik": "^2.2.6",
     "lodash": "^4.17.21",
     "lodash.debounce": "^4.0.8",
@@ -36,6 +40,7 @@
     "react-router-dom": "^5.2.0",
     "reakit": "^1.3.8",
     "store2": "^2.12.0",
+    "use-debounce": "^8.0.1",
     "use-query-params": "^1.2.2",
     "vite-plugin-rewrite-all": "^0.1.2",
     "wretch": "^1.7.4",
@@ -44,7 +49,7 @@
   },
   "devDependencies": {
     "@octokit/types": "^6.12.2",
-    "@tailwindcss/forms": "^0.2.1",
+    "@tailwindcss/forms": "^0.5.1",
     "@types/nprogress": "^0.2.0",
     "@types/react": "^17.0.0",
     "@types/react-dom": "^17.0.0",
@@ -53,7 +58,7 @@
     "@vitejs/plugin-react-refresh": "^1.3.1",
     "autoprefixer": "^10.2.5",
     "postcss": "^8.2.8",
-    "tailwindcss": "^2.1.2",
+    "tailwindcss": "^3.0.24",
     "typescript": "^4.1.2",
     "vite": "^2.0.5"
   }

diff --git a/public/duckdb/duckdb-browser-blocking.cjs b/public/duckdb/duckdb-browser-blocking.cjs
diff --git a/public/duckdb/duckdb-browser-blocking.cjs.map b/public/duckdb/duckdb-browser-blocking.cjs.map
diff --git a/public/duckdb/duckdb-browser-blocking.d.ts b/public/duckdb/duckdb-browser-blocking.d.ts
@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb-browser-blocking';
diff --git a/public/duckdb/duckdb-browser-blocking.mjs b/public/duckdb/duckdb-browser-blocking.mjs
diff --git a/public/duckdb/duckdb-browser-blocking.mjs.map b/public/duckdb/duckdb-browser-blocking.mjs.map
diff --git a/public/duckdb/duckdb-browser-coi.pthread.worker.js b/public/duckdb/duckdb-browser-coi.pthread.worker.js
diff --git a/public/duckdb/duckdb-browser-coi.pthread.worker.js.map b/public/duckdb/duckdb-browser-coi.pthread.worker.js.map
diff --git a/public/duckdb/duckdb-browser-coi.worker.js b/public/duckdb/duckdb-browser-coi.worker.js
diff --git a/public/duckdb/duckdb-browser-coi.worker.js.map b/public/duckdb/duckdb-browser-coi.worker.js.map
diff --git a/public/duckdb/duckdb-browser-eh.worker.js b/public/duckdb/duckdb-browser-eh.worker.js
diff --git a/public/duckdb/duckdb-browser-eh.worker.js.map b/public/duckdb/duckdb-browser-eh.worker.js.map
diff --git a/public/duckdb/duckdb-browser-mvp.worker.js b/public/duckdb/duckdb-browser-mvp.worker.js
diff --git a/public/duckdb/duckdb-browser-mvp.worker.js.map b/public/duckdb/duckdb-browser-mvp.worker.js.map
diff --git a/public/duckdb/duckdb-browser.cjs b/public/duckdb/duckdb-browser.cjs
diff --git a/public/duckdb/duckdb-browser.cjs.map b/public/duckdb/duckdb-browser.cjs.map
diff --git a/public/duckdb/duckdb-browser.d.ts b/public/duckdb/duckdb-browser.d.ts
@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb';
diff --git a/public/duckdb/duckdb-browser.mjs b/public/duckdb/duckdb-browser.mjs
diff --git a/public/duckdb/duckdb-browser.mjs.map b/public/duckdb/duckdb-browser.mjs.map
diff --git a/public/duckdb/duckdb-coi.wasm b/public/duckdb/duckdb-coi.wasm
diff --git a/public/duckdb/duckdb-eh.wasm b/public/duckdb/duckdb-eh.wasm
diff --git a/public/duckdb/duckdb-mvp.wasm b/public/duckdb/duckdb-mvp.wasm
diff --git a/public/duckdb/duckdb-node-blocking.cjs b/public/duckdb/duckdb-node-blocking.cjs
diff --git a/public/duckdb/duckdb-node-blocking.cjs.map b/public/duckdb/duckdb-node-blocking.cjs.map
diff --git a/public/duckdb/duckdb-node-blocking.d.ts b/public/duckdb/duckdb-node-blocking.d.ts
@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb-node-blocking';
diff --git a/public/duckdb/duckdb-node-eh.worker.cjs b/public/duckdb/duckdb-node-eh.worker.cjs
diff --git a/public/duckdb/duckdb-node-eh.worker.cjs.map b/public/duckdb/duckdb-node-eh.worker.cjs.map
diff --git a/public/duckdb/duckdb-node-mvp.worker.cjs b/public/duckdb/duckdb-node-mvp.worker.cjs
diff --git a/public/duckdb/duckdb-node-mvp.worker.cjs.map b/public/duckdb/duckdb-node-mvp.worker.cjs.map
diff --git a/public/duckdb/duckdb-node.cjs b/public/duckdb/duckdb-node.cjs
diff --git a/public/duckdb/duckdb-node.cjs.map b/public/duckdb/duckdb-node.cjs.map
diff --git a/public/duckdb/duckdb-node.d.ts b/public/duckdb/duckdb-node.d.ts
@@ -0,0 +1 @@
+export * from './types/src/targets/duckdb';
diff --git a/src/api/index.ts b/src/api/index.ts
@@ -139,6 +139,41 @@ export function fetchCommits(params: FileParams) {
     });
 }
 
+export async function fetchRawDataFile(params: FileParamsWithSHA) {
+  const { filename, name, owner, sha } = params;
+  if (!filename) return "";
+  const fileType = filename.split(".").pop() || "";
+  const validTypes = [
+    "csv",
+    "tsv",
+    "json",
+    "geojson",
+    "topojson",
+    "yml",
+    "yaml",
+  ];
+  if (!validTypes.includes(fileType)) return "";
+  const text = await wretch(
+    `https://raw.githubusercontent.com/${owner}/${name}/${sha}/${filename}`
+  )
+    .get()
+    .notFound(async () => {
+      if (cachedPat) {
+        const data = await githubWretch
+          .url(`/repos/${owner}/${name}/contents/${filename}`)
+          .get()
+          .json();
+        const content = atob(data.content);
+        return content;
+      } else {
+        throw new Error("Data file not found");
+      }
+    })
+    .text();
+
+  return text;
+}
+
 export async function fetchDataFile(params: FileParamsWithSHA) {
   const { filename, name, owner, sha } = params;
   if (!filename) return [];

diff --git a/src/components/db-explorer.tsx b/src/components/db-explorer.tsx
@@ -0,0 +1,209 @@
+import { sql } from "@codemirror/lang-sql";
+import * as duckdb from "@duckdb/duckdb-wasm";
+import CodeMirror from "@uiw/react-codemirror";
+import React, { useEffect, useMemo, useRef, useState } from "react";
+import { useQuery } from "react-query";
+import { useDebounce } from "use-debounce";
+import { useRawDataFile } from "../hooks";
+import { LoadingState } from "./loading-state";
+import { Spinner } from "./spinner";
+
+interface Props {
+  sha: string;
+  filename: string;
+  owner: string;
+  name: string;
+}
+
+interface DBExplorerInnerProps {
+  content: string;
+  filename: string;
+  extension: string;
+  sha: string;
+}
+
+const VALID_EXTENSIONS = ["csv", "json"];
+
+function DBExplorerInner(props: DBExplorerInnerProps) {
+  const { content, extension, filename, sha } = props;
+  const connectionRef = useRef<duckdb.AsyncDuckDBConnection>();
+  const [query, setQuery] = useState("");
+  const [debouncedQuery] = useDebounce(query, 1000);
+  const [dbStatus, setDbStatus] = useState<"error" | "idle" | "success">(
+    "idle"
+  );
+
+  const execQuery = async (query: string) => {
+    if (!connectionRef.current) return;
+    const queryRes = await connectionRef.current.query(query);
+    const asArray = queryRes.toArray();
+    return {
+      numRows: queryRes.numRows,
+      numCols: queryRes.numCols,
+      results: asArray.map((row) => {
+        return row.toJSON();
+      }),
+    };
+  };
+
+  const { data, status, error } = useQuery(
+    ["query-results", filename, sha, debouncedQuery],
+    () => execQuery(debouncedQuery),
+    {
+      refetchOnWindowFocus: false,
+      retry: false,
+    }
+  );
+
+  useEffect(() => {
+    const initDuckDb = async () => {
+      const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
+        mvp: {
+          mainModule: "/duckdb/duckdb.wasm",
+          mainWorker: "/duckdb/duckdb-browser-mvp.worker.js",
+        },
+        eh: {
+          mainModule: "/duckdb/duckdb-eh.wasm",
+          mainWorker: "/duckdb/duckdb-browser-eh.worker.js",
+        },
+      };
+      const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
+      const worker = new Worker(bundle.mainWorker!);
+      const logger = new duckdb.ConsoleLogger();
+      const db = new duckdb.AsyncDuckDB(logger, worker);
+      await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+
+      const c = await db.connect();
+      connectionRef.current = c;
+
+      if (extension === "csv") {
+        await db.registerFileText(`data.csv`, content);
+        await c.insertCSVFromPath(`data.csv`, { name: "data" });
+      } else if (extension === "json") {
+        await db.registerFileText(`data.json`, content);
+        await c.insertJSONFromPath("data.json", { name: "data" });
+      }
+
+      setDbStatus("success");
+      setQuery("select * from data");
+    };
+
+    initDuckDb();
+
+    return () => {
+      if (connectionRef.current) {
+        connectionRef.current.close();
+      }
+    };
+  }, [content]);
+
+  const sqlSchema = useMemo(() => {
+    if (!content) return [];
+
+    if (extension === "csv") {
+      const names = content.split("\n")[0].split(",");
+      return names.map((name) => name.replace(/"/g, ""));
+    } else if (extension === "json") {
+      return Object.keys(JSON.parse(content)[0]);
+    } else {
+      return [];
+    }
+  }, [content]);
+
+  return (
+    <div className="flex-1 flex-shrink-0 overflow-hidden flex flex-col z-0">
+      {dbStatus === "idle" && <LoadingState text="Initializing DuckDB 🦆" />}
+      {dbStatus === "success" && (
+        <>
+          <div className="border-b bg-gray-50 sticky top-0 z-20">
+            <CodeMirror
+              value={query}
+              height={"120px"}
+              className="w-full"
+              extensions={[
+                sql({
+                  defaultTable: "data",
+                  schema: {
+                    data: sqlSchema,
+                  },
+                }),
+              ]}
+              onChange={(value) => {
+                setQuery(value);
+              }}
+            />
+          </div>
+          <div className="flex-1 flex flex-col h-full overflow-scroll">
+            <div className="border-b p-2 flex items-center space-x-2 sticky top-0 bg-white z-10">
+              <div>
+                <p className="text-xs uppercase tracking-widest font-medium">
+                  # Rows
+                </p>
+                {status === "loading" ? (
+                  <span className="skeleton">Loading</span>
+                ) : (
+                  <span>{data && data.numRows}</span>
+                )}
+              </div>
+              <div>
+                <p className="text-xs uppercase tracking-widest font-medium">
+                  # Columns
+                </p>
+                {status === "loading" ? (
+                  <span className="skeleton">Loading</span>
+                ) : (
+                  <span>{data && data.numCols}</span>
+                )}
+              </div>
+            </div>
+            {status === "error" && error && (
+              <div className="bg-red-50 border-b border-red-600 p-2 text-sm text-red-600">
+                {(error as Error)?.message || "An unexpected error occurred."}
+              </div>
+            )}
+            <div className="relative">
+              {status === "loading" && (
+                <div className="absolute top-4 right-4 z-20">
+                  <Spinner />
+                </div>
+              )}
+              <pre>{JSON.stringify(data, null, 2)}</pre>
+            </div>
+          </div>
+        </>
+      )}
+    </div>
+  );
+}
+
+export function DBExplorer(props: Props) {
+  const { sha, filename, owner, name } = props;
+  const { data, status } = useRawDataFile(
+    {
+      sha,
+      filename,
+      owner,
+      name,
+    },
+    {
+      refetchOnWindowFocus: false,
+      retry: false,
+    }
+  );
+
+  const extension = filename.split(".").pop() || "";
+
+  return (
+    <>
+      {status === "loading" && <LoadingState />}
+      {status === "success" && data && VALID_EXTENSIONS.includes(extension) && (
+        <DBExplorerInner
+          sha={sha}
+          filename={filename}
+          extension={extension}
+          content={data}
+        />
+      )}
+    </>
+  );
+}
diff --git a/src/components/json-detail-container.tsx b/src/components/json-detail-container.tsx
@@ -94,7 +94,7 @@ export function JSONDetail(props: JSONDetailProps) {
         sort: newState.sort.join(","),
         stickyColumnName: newState.stickyColumnName,
         filters: encodeFilterString(newState.filters),
-      }, 
+      },
       "replaceIn"
     );
   };

diff --git a/src/components/repo-detail.tsx b/src/components/repo-detail.tsx
@@ -30,6 +30,7 @@ import { Picker } from "./picker";
 import { FilePicker } from "./file-picker";
 import { DisplayCommit } from "./display-commit";
 import truncate from "lodash/truncate";
+import { DBExplorer } from "./db-explorer";
 
 interface RepoDetailProps extends RouteComponentProps<Repo> {}
 
@@ -277,14 +278,20 @@ export function RepoDetail(props: RepoDetailProps) {
 
       <React.Fragment>
         {selectedSha && Boolean(filename) && filesStatus !== "error" && (
-          <JSONDetail
-            key={selectedSha}
+          <DBExplorer
+            sha={selectedSha}
             filename={filename || ""}
             owner={owner as string}
             name={name as string}
-            previousSha={selectedShaPrevious}
-            sha={selectedSha}
           />
+          // <JSONDetail
+          //   key={selectedSha}
+          //   filename={filename || ""}
+          //   owner={owner as string}
+          //   name={name as string}
+          //   previousSha={selectedShaPrevious}
+          //   sha={selectedSha}
+          // />
         )}
       </React.Fragment>
       {match &&

diff --git a/src/hooks/index.ts b/src/hooks/index.ts
@@ -10,6 +10,7 @@ import {
   listCommitsResponse,
   fetchFilesFromRepo,
   fetchOrgRepos,
+  fetchRawDataFile,
 } from "../api";
 import { Repo, FlatDataTab, Repository } from "../types";
 import React from "react";
@@ -45,6 +46,21 @@ export function useDataFile(
   });
 }
 
+export function useRawDataFile(
+  params: FileParamsWithSHA,
+  config?: UseQueryOptions<string>
+) {
+  return useQuery(
+    ["raw-data", params],
+    async () => await fetchRawDataFile(params),
+    {
+      retry: false,
+      refetchOnWindowFocus: false,
+      ...config,
+    }
+  );
+}
+
 nprogress.configure({ showSpinner: false });
 
 export function useProgressBar(numFetching: number) {