-
Notifications
You must be signed in to change notification settings - Fork 17
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: duckdb-wasm query UI [DRAFT] #39
Draft
mattrothenberg
wants to merge
19
commits into
main
Choose a base branch
from
mr/duckdb-spike
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Draft
Changes from 13 commits
Commits
Show all changes
19 commits
Select commit
Hold shift + click to select a range
fa9d83a
wasm spike successful
713488f
auto fetch
4127796
wip
6f75755
remove janky require shim
c225595
remove vite config changes
12955b2
error handling
e0b17f4
fix stale transaction error
abc4d0b
pull in grid component
c66c2a2
rudimentary toggle UI
54376d9
Update src/components/repo-detail.tsx
mattrothenberg c558164
refactor
99055d6
remove co-located duckdb package
5efc418
use filename as table name
c848a11
handle SQL errors
7c5a885
move SQL out to query param
93c654f
fix: conditionally insert CSV or JSON
7fff932
tweaks
c83b98c
switch statement for db initialization
62bdc9f
suppress fade animation for SQL results
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,215 @@ | ||
| import { sql } from "@codemirror/lang-sql"; | ||
| import { Grid } from "@githubocto/flat-ui"; | ||
| import CodeMirror from "@uiw/react-codemirror"; | ||
| import React, { useEffect, useMemo, useRef, useState } from "react"; | ||
| import { useQuery } from "react-query"; | ||
| import { useDebounce } from "use-debounce"; | ||
| import Bug from "../bug.svg"; | ||
| import { useDataFile } from "../hooks"; | ||
| import { ErrorState } from "./error-state"; | ||
| import { LoadingState } from "./loading-state"; | ||
| import { Spinner } from "./spinner"; | ||
|
|
||
| import * as duckdb from "@duckdb/duckdb-wasm"; | ||
| import duckdb_wasm from "@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url"; | ||
| import duckdb_wasm_next from "@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url"; | ||
| import eh_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url"; | ||
|
|
||
| interface Props { | ||
| sha: string; | ||
| filename: string; | ||
| owner: string; | ||
| name: string; | ||
| } | ||
|
|
||
| interface DBExplorerInnerProps { | ||
| content: string; | ||
| filename: string; | ||
| extension: string; | ||
| sha: string; | ||
| } | ||
|
|
||
| const VALID_EXTENSIONS = ["csv", "json"]; | ||
|
|
||
| function DBExplorerInner(props: DBExplorerInnerProps) { | ||
| const { content, extension, filename, sha } = props; | ||
| const filenameWithoutExtension = filename.split(".").slice(0, -1).join("."); | ||
| const connectionRef = useRef<duckdb.AsyncDuckDBConnection | null>(null); | ||
| const [query, setQuery] = useState(""); | ||
| const [debouncedQuery] = useDebounce(query, 500); | ||
| const [dbStatus, setDbStatus] = useState<"error" | "idle" | "success">( | ||
| "idle" | ||
| ); | ||
|
|
||
| const execQuery = async (query: string) => { | ||
| if (!connectionRef.current) return; | ||
| const queryRes = await connectionRef.current.query(query); | ||
| const asArray = queryRes.toArray(); | ||
| return { | ||
| numRows: queryRes.numRows, | ||
| numCols: queryRes.numCols, | ||
| results: asArray.map((row) => { | ||
| return row.toJSON(); | ||
mattrothenberg marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| }), | ||
| }; | ||
| }; | ||
|
|
||
| const { data, status, error } = useQuery( | ||
| ["query-results", filename, sha, debouncedQuery], | ||
| () => execQuery(debouncedQuery), | ||
| { | ||
| refetchOnWindowFocus: false, | ||
| retry: false, | ||
| } | ||
| ); | ||
|
|
||
| useEffect(() => { | ||
| const initDuckDb = async () => { | ||
| const MANUAL_BUNDLES: duckdb.DuckDBBundles = { | ||
| mvp: { | ||
| mainModule: duckdb_wasm, | ||
| mainWorker: eh_worker, | ||
| }, | ||
| eh: { | ||
| mainModule: duckdb_wasm_next, | ||
| mainWorker: eh_worker, | ||
| }, | ||
| }; | ||
| const bundle = await duckdb.selectBundle(MANUAL_BUNDLES); | ||
| const worker = new Worker(bundle.mainWorker!); | ||
| const logger = new duckdb.ConsoleLogger(); | ||
| const db = new duckdb.AsyncDuckDB(logger, worker); | ||
| await db.instantiate(bundle.mainModule, bundle.pthreadWorker); | ||
|
|
||
| const c = await db.connect(); | ||
| connectionRef.current = c; | ||
|
|
||
| try { | ||
| await db.registerFileText(filename, content); | ||
| await c.insertCSVFromPath(filename, { name: filenameWithoutExtension }); | ||
| setDbStatus("success"); | ||
| setQuery(`select * from '${filenameWithoutExtension}'`); | ||
mattrothenberg marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } catch { | ||
| setDbStatus("error"); | ||
| } | ||
| }; | ||
|
|
||
| initDuckDb(); | ||
|
|
||
| return () => { | ||
| if (connectionRef.current) { | ||
| connectionRef.current.close(); | ||
| connectionRef.current = null; | ||
| setDbStatus("idle"); | ||
| } | ||
| }; | ||
| }, [content, sha, filename]); | ||
|
|
||
| const sqlSchema = useMemo(() => { | ||
| if (!content) return []; | ||
|
|
||
| if (extension === "csv") { | ||
| const names = content.split("\n")[0].split(","); | ||
| return names.map((name) => name.replace(/"/g, "")); | ||
| } else if (extension === "json") { | ||
| try { | ||
| return Object.keys(JSON.parse(content)[0]); | ||
| } catch { | ||
| return []; | ||
| } | ||
| } else { | ||
| return []; | ||
| } | ||
| }, [content]); | ||
|
|
||
| return ( | ||
| <div className="flex-1 flex-shrink-0 overflow-hidden flex flex-col z-0"> | ||
| {dbStatus === "idle" && <LoadingState text="Initializing DuckDB 🦆" />} | ||
| {dbStatus === "error" && ( | ||
| <ErrorState img={Bug} alt="Database initialization error"> | ||
| Couldn't initialize DuckDB 😕 | ||
| </ErrorState> | ||
| )} | ||
| {dbStatus === "success" && ( | ||
| <> | ||
| <div className="border-b bg-gray-50 sticky top-0 z-20"> | ||
| <CodeMirror | ||
| value={query} | ||
| height={"120px"} | ||
| className="w-full" | ||
| extensions={[ | ||
| sql({ | ||
| defaultTable: "data", | ||
| schema: { | ||
| data: sqlSchema, | ||
| }, | ||
| }), | ||
| ]} | ||
| onChange={(value) => { | ||
| setQuery(value); | ||
| }} | ||
| /> | ||
| </div> | ||
| <div className="flex-1 flex flex-col h-full overflow-scroll"> | ||
| {status === "error" && error && ( | ||
| <div className="bg-red-50 border-b border-red-600 p-2 text-sm text-red-600"> | ||
| {(error as Error)?.message || "An unexpected error occurred."} | ||
| </div> | ||
| )} | ||
| <div className="relative flex-1 h-full"> | ||
| {status === "loading" && ( | ||
| <div className="absolute top-4 right-4 z-20"> | ||
| <Spinner /> | ||
| </div> | ||
| )} | ||
| {data && ( | ||
| <Grid | ||
| data={data.results} | ||
| diffData={undefined} | ||
| defaultSort={undefined} | ||
| defaultStickyColumnName={undefined} | ||
| defaultFilters={{}} | ||
| downloadFilename={filename} | ||
| onChange={() => {}} | ||
| /> | ||
| )} | ||
| </div> | ||
| </div> | ||
| </> | ||
| )} | ||
| </div> | ||
| ); | ||
| } | ||
|
|
||
| export function DBExplorer(props: Props) { | ||
| const { sha, filename, owner, name } = props; | ||
| const { data, status } = useDataFile( | ||
| { | ||
| sha, | ||
| filename, | ||
| owner, | ||
| name, | ||
| }, | ||
| { | ||
| refetchOnWindowFocus: false, | ||
| retry: false, | ||
| } | ||
| ); | ||
|
|
||
| const extension = filename.split(".").pop() || ""; | ||
| const content = data ? data[0].content : ""; | ||
|
|
||
| return ( | ||
| <> | ||
| {status === "loading" && <LoadingState />} | ||
| {status === "success" && data && VALID_EXTENSIONS.includes(extension) && ( | ||
| <DBExplorerInner | ||
| sha={sha} | ||
| filename={filename} | ||
| extension={extension} | ||
| content={content} | ||
| /> | ||
| )} | ||
| </> | ||
| ); | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You don't need to convert the result to an array. Arrow tables already behave like arrays of objects if you iterate over them for
for.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@domoritz Fantastic! This felt kludgy and I suspected there was a better way.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
mapandforEachwon't work so you may have to refactor your code a bit but it would be worth it since you avoid a lot of upfront conversion.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@domoritz Good call. We're ultimately passing this data down to our
flat-uicomponent , so it needs to be in a shape that this component understands (or we need to refactor the component)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You may need to refactor the component. Shouldn't be a big deal, though, and would be great for interoperability anyway.