use tiktoken

agnaistic · Sep 24, 2024 · a0f74df · a0f74df
1 parent f93416e
commit a0f74df
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 410,576 deletions.
diff --git a/package.json b/package.json
@@ -93,6 +93,7 @@
     "google-auth-library": "^9.11.0",
     "gpt-3-encoder": "^1.1.4",
     "iconv-lite": "0.6.3",
+    "js-tiktoken": "^1.0.14",
     "jsonwebtoken": "^9.0.0",
     "localtunnel": "^2.0.2",
     "minimist": "^1.2.8",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml