fix: GPT-3.5 tokenizer

clemlesne · Aug 17, 2024 · 09fab9f · 09fab9f
1 parent bd50c32
commit 09fab9f
Show file tree

Hide file tree

Showing 3 changed files with 100,264 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -87,6 +87,8 @@ build:
 	pyinstaller \
 		--add-data resources:resources \
 		--clean \
+		--hidden-import=tiktoken_ext \
+		--hidden-import=tiktoken_ext.openai_public \
 		--icon resources/logo.ico \
 		--name scrape-it-now \
 		--onefile \

diff --git a/app/index.py b/app/index.py
@@ -1,4 +1,5 @@
 import asyncio, math
+from os import environ as env
 
 import tiktoken
 from azure.core.exceptions import (
@@ -30,6 +31,7 @@
     hash_url,
     index_index_name,
     index_queue_name,
+    resources_dir,
     scrape_container_name,
 )
 from app.helpers.threading import run_workers
@@ -355,6 +357,10 @@ async def run(
 ) -> None:
     logger.info("Starting indexing job %s", job)
 
+    # Patch Tiktoken
+    # See: https://stackoverflow.com/a/76107077
+    env["TIKTOKEN_CACHE_DIR"] = resources_dir("tiktoken")
+
     run_workers(
         azure_openai_api_key=azure_openai_api_key,
         azure_openai_embedding_deployment=azure_openai_embedding_deployment,