chore: fix circular dependency

.github/workflows/test.yml

@@ -36,3 +36,6 @@ jobs:
         working-directory: ./packages/core
       - name: Run Type Check
         run: pnpm run type-check
+      - name: Run Circular Dependency Check
+        run: pnpm run circular-check
+        working-directory: ./packages/core

packages/core/jest.config.cjs

@@ -2,5 +2,5 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
-  testPathIgnorePatterns: ["/lib/"],
+  testPathIgnorePatterns: ["/lib/", "/node_modules/", "/dist/"],
 };

packages/core/package.json

@@ -81,6 +81,6 @@
     "modify-package-json": "node ./scripts/modify-package-json.mjs",
     "prepublish": "pnpm run modify-package-json && echo \"please cd ./dist and run pnpm publish\" && exit 1",
     "dev": "bunchee -w",
-    "circular-check": "madge --circular ./src/*.ts"
+    "circular-check": "madge -c ./src/index.ts"
   }
 }

packages/core/src/OutputParser.ts

@@ -1,20 +1,4 @@
-import { SubQuestion } from "./QuestionGenerator";
-
-/**
- * An OutputParser is used to extract structured data from the raw output of the LLM.
- */
-export interface BaseOutputParser<T> {
-  parse(output: string): T;
-  format(output: string): string;
-}
-
-/**
- * StructuredOutput is just a combo of the raw output and the parsed output.
- */
-export interface StructuredOutput<T> {
-  rawOutput: string;
-  parsedOutput: T;
-}
+import { BaseOutputParser, StructuredOutput, SubQuestion } from "./types";
 
 /**
  * Error class for output parsing. Due to the nature of LLMs, anytime we use LLM

packages/core/src/Prompt.ts

@@ -1,6 +1,5 @@
 import { ChatMessage } from "./llm/types";
-import { SubQuestion } from "./QuestionGenerator";
-import { ToolMetadata } from "./Tool";
+import { SubQuestion, ToolMetadata } from "./types";
 
 /**
  * A SimplePrompt is a function that takes a dictionary of inputs and returns a string.

packages/core/src/QueryEngine.ts

@@ -1,13 +1,8 @@
 import { NodeWithScore, TextNode } from "./Node";
-import {
-  BaseQuestionGenerator,
-  LLMQuestionGenerator,
-  SubQuestion,
-} from "./QuestionGenerator";
+import { LLMQuestionGenerator } from "./QuestionGenerator";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { QueryEngineTool, ToolMetadata } from "./Tool";
 import { Event } from "./callbacks/CallbackManager";
 import { randomUUID } from "./env";
 import { BaseNodePostprocessor } from "./postprocessors";
@@ -16,34 +11,15 @@ import {
   CompactAndRefine,
   ResponseSynthesizer,
 } from "./synthesizers";
-
-/**
- * Parameters for sending a query.
- */
-export interface QueryEngineParamsBase {
-  query: string;
-  parentEvent?: Event;
-}
-
-export interface QueryEngineParamsStreaming extends QueryEngineParamsBase {
-  stream: true;
-}
-
-export interface QueryEngineParamsNonStreaming extends QueryEngineParamsBase {
-  stream?: false | null;
-}
-
-/**
- * A query engine is a question answerer that can use one or more steps.
- */
-export interface BaseQueryEngine {
-  /**
-   * Query the query engine and get a response.
-   * @param params
-   */
-  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
-  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
-}
+import {
+  BaseQueryEngine,
+  BaseQuestionGenerator,
+  QueryEngineParamsNonStreaming,
+  QueryEngineParamsStreaming,
+  QueryEngineTool,
+  SubQuestion,
+  ToolMetadata,
+} from "./types";
 
 /**
  * A query engine that uses a retriever to query an index and then synthesizes the response.

packages/core/src/QuestionGenerator.ts

@@ -1,28 +1,18 @@
-import {
-  BaseOutputParser,
-  StructuredOutput,
-  SubQuestionOutputParser,
-} from "./OutputParser";
+import { SubQuestionOutputParser } from "./OutputParser";
 import {
   SubQuestionPrompt,
   buildToolsText,
   defaultSubQuestionPrompt,
 } from "./Prompt";
-import { ToolMetadata } from "./Tool";
 import { OpenAI } from "./llm/LLM";
 import { LLM } from "./llm/types";
-
-export interface SubQuestion {
-  subQuestion: string;
-  toolName: string;
-}
-
-/**
- * QuestionGenerators generate new questions for the LLM using tools and a user query.
- */
-export interface BaseQuestionGenerator {
-  generate(tools: ToolMetadata[], query: string): Promise<SubQuestion[]>;
-}
+import {
+  BaseOutputParser,
+  BaseQuestionGenerator,
+  StructuredOutput,
+  SubQuestion,
+  ToolMetadata,
+} from "./types";
 
 /**
  * LLMQuestionGenerator uses the LLM to generate new questions for the LLM using tools and a user query.

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -6,7 +6,7 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "../llm/azure";
-import { OpenAISession, getOpenAISession } from "../llm/openai";
+import { OpenAISession, getOpenAISession } from "../llm/open_ai";
 import { BaseEmbedding } from "./types";
 
 export const ALL_OPENAI_EMBEDDING_MODELS = {

packages/core/src/embeddings/types.ts

@@ -1,16 +1,6 @@
 import { BaseNode, MetadataMode } from "../Node";
 import { TransformComponent } from "../ingestion";
-import { similarity } from "./utils";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
+import { SimilarityType, similarity } from "./utils";
 
 export abstract class BaseEmbedding implements TransformComponent {
   similarity(

packages/core/src/embeddings/utils.ts

@@ -2,8 +2,17 @@ import _ from "lodash";
 import { ImageType } from "../Node";
 import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
 import { defaultFS } from "../env";
-import { VectorStoreQueryMode } from "../storage";
-import { SimilarityType } from "./types";
+import { VectorStoreQueryMode } from "../storage/vectorStore/types";
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
 
 /**
  * The similarity between two embeddings.

packages/core/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -4,14 +4,14 @@ import {
   defaultCondenseQuestionPrompt,
   messagesToHistoryStr,
 } from "../../Prompt";
-import { BaseQueryEngine } from "../../QueryEngine";
 import { Response } from "../../Response";
 import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
 import { ChatMessage, LLM } from "../../llm";
 import { extractText, streamReducer } from "../../llm/utils";
+import { BaseQueryEngine } from "../../types";
 import {
   ChatEngine,
   ChatEngineParamsNonStreaming,

packages/core/src/index.ts

@@ -10,7 +10,6 @@ export * from "./Response";
 export * from "./Retriever";
 export * from "./ServiceContext";
 export * from "./TextSplitter";
-export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
 export * from "./embeddings";
@@ -21,7 +20,7 @@ export * from "./ingestion";
 export * from "./llm";
 export * from "./nodeParsers";
 export * from "./postprocessors";
-export * from "./readers/AssemblyAI";
+export * from "./readers/AssemblyAIReader";
 export * from "./readers/CSVReader";
 export * from "./readers/DocxReader";
 export * from "./readers/HTMLReader";
@@ -33,3 +32,4 @@ export * from "./readers/SimpleMongoReader";
 export * from "./readers/base";
 export * from "./storage";
 export * from "./synthesizers";
+export type * from "./types";

packages/core/src/indices/BaseIndex.ts

@@ -1,5 +1,4 @@
 import { BaseNode, Document, jsonToNode } from "../Node";
-import { BaseQueryEngine } from "../QueryEngine";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
 import { randomUUID } from "../env";
@@ -8,6 +7,7 @@ import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";
 import { VectorStore } from "../storage/vectorStore/types";
 import { BaseSynthesizer } from "../synthesizers";
+import { BaseQueryEngine } from "../types";
 
 /**
  * The underlying structure of each index.

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -1,6 +1,6 @@
 import { BaseNode, Document, MetadataMode } from "../../Node";
 import { defaultKeywordExtractPrompt } from "../../Prompt";
-import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { RetrieverQueryEngine } from "../../QueryEngine";
 import { BaseRetriever } from "../../Retriever";
 import {
   ServiceContext,
@@ -13,6 +13,7 @@ import {
   storageContextFromDefaults,
 } from "../../storage";
 import { BaseSynthesizer } from "../../synthesizers";
+import { BaseQueryEngine } from "../../types";
 import {
   BaseIndex,
   BaseIndexInit,

packages/core/src/indices/summary/SummaryIndex.ts

@@ -1,6 +1,6 @@
 import _ from "lodash";
 import { BaseNode, Document } from "../../Node";
-import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { RetrieverQueryEngine } from "../../QueryEngine";
 import { BaseRetriever } from "../../Retriever";
 import {
   ServiceContext,
@@ -18,6 +18,7 @@ import {
   CompactAndRefine,
   ResponseSynthesizer,
 } from "../../synthesizers";
+import { BaseQueryEngine } from "../../types";
 import {
   BaseIndex,
   BaseIndexInit,

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -6,7 +6,7 @@ import {
   ObjectType,
   splitNodesByType,
 } from "../../Node";
-import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { RetrieverQueryEngine } from "../../QueryEngine";
 import { BaseRetriever } from "../../Retriever";
 import {
   ServiceContext,
@@ -26,6 +26,7 @@ import {
   storageContextFromDefaults,
 } from "../../storage";
 import { BaseSynthesizer } from "../../synthesizers";
+import { BaseQueryEngine } from "../../types";
 import {
   BaseIndex,
   BaseIndexInit,

packages/core/src/llm/LLM.ts

@@ -25,9 +25,9 @@ import {
   shouldUseAzure,
 } from "./azure";
 import { BaseLLM } from "./base";
-import { OpenAISession, getOpenAISession } from "./openai";
+import { OpenAISession, getOpenAISession } from "./open_ai";
 import { PortkeySession, getPortkeySession } from "./portkey";
-import { ReplicateSession } from "./replicate";
+import { ReplicateSession } from "./replicate_ai";
 import {
   ChatMessage,
   ChatResponse,

packages/core/src/llm/index.ts

@@ -1,5 +1,10 @@
 export * from "./LLM";
-export * from "./mistral";
+export {
+  ALL_AVAILABLE_MISTRAL_MODELS,
+  MistralAI,
+  MistralAISession,
+} from "./mistral";
 export { Ollama } from "./ollama";
+export * from "./open_ai";
 export { TogetherLLM } from "./together";
 export * from "./types";

packages/core/src/llm/replicate.ts → packages/core/src/llm/replicate_ai.ts

@@ -28,5 +28,3 @@ export function getReplicateSession(replicateKey: string | null = null) {
 
   return defaultReplicateSession;
 }
-
-export * from "openai";

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -4,7 +4,7 @@ import {
   getTopKEmbeddings,
   getTopKEmbeddingsLearner,
   getTopKMMREmbeddings,
-} from "../../embeddings";
+} from "../../embeddings/utils";
 import { defaultFS, path } from "../../env";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_PERSIST_DIR } from "../constants";

packages/core/src/tests/CallbackManager.test.ts

@@ -13,7 +13,7 @@ import { ResponseSynthesizer, SimpleResponseBuilder } from "../synthesizers";
 import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI";
 
 // Mock the OpenAI getOpenAISession function during testing
-jest.mock("../llm/openai", () => {
+jest.mock("../llm/open_ai", () => {
   return {
     getOpenAISession: jest.fn().mockImplementation(() => null),
   };

packages/core/src/tests/MetadataExtractors.test.ts

@@ -21,7 +21,7 @@ import {
 } from "./utility/mockOpenAI";
 
 // Mock the OpenAI getOpenAISession function during testing
-jest.mock("../llm/openai", () => {
+jest.mock("../llm/open_ai", () => {
   return {
     getOpenAISession: jest.fn().mockImplementation(() => null),
   };