feat: create local client node to process jobs queue

.github/workflows/ai-bot-pull-request.yml

@@ -0,0 +1,99 @@
+name: API - Pull Request
+
+on:
+  pull_request:
+    branches:
+      - daily
+      - release/alpha
+      - master
+    paths:
+      - ".github/workflows/**"
+      - "packages/@liexp/**/src/**"
+      - "services/ai-bot/**"
+
+jobs:
+  install:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/install-deps
+        with:
+          hash: ${{ hashFiles('**/pnpm-lock.yaml') }}
+
+  build-packages:
+    runs-on: ubuntu-latest
+    needs: [install]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/install-deps
+        with:
+          hash: ${{ hashFiles('**/pnpm-lock.yaml') }}
+
+      - uses: ./.github/actions/build-packages
+        with:
+          hash: ${{ github.sha }}
+
+  lint:
+    runs-on: ubuntu-latest
+    needs: [install, build-packages]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/install-deps
+        with:
+          hash: ${{ hashFiles('**/pnpm-lock.yaml') }}
+
+      - uses: ./.github/actions/build-packages
+        with:
+          hash: ${{ github.sha }}
+
+      - name: Lint code
+        run: pnpm ai-bot lint
+
+  build:
+    needs: [install, build-packages]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/install-deps
+        with:
+          hash: ${{ hashFiles('**/pnpm-lock.yaml') }}
+
+      - uses: ./.github/actions/build-packages
+        with:
+          hash: ${{ github.sha }}
+
+      - name: Run migrations
+        run: pnpm ai-bot build
+
+  # docker_build:
+  #   if: ${{ github.head_ref == 'release/alpha' }}
+  #   needs: [install, build-packages, lint, build]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - uses: ./.github/actions/install-deps
+  #       with:
+  #         hash: ${{ hashFiles('**/pnpm-lock.yaml') }}
+
+  #     - name: Log in to registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.actor }}
+  #         password: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
+
+  #     - name: Build API image
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: .
+  #         file: ./api.Dockerfile
+  #         push: false
+  #         tags: ${{ env.API_IMAGE_NAME }}

lies.exposed.code-workspace

@@ -40,6 +40,10 @@
       "name": "🚀 api",
       "path": "services/api",
     },
+    {
+      "name": "🚀 ai-bot",
+      "path": "services/ai-bot",
+    },
     {
       "name": "storybook",
       "path": "services/storybook",

package.json

@@ -8,6 +8,7 @@
   "author": "Andrea Ascari <[email protected]>",
   "scripts": {
     "admin-web": "pnpm --filter 'admin-web'",
+    "ai-bot": "pnpm --filter 'ai-bot'",
     "api": "pnpm --filter 'api'",
     "backend": "pnpm --filter '@liexp/backend'",
     "build": "pnpm -r run build",

services/api/src/providers/ai/langchain.provider.ts → packages/@liexp/backend/src/providers/ai/langchain.provider.ts

@@ -58,8 +58,9 @@ export const GetLangchainProvider = (
   opts: LangchainProviderOptions,
 ): LangchainProvider => {
   const chat = new ChatOpenAI({
-    model: "gpt-3.5-turbo",
-    // model: 'gpt-4',
+    // model: "gpt-3.5-turbo",
+    model: "gpt-4o",
+    // model: "text-embedding-ada-002",
     // model: 'all-MiniLM-L6-v2',
     temperature: 0,
     apiKey: "no-key-is-a-good-key",

packages/@liexp/core/src/env/utils.ts

@@ -1,5 +1,7 @@
 import * as path from "path";
+import D from "debug";
 import * as dotenv from "dotenv";
+import { type Either } from "fp-ts/lib/Either.js";
 
 export const loadENV = (
   root?: string,
@@ -18,3 +20,18 @@ export const loadENV = (
     override: override ?? false,
   });
 };
+
+export const loadAndParseENV =
+  <E, A>(parseENV: (i: unknown) => Either<E, A>) =>
+  (root: string): Either<E, A> => {
+    process.env.NODE_ENV = process.env.NODE_ENV ?? "development";
+
+    if (process.env.NODE_ENV === "development") {
+      loadENV(root, ".env.local");
+      loadENV(root, ".env");
+
+      D.enable(process.env.DEBUG ?? "*");
+    }
+
+    return parseENV(process.env);
+  };

packages/@liexp/shared/src/io/http/Queue.ts

@@ -1,4 +1,5 @@
 import * as t from "io-ts";
+import { UUID } from "io-ts-types/lib/UUID.js";
 import { optionFromUndefined } from "../Common/optionFromUndefined.js";
 import { ResourcesNames } from "./ResourcesNames.js";
 
@@ -15,6 +16,7 @@ export type QueueTypes = t.TypeOf<typeof QueueTypes>;
 export const Status = t.union([
   t.literal("pending"),
   t.literal("processing"),
+  t.literal("done"),
   t.literal("completed"),
   t.literal("failed"),
 ]);
@@ -24,7 +26,7 @@ export const GetQueueParams = t.type(
   {
     resource: QueueResourceNames,
     type: QueueTypes,
-    id: t.string,
+    id: UUID,
   },
   "GetQueueParams",
 );
@@ -40,10 +42,28 @@ export const GetQueueListQuery = t.type(
 );
 export type GetQueueListQuery = t.TypeOf<typeof GetQueueListQuery>;
 
+export const CreateQueueURLData = t.strict(
+  {
+    url: t.string,
+    result: t.union([t.string, t.undefined]),
+    type: t.union([t.literal("link"), t.literal("pdf"), t.undefined]),
+  },
+  "CreateQueueURLData",
+);
+export type CreateQueueURLData = t.TypeOf<typeof CreateQueueURLData>;
+
+export const CreateQueueTextData = t.strict(
+  { text: t.string, result: t.union([t.string, t.undefined]) },
+  "CreateQueueTextData",
+);
+export type CreateQueueTextData = t.TypeOf<typeof CreateQueueTextData>;
+
 export const CreateQueue = t.strict(
   {
-    id: t.string,
-    data: t.any,
+    id: UUID,
+    data: t.union([CreateQueueURLData, CreateQueueTextData]),
+    resource: QueueResourceNames,
+    type: QueueTypes,
   },
   "CreateQueue",
 );
@@ -53,8 +73,6 @@ export type CreateQueue = t.TypeOf<typeof CreateQueue>;
 export const Queue = t.strict(
   {
     ...CreateQueue.type.props,
-    resource: QueueResourceNames,
-    type: QueueTypes,
     status: Status,
     error: t.union([t.record(t.string, t.any), t.null]),
   },

packages/@liexp/shared/src/utils/task.utils.ts

@@ -3,15 +3,17 @@ import * as T from "fp-ts/lib/Task.js";
 import type * as TE from "fp-ts/lib/TaskEither.js";
 import { pipe } from "fp-ts/lib/function.js";
 
-export const throwTE = async <E, A>(te: TE.TaskEither<E, A>): Promise<A> => {
+async function throwTE<E, A>(te: TE.TaskEither<E, A>): Promise<A> {
   return te().then((rr) => {
     if (rr._tag === "Left") {
       // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
       return Promise.reject(rr.left);
     }
     return Promise.resolve(rr.right);
   });
-};
+}
+
+export { throwTE };
 
 /**
  *

pnpm-workspace.yaml

@@ -2,6 +2,7 @@
 packages:
   - "packages/@liexp/*"
   - "services/api"
+  - "services/ai-bot"
   - "services/web"
   - "services/admin-web"
   - "services/storybook"

services/ai-bot/.env

@@ -0,0 +1,2 @@
+API_URL=http://api.liexp.dev/v1
+LOCALAI_URL=http://localai.liexp.dev:8080/v1

services/ai-bot/eslint.config.js

@@ -0,0 +1,15 @@
+import baseEslintConfig from "@liexp/core/lib/eslint/base.config.js";
+import tseslint from "typescript-eslint";
+
+const eslintConfig = tseslint.config(...baseEslintConfig, {
+  files: ["bin/**/*.ts", "src/**/*.ts"],
+  ignores: ["**/*.d.ts"],
+  languageOptions: {
+    parserOptions: {
+      project: ["./tsconfig.json"],
+      tsconfigRootDir: import.meta.dirname,
+    },
+  },
+});
+
+export default eslintConfig;

services/ai-bot/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@liexp/ai-bot",
+  "version": "1.0.0",
+  "bin": {
+    "liexp-ai-bot": "./build/liexp-ai-bot.js"
+  },
+  "type": "module",
+  "imports": {
+    "#common/*": "./build/common/*",
+    "#flows/*": "./build/flows/*"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "bin:run": "node ./build/bin/cli.js",
+    "build": "tsc -b tsconfig.build.json",
+    "clean": "rm -rf build",
+    "dev": "node --watch build/run.js",
+    "dev:alpha": "DOTENV_CONFIG_PATH=.env.alpha pnpm dev",
+    "docker:dev": "pnpm dev",
+    "docker:worker-dev": "node --watch build/worker/index.js",
+    "lint": "eslint src",
+    "start": "node build/run.js"
+  },
+  "dependencies": {
+    "@liexp/core": "workspace:*",
+    "@liexp/shared": "workspace:*",
+    "@liexp/backend": "workspace:*",
+    "fp-ts": "^2",
+    "io-ts": "^2"
+  },
+  "devDependencies": {}
+}

services/ai-bot/scripts/install-global-bin.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e -x
+
+pnpm pack
+
+npm install -g liexp-ai-bot-*.tgz

services/ai-bot/src/common/error/index.ts

@@ -0,0 +1,24 @@
+import { type FSError } from "@liexp/backend/lib/providers/fs/fs.provider.js";
+import { type PuppeteerError } from "@liexp/backend/lib/providers/puppeteer.provider.js";
+import { type APIError } from "@liexp/shared/lib/io/http/Error/APIError.js";
+import { type PDFError } from "@liexp/shared/lib/providers/pdf/pdf.provider.js";
+import { IOError } from "ts-shared/lib/errors.js";
+
+export type ApiBotError =
+  | PuppeteerError
+  | PDFError
+  | FSError
+  | APIError
+  | IOError;
+
+export const toApiBotError = (e: unknown): ApiBotError => {
+  if (e instanceof IOError) {
+    return e;
+  }
+
+  return new IOError("Unknown error", {
+    kind: "ServerError",
+    meta: [e, JSON.stringify(e)],
+    status: "Unknown Error",
+  });
+};

services/ai-bot/src/context.ts

@@ -0,0 +1,30 @@
+import {
+  type LoggerContext,
+  type FSClientContext,
+  type HTTPProviderContext,
+  type PDFProviderContext,
+} from "@liexp/backend/lib/context/index.js";
+import { type LangchainProvider } from "@liexp/backend/lib/providers/ai/langchain.provider.js";
+import { type Endpoints } from "@liexp/shared/lib/endpoints/index.js";
+import { type EndpointsRESTClient } from "@liexp/shared/lib/providers/EndpointsRESTClient/types.js";
+import { type APIRESTClient } from "@liexp/shared/lib/providers/api-rest.provider.js";
+
+interface EndpointsRESTClientContext {
+  endpointsRESTClient: EndpointsRESTClient<Endpoints>;
+}
+
+interface APIRESTClientContext {
+  apiRESTClient: APIRESTClient;
+}
+
+interface LangchainContext {
+  langchain: LangchainProvider;
+}
+
+export type ClientContext = LangchainContext &
+  HTTPProviderContext &
+  PDFProviderContext &
+  APIRESTClientContext &
+  EndpointsRESTClientContext &
+  FSClientContext &
+  LoggerContext;