feat: local deployment

Author: 0xkenj1

.dockerignore

@@ -0,0 +1,10 @@
+# Ignore all .env files in any directory or subdirectory
+**/.env
+
+# Ignore node_modules and dist directories in any directory or subdirectory
+**/node_modules
+**/dist
+
+**/.git
+**/.turbo
+**/*.md

Dockerfile

@@ -0,0 +1,17 @@
+# Based on example: 
+FROM node:20-slim AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+FROM base AS build
+COPY . /usr/src/app
+WORKDIR /usr/src/app
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm run -r build
+RUN pnpm deploy --filter=./apps/processing --prod /prod/processing
+
+FROM base AS processing
+COPY --from=build /prod/processing /prod/processing
+WORKDIR /prod/processing
+CMD [ "pnpm", "start" ]

README.md

@@ -10,7 +10,7 @@ This repository is a monorepo that contains 1 package and 2 applications:
 -   @ts-turborepo-boilerplate/sample-app: A demo application demonstrating the use of sample-lib.
 -   @grants-stack-indexer/indexer: An [`envio`](https://docs.envio.dev/) indexer, that collects all the relevant events from Allo contracts.
 
-## 🚀 Local Deployment (Dockerized)
+## 🚀 Local Deployment (Development)
 
 ### Prerequisites
 
@@ -19,15 +19,27 @@ Ensure you have the following installed on your machine:
 -   [Docker](https://www.docker.com/get-started)
 -   [Docker Compose](https://docs.docker.com/compose/install/)
 
-### Run
+### Setup
 
-To deploy locally with Docker, run the following command:
+1. Build and start the services in detached mode:
 
 ```
 docker-compose up -d --build
 ```
 
-Once the deployment is complete, you can access Hasura by navigating to:
+2. After starting Docker Compose, run the following command to apply the database migrations:
+
+```
+pnpm script:db:migrate
+```
+
+3. Navigate to the processing service directory and start it with:
+
+```
+cd apps/processing && pnpm dev
+```
+
+Once the setup is completed you can access Hasura by navigating to:
 
 ```
 http://localhost:8080/

apps/indexer/package.json

@@ -13,7 +13,7 @@
     },
     "dependencies": {
         "chai": "4.3.10",
-        "envio": "2.7.2",
+        "envio": "2.7.3",
         "ethers": "6.8.0",
         "yaml": "2.5.1"
     },

apps/indexer/pnpm-lock.yaml

@@ -0,0 +1,5 @@
+node_modules
+.git
+.gitignore
+*.md
+generated

apps/processing/.dockerignore

@@ -0,0 +1,11 @@
+FROM node:20-slim AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+COPY . /app
+WORKDIR /app
+
+RUN --mount=type=cache,id=pnpm,target=$PNPM_HOME/store pnpm install --frozen-lockfile
+RUN pnpm dlx envio codegen
+
+CMD pnpm dlx envio local db-migrate up && pnpm dlx envio start

apps/processing/Dockerfile

@@ -4,8 +4,9 @@ services:
         restart: always
         ports:
             - "${DATALAYER_POSTGRES_EXPOSED_PORT:-5432}:5432"
-        volumes:
-            - db_data:/var/lib/datalayer-postgresql/data
+        # TODO: remove the comment when processing service persists latest event processed
+        # volumes:
+        #     - db_data:/var/lib/datalayer-postgres/data
         env_file:
             - .env
         environment:
@@ -19,8 +20,7 @@ services:
             interval: 5s
             timeout: 5s
             retries: 5
-
-    api-hasura:
+    datalayer-graphql:
         image: hasura/graphql-engine:v2.44.0
         ports:
             - "${DATALAYER_HASURA_EXPOSED_PORT:-8082}:8080"
@@ -56,29 +56,33 @@ services:
             start_period: 5s
         networks:
             - datalayer
-
-    envio-postgres:
+    indexer-postgres:
         image: postgres:16
         restart: always
         ports:
             - "${ENVIO_POSTGRES_EXPOSED_PORT:-5433}:5432"
         volumes:
-            - db_data:/var/lib/envio-postgresql/data
+            - db_data:/var/lib/indexer-postgres/data
         env_file:
             - .env
         environment:
             POSTGRES_DB: ${ENVIO_PG_DATABASE}
             POSTGRES_USER: ${ENVIO_PG_USER}
             POSTGRES_PASSWORD: ${ENVIO_POSTGRES_PASSWORD}
+        healthcheck:
+            test: ["CMD-SHELL", "pg_isready -U ${DATALAYER_PG_USER}"]
+            interval: 5s
+            timeout: 5s
+            retries: 5
         networks:
-            - indexer-service
-    graphql-engine:
+            - indexer
+    indexer-graphql:
         image: hasura/graphql-engine:v2.23.0
         ports:
             - "${HASURA_EXPOSED_PORT:-8080}:${PORT:-8080}"
         user: 1001:1001
         depends_on:
-            - "envio-postgres"
+            - "indexer-postgres"
         restart: always
         env_file:
             - .env
@@ -89,24 +93,26 @@ services:
             retries: 50
             start_period: 5s
         networks:
-            - indexer-service
+            - indexer
     indexer-service:
         build:
             context: ./apps/indexer
             dockerfile: Dockerfile
-        depends_on:
-            - "envio-postgres"
-            - "graphql-engine"
         restart: always
+        depends_on:
+            indexer-postgres:
+                condition: service_healthy
+            indexer-graphql:
+                condition: service_healthy
         env_file:
             - .env
         networks:
-            - indexer-service
+            - indexer
 volumes:
     db_data:
     ganache-data:
 networks:
-    indexer-service:
+    indexer:
         name: indexer_test_network
     datalayer:
         name: datalayer_test_network

docker-compose.yaml

@@ -100,6 +100,7 @@ export class Orchestrator {
                     await delay(this.fetchDelayInMs);
                     continue;
                 }
+                await this.eventsRegistry.saveLastProcessedEvent(event);
 
                 event = await this.enhanceStrategyId(event);
                 if (event.contractName === "Strategy" && "strategyId" in event) {
@@ -126,7 +127,6 @@ export class Orchestrator {
                         )}`,
                     );
                 }
-                await this.eventsRegistry.saveLastProcessedEvent(event);
             } catch (error: unknown) {
                 // TODO: improve error handling, retries and notify
                 if (

packages/data-flow/src/orchestrator.ts

@@ -502,7 +502,7 @@ describe("Orchestrator", { sequential: true }, () => {
 
             expect(eventsProcessorSpy).toHaveBeenCalledTimes(2);
             expect(orchestrator["dataLoader"].applyChanges).toHaveBeenCalledTimes(1);
-            expect(mockEventsRegistry.saveLastProcessedEvent).toHaveBeenCalledTimes(1);
+            expect(mockEventsRegistry.saveLastProcessedEvent).toHaveBeenCalledTimes(2);
             expect(consoleSpy).toHaveBeenCalledTimes(1);
             expect(consoleSpy).toHaveBeenCalledWith(
                 expect.stringContaining(`Error processing event: ${stringify(errorEvent)}`),

packages/data-flow/test/unit/orchestrator.spec.ts

@@ -32,6 +32,7 @@ export class EnvioIndexerClient implements IIndexerClient {
                         $limit: Int!
                     ) {
                         raw_events(
+                            order_by: [{ block_number: asc }, { log_index: asc }]
                             where: {
                                 chain_id: { _eq: $chainId }
                                 _or: [
@@ -69,7 +70,7 @@ export class EnvioIndexerClient implements IIndexerClient {
             if (error instanceof InvalidIndexerResponse) {
                 throw error;
             }
-            throw new IndexerClientError(JSON.stringify(error));
+            throw new IndexerClientError(JSON.stringify(error, Object.getOwnPropertyNames(error)));
         }
     }
 }