Merge pull request #45 from undp/containerize-v1

Containerize v1

.dockerignore

@@ -0,0 +1 @@
+**/node_modules

.github/workflows/frontend-deployment.yml

@@ -15,6 +15,7 @@ jobs:
       COUNTRY_NAME: 'Antarctic Region'
       COUNTRY_FLAG_URL: 'https://carbon-common-dev.s3.amazonaws.com/flag.png'
       COUNTRY_CODE: 'NG'
+      MAPBOXGL_ACCESS_TOKEN: ${{ secrets.MAPBOXGL_ACCESS_TOKEN }}
     steps:
       - uses: actions/checkout@v3
       - name: Cache modules

.github/workflows/server-deployments.yml

@@ -5,7 +5,7 @@ on:
       - develop
       - main
     paths:
-      - lambda/**
+      - backend/**
       - libs/**
 
 env:
@@ -24,7 +24,7 @@ jobs:
         fetch-depth: 0
     - id: check_file_changed
       run: |
-        if [[ $(git diff --name-only HEAD^ HEAD | grep lambda/services/package.json) ]] || [[ $(git diff --name-only HEAD^ HEAD | grep libs/) ]]; then
+        if [[ $(git diff --name-only HEAD^ HEAD | grep backend/services/package.json) ]] || [[ $(git diff --name-only HEAD^ HEAD | grep libs/) ]]; then
           echo "deps_changed=True" >> $GITHUB_OUTPUT
         else
           echo "deps_changed=False" >> $GITHUB_OUTPUT
@@ -54,29 +54,29 @@ jobs:
         run: yarn run build
       - name: Copy package.json
         if: ${{ needs.check_dependency_change.outputs.deps_changed == 'True' }}
-        working-directory: ./lambda/layer/dependency_layer
+        working-directory: ./backend/layer/dependency_layer
         run: cp ../../services/package.json ./ && cp ../../services/yarn.lock ./ && cp -r ../../../libs ../../
       - name: Install Dependency
         if: ${{ needs.check_dependency_change.outputs.deps_changed == 'True' }}
-        working-directory: ./lambda/layer/dependency_layer
+        working-directory: ./backend/layer/dependency_layer
         run: yarn install --production --frozen-lockfile
       # - name: Copy Dependency
       #   if: ${{ needs.check_dependency_change.outputs.deps_changed == 'True' }}
-      #   working-directory: ./lambda/layer
+      #   working-directory: ./backend/layer
       #   run: |
       #     cp -r node_modules ./dependency_layer
       #     ls
       - name: serverless deploy develop
         if: ${{ needs.check_dependency_change.outputs.deps_changed == 'True' && github.ref == 'refs/heads/develop'}}
         uses: serverless/[email protected]
         with:
-          args: -c "cd ./lambda/layer && serverless deploy --stage dev"
+          args: -c "cd ./backend/layer && serverless deploy --stage dev"
           entrypoint: /bin/sh
       - name: serverless deploy main
         if: ${{ needs.check_dependency_change.outputs.deps_changed == 'True' && github.ref == 'refs/heads/main'}}
         uses: serverless/[email protected]
         with:
-          args: -c "cd ./lambda/layer && serverless deploy --stage prod"
+          args: -c "cd ./backend/layer && serverless deploy --stage prod"
           entrypoint: /bin/sh
   deploy:
     name: deploy
@@ -89,7 +89,7 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - name: Install Dependency
-        working-directory: ./lambda/services
+        working-directory: ./backend/services
         run: yarn install --frozen-lockfile
       - name: version
         run: |
@@ -106,13 +106,13 @@ jobs:
         if: github.ref == 'refs/heads/develop'
         uses: serverless/[email protected]
         with:
-          args: -c "cd ./lambda/services && serverless deploy --stage dev --param='depLayerArn1=${{ steps.layerArn.outputs.ARN1 }}' --param='depLayerArn2=${{ steps.layerArn.outputs.ARN2 }}'"
+          args: -c "cd ./backend/services && serverless deploy --stage dev --param='depLayerArn1=${{ steps.layerArn.outputs.ARN1 }}' --param='depLayerArn2=${{ steps.layerArn.outputs.ARN2 }}'"
           entrypoint: /bin/sh
       - name: serverless deploy main
         if: github.ref == 'refs/heads/main'
         uses: serverless/[email protected]
         with:
-          args: -c "cd ./lambda/services && serverless deploy --stage prod"
+          args: -c "cd ./backend/services && serverless deploy --stage prod"
           entrypoint: /bin/sh
 
   # automated-api-tests:

.gitignore

@@ -2,7 +2,9 @@
 **/.serverless/
 **/dist/
 **/.build/
-lambda/services/.env.local
+backend/services/.env.local
 **/yarn-error.log
 **/coverage
 .env.local
+./data/*
+!./data/README.md

.vscode/launch.json

@@ -8,7 +8,7 @@
         "name": "Launch Serverless",
         "request": "launch",
         "type": "pwa-node",
-        "cwd": "${workspaceFolder}/lambda/services/",
+        "cwd": "${workspaceFolder}/backend/services/",
         "program": "/usr/local/bin/serverless",
         "args": [
           "offline",

README.md

@@ -3,6 +3,7 @@
 * [Standards](#standards)
 * [Architecture](#architecture)
 * [Project Structure](#structure)
+* [Run as Containers](#container)
 * [Run Services Locally](#local)
 * [Run Services on Cloud](#cloud)
 * [User Onboarding](#user)
@@ -34,13 +35,12 @@ https://digitalprinciples.org/
 
 <a name="architecture"></a>
 ## System Architecture
-UNDP Carbon Registry is based on service oriented architecture (SOA). It can be ported and hosted on any Function As A Service (FaaS) stack.
-![alt text](./documention/imgs/System%20Architecture.svg)
+UNDP Carbon Registry is based on service oriented architecture (SOA). Following diagram visualize the basic components in the system.
 
-As per the above diagram, system contains 4 main services.
+![alt text](./documention/imgs/System%20Architecture.svg)
 
 <a name="services"></a>
-### **Services**
+### **System Services**
 #### *National Service*
 
 Authenticate, Validate and Accept user (Government, Programme Developer/Certifier) API requests related to the following functionalities,
@@ -58,20 +58,42 @@ Uses the Carbon Credit Calculator and Serial Number Generator node modules to es
 Uses Ledger interface to persist programme and credit life cycles.
 
 #### *Analytics Service*
-Serve all the system analytics. Generate all the statistic using the operational database. 
+Serve all the system analytics. Generate all the statistics using the operational database. 
 Horizontally scalable. 
 
 #### *Replicator Service*
-Replicate ledger database new items to a operational database asynchronously. During the replication process it injects additional query information to the data. 
-The current setup uses AWS QLDB as the ledger database. When it creates or updates data, the change is added to a AWS Kinesis Data Stream and the Replicator service consumes the stream.
+Asynchronously replicate ledger database events in to the operational database. During the replication process it injects additional information to the data for query purposes (Eg: Location information). 
+Currently implemented for QLDB and PostgresSQL ledgers. By implementing [replicator interface](./backend/services/src/ledger-replicator/replicator-interface.service.ts) can support more ledger replicators. 
+Replicator select based on the `LEDGER_TYPE` environment variable. Support types `QLDB`, `PGSQL(Default)`.
+
+### **Deployment**
+System services can deploy in 2 ways.
+- **As a Container** - Each service boundary containerized in to a docker container and can deploy on any container orchestration service. [Please refer Docker Compose file](./docker-compose.yml)
+- **As a Function** - Each service boundary packaged as a function (Serverless) and host on any Function As A Service (FaaS) stack. [Please refer Serverless configuration file](./backend/services/serverless.yml)
+
+
+### **External Service Providers**
+All the external services access through a generic interface. It will decouple the system implementation from the external services and enable extendability to multiple services.
+
+**Geo Location Service** 
 
-#### *Operational Service*
-Service that use to do following system operations,
-1. Data migrations.
-2. User data creation and update.
-3. Resource creation.
+Currently implemented for 2 options.
+1. File based approach. User has to manually add the regions with the geo coordinates. [Sample File](./backend/services/regions.csv). To apply new file changes, replicator service needs to restart. 
+2. [Mapbox](https://mapbox.com). Dynamically query geo coordinates from the Mapbox API. 
 
-Internal service. Cannot be invoked by external sources. 
+Can add more options by implementing [location interface](./backend/services/src/shared/location/location.interface.ts)
+
+Change by environment variable `LOCATION_SERVICE`. Supported types `MAPBOX`, `FILE(Default)`
+
+**File Service**
+
+Implemented 2 options for static file hosting.
+1. NestJS static file hosting using the local storage and container volumes.
+2. AWS S3 file storage.
+
+Can add more options by implementing [file handler interface](./backend/services/src/shared/file-handler/filehandler.interface.ts)
+
+Change by environment variable `FILE_SERVICE`. Supported types `S3`, `LOCAL(Default)`
 
 ### **Database Architecture**
 Primary/secondary database architecture used to store carbon programme and account balances. 
@@ -91,8 +113,9 @@ Operational Database is the secondary database. Eventually replicated to this fr
 
 **Ledger Database Interface**
 
-This enables the capability to add any blockchain or ledger database support to the carbon registry without functionality module changes. Currently the production system interface is implemented for AWS QLDB. For testing purposes the interface is implemented for PostgresSQL as well.
+This enables the capability to add any blockchain or ledger database support to the carbon registry without functionality module changes. Currently implemented for PostgresSQL and AWS QLDB.
 
+**PostgresSQL Ledger Implementation** storage all the carbon programme and credit events in a separate event database with the sequence number. Support all the ledger functionalities except immutability.  
 
 
 Single database approach used for user and company management. 
@@ -106,7 +129,7 @@ Carbon Registry contains 3 ledger tables.
 
 The below diagram demonstrates the the ledger behavior of programme create, authorise, issue and transfer processes. Blue color document icon denotes a single data block in a ledger.
 
-![alt text](./documention/imgs/Ledger.png)
+![alt text](./documention/imgs/Ledger.svg)
 
 ### **Authentication**
 - JWT Authentication - All endpoints based on role permissions.
@@ -118,13 +141,11 @@ The below diagram demonstrates the the ledger behavior of programme create, auth
     .
     ├── .github                         # CI/CD [Github Actions files]
     ├── deployment                      # Declarative configuration files for initial resource creation and setup [AWS Cloudformation]
-    ├── lambda                          # System service implementation [NestJS applications, Serverless + AWS Lambda]
-        ├── layer                       # Service dependency layer [AWS Lambda Layers]
-            ├── serverless.yml          # Service dependency layer deployment scripts [ Serverless + AWS Lambda Layer]
-        ├── services                    # Services implementation [AWS lambda - NestJS application]
+    ├── backend                         # System service implementation
+        ├── services                    # Services implementation [NestJS application]
             ├── src
                 ├── national-api        # National API [NestJS module]      
-                ├── stats-api           # Statistic API [NestJS module]
+                ├── stats-api           # Statistics API [NestJS module]
                 ├── ledger-replicator   # Blockchain Database data replicator [QLDB to Postgres]
                 ├── shared              # Shared resources [NestJS module]     
             ├── serverless.yml          # Service deployment scripts [Serverless + AWS Lambda]
@@ -133,8 +154,19 @@ The below diagram demonstrates the the ledger behavior of programme create, auth
         ├── serial-number-gen           # Implementation for the carbon programme serial number calculation [Node module + Typescript]
     ├── web                             # System web frontend implementation [ReactJS]
     ├── .gitignore
+    ├── docker-compose.yml              # Docker container definitions
     └── README.md
 
+<a name="container"></a>
+## Run Services As Containers
+- Update [docker compose file](./docker-compose.yml) env variables as required.
+- Run `docker-compose up -d`. This will build and start containers for following services,
+    - PostgresDB container
+    - National service
+    - Analytics service
+    - Replicator service
+    - React web server with Nginx. 
+
 <a name="local"></a>
 ## Run Services Locally
 - Setup postgreSQL locally and create a new database.
@@ -144,7 +176,7 @@ The below diagram demonstrates the the ledger behavior of programme create, auth
     - DB_USER (Default root)
     - DB_PASSWORD
     - DB_NAME (Default carbondbdev)
-- Move to folder `cd lambda/service`
+- Move to folder `cd backend/service`
 - Run `yarn run sls:install `
 - Initial user data setup `serverless invoke local --stage=local --function setup --data '{"rootEmail": "<Root user email>","systemCountryCode": "<System country Alpha 2 code>", "name": "<System country name>", "logoBase64": "<System country logo base64>"}'`
 - Start all the services by executing `sls offline --stage=local`

backend/services/.dockerignore

@@ -0,0 +1 @@
+**/node_modules

lambda/services/.env.dev → backend/services/.env.dev

@@ -2,8 +2,11 @@ STAGE=dev
 DB_HOST=carbondb.caslz2nn5xor.us-east-1.rds.amazonaws.com
 DB_PORT=5432
 DB_USER=root
-DB_PASSWORD=abcd1234
 DB_NAME=carbondbdev
 LOG_LEVEL=debug
 carbon_dev_common=carbon-www-common
-SOURCE_EMAIL=[email protected]
+SOURCE_EMAIL=[email protected]
+IS_EMAIL_DISABLED=true
+LEDGER_TYPE=QLDB
+FILE_SERVICE=S3
+LOCATION_SERVICE=MAPBOX

backend/services/Dockerfile

@@ -0,0 +1,22 @@
+# Base image
+FROM node:16-alpine
+
+# Create app directory
+WORKDIR /app/backend/services
+
+COPY ./backend/services/package.json ./
+COPY ./backend/services/yarn.lock ./
+
+COPY ./libs ../../libs
+
+# Install app dependencies
+RUN yarn run sls:installProd
+
+# Bundle app source
+COPY ./backend/services .
+
+ENV NODE_ENV production
+RUN yarn add @nestjs/cli --dev && yarn run build && yarn remove @nestjs/cli && yarn cache clean
+
+# Start the server using the production build
+CMD [ "node", "dist/main.js" ]

lambda/services/package.json → backend/services/package.json

@@ -8,8 +8,9 @@
   "scripts": {
     "prebuild": "rimraf dist",
     "build": "nest build",
-    "build:libs": "cd ../../libs/serial-number-gen/ && yarn install && yarn run build && cd ../../libs/carbon-credit-calculator && yarn install && yarn run devBuild && cd ../../lambda/services",
-    "sls:install": "yarn run build:libs && cd ../../lambda/services && yarn install --frozen-lockfile",
+    "build:libs": "cd ../../libs/serial-number-gen/ && yarn install && yarn run build && cd ../../libs/carbon-credit-calculator && yarn install && yarn run devBuild && cd ../../backend/services",
+    "sls:install": "yarn run build:libs && cd ../../backend/services && yarn install --frozen-lockfile",
+    "sls:installProd": "yarn run build:libs && cd ../../backend/services && yarn install --production --frozen-lockfile",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "nest start",
     "start:dev": "nest start --watch",

backend/services/public/README.md

@@ -0,0 +1 @@
+# Static file locations

backend/services/regions.csv

@@ -0,0 +1,38 @@
+Name,Country,Latitude,Longitude
+Abia,Nigeria,5.454095299,7.5153071
+Adamawa,Nigeria,9.512977199,12.3881887
+Akwa Ibom,Nigeria,4.9408638,7.8412267
+Anambra,Nigeria,6.218313599,6.9531842
+Bauchi,Nigeria,10.6228284,10.0287754
+Bayelsa,Nigeria,4.7629786,6.028898
+Benue,Nigeria,7.350574699,8.7772877
+Borno,Nigeria,12.1875392,13.3080034
+Cross River,Nigeria,5.867196599,8.5204774
+Delta,Nigeria,5.527306099,6.1784167
+Ebonyi,Nigeria,6.199691799,8.0348906
+Edo,Nigeria,6.607657499,5.9722713
+Ekiti,Nigeria,7.736890999,5.2738326
+Enugu,Nigeria,6.553609399,7.4143061
+Federal Capital Territory,Nigeria,8.831122799,7.1724673
+Gombe,Nigeria,10.38301,11.206567
+Imo,Nigeria,5.585945599,7.0669651
+Jigawa,Nigeria,12.3252362,9.5103296
+Kaduna,Nigeria,10.3825318,7.8533226
+Kano,Nigeria,11.8948389,8.5364136
+Katsina,Nigeria,12.5630825,7.6207063
+Kebbi,Nigeria,11.4167574,4.1074545
+Kogi,Nigeria,7.794960199,6.6868669
+Kwara,Nigeria,8.836789099,4.6688487
+Lagos,Nigeria,6.526903299,3.5774005
+Nasarawa,Nigeria,8.438786799,8.2382849
+Niger,Nigeria,9.932608299,5.6511088
+Ogun,Nigeria,6.978858199,3.4389293
+Ondo,Nigeria,7.020968599,5.0567477
+Osun,Nigeria,7.548404699,4.4978307
+Oyo,Nigeria,8.215124899,3.5642897
+Plateau,Nigeria,9.058344599,9.6826289
+Rivers,Nigeria,4.8416028,6.8604088
+Sokoto,Nigeria,13.0611195,5.3152203
+Taraba,Nigeria,8.014133399,10.7376336
+Yobe,Nigeria,12.1233242,11.5065937
+Zamfara,Nigeria,12.0078998,6.4191432

lambda/services/serverless.yml → backend/services/serverless.yml

@@ -22,6 +22,7 @@ package:
     - ".env"
     - .env.${self:provider.stage}
     - "countries.json"
+    - "regions.csv"
 
 plugins:
   - "serverless-plugin-typescript"
@@ -90,6 +91,7 @@ functions:
           enabled: true
     ssmToEnvironment:
       - DB_PASSWORD
+      - MAPBOX_PK
     environment:
       NODE_PATH: "./:/opt/node_modules"
     layers:
@@ -127,6 +129,7 @@ custom:
     USER_JWT_SECRET: /${self:provider.stage}/USER_JWT_SECRET~true
     ADMIN_JWT_SECRET: /${self:provider.stage}/ADMIN_JWT_SECRET~true
     SES_PASSWORD: /${self:provider.stage}/SES_PASSWORD~true
+    MAPBOX_PK: /${self:provider.stage}/MAPBOX_PK~true
   # sesTemplates:
   #   addStage: true                             # Specifies whether to add stage to template name (default false)
   #   configFile: './src/shared/email/email.template.ts' # Config file path (default './ses-email-templates/index.js')

lambda/services/src/ledger-replicator/handler.ts → backend/services/src/ledger-replicator/handler.ts

@@ -3,12 +3,12 @@ import { Handler, Context } from 'aws-lambda';
 import { Server } from 'http';
 import { NestFactory } from '@nestjs/core';
 import { LedgerReplicatorModule } from './ledger-replicator.module';
-import { LedgerReplicatorService } from './ledger-replicator.service';
 import { getLogger } from '../shared/server';
+import { LedgerReplicatorInterface } from './replicator-interface.service';
 
 export const handler: Handler = async (event: any, context: Context) => {
    const app = await NestFactory.createApplicationContext(LedgerReplicatorModule, {
       logger: getLogger(LedgerReplicatorModule),
     });
-    await app.get(LedgerReplicatorService).replicate(event);
+    await app.get(LedgerReplicatorInterface).replicate(event);
 }