Add translation service

Dockerfile

@@ -1,22 +1,27 @@
-FROM python:3.12.3-alpine
-
-RUN apt-get update && apt-get -y -q --no-install-recommends install libgomp1
-RUN apt-get -y install git
-RUN mkdir -p /app/src /app/docker_volume
+FROM python:3.12.4-slim-bullseye
 
+RUN apt-get update && apt-get install make
 RUN addgroup --system python && adduser --system --group python
-RUN chown -R python:python /app
+RUN mkdir opt/app
+RUN chown -R python:python opt/app
 USER python
 
-ENV VIRTUAL_ENV=/app/venv
+ENV VIRTUAL_ENV=/opt/app/venv
 RUN python -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
+WORKDIR /opt/app
+
 COPY requirements.txt requirements.txt
 RUN pip install --upgrade pip
 RUN pip --default-timeout=1000 install -r requirements.txt
 
-WORKDIR /app
-COPY ./src ./src
+COPY ./data ./data
+COPY app.py app.py
+COPY worker_metadata.py worker_metadata.py
+COPY worker_paragraphs.py worker_paragraphs.py
+COPY worker_translations.py worker_translations.py
+COPY Makefile Makefile
 
-ENV PYTHONPATH "${PYTHONPATH}:/app/src"
+ENV PYTHONPATH "${PYTHONPATH}:/opt/app"
+ENTRYPOINT ["make", "-f", "/opt/app/Makefile", "start_inside_docker"]

Makefile

@@ -14,4 +14,18 @@ install_venv:
 start:
 	. .venv/bin/activate; python -m pip install --upgrade pip
 	. .venv/bin/activate; python -m pip install -r requirements.txt
-	. .venv/bin/activate; command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5051 & command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5056 & command python -m worker_metadata & command python -m worker_paragraphs
+	. .venv/bin/activate; command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5051 & \
+	command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5056 & \
+	command python -m worker_metadata & \
+	command python -m worker_paragraphs & \
+	command python -m worker_translations
+
+start_inside_docker:
+	command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5051 & \
+	command gunicorn -k uvicorn.workers.UvicornWorker app:app --bind 0.0.0.0:5056 & \
+	command python -m worker_metadata & \
+	command python -m worker_paragraphs & \
+	command python -m worker_translations
+
+docker:
+	docker compose up --build

README.md

@@ -1,10 +1,19 @@
 ### To run the dummy service: 
 
+```bash
+make docker 
+```
+
+### Alternatively, to run the dummy service:
+
+```bash
 make install_venv
 make start
+```
 
-### Dummy service for 
 
-pdf_paragraphs_extraction
+### Dummy service for 
 
-pdf_metadata_extraction
+1. pdf_paragraphs_extraction
+2. pdf_metadata_extraction
+3. docker-translation-service

data/Translation.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+
+class Translation(BaseModel):
+    text: str
+    language: str
+    success: bool
+    error_message: str

data/TranslationResponseMessage.py

@@ -0,0 +1,12 @@
+from data.Translation import Translation
+from data.TranslationTaskMessage import TranslationTaskMessage
+
+
+class TranslationResponseMessage(TranslationTaskMessage):
+    translations: list[Translation]
+
+
+if __name__ == '__main__':
+    t = TranslationTaskMessage(namespace="namespace", key="key", text="text", language_from="language_from", languages_to=["languages_to"])
+    a = TranslationResponseMessage(**t.dict(), translations=[])
+    print(a)

data/TranslationTaskMessage.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+
+
+class TranslationTaskMessage(BaseModel):
+    namespace: str
+    key: str | list[str]
+    text: str
+    language_from: str
+    languages_to: list[str]
+

docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  dummy_extractor_services:
+    container_name: dummy_extractor_services
+    init: true
+    restart: unless-stopped
+    build:
+      context: .
+      dockerfile: Dockerfile
+    network_mode: "host"

worker_metadata.py

@@ -82,6 +82,7 @@ def send_logs(self, task):
         self.logs_queue.sendMessage().message(log_message.dump()).execute()
 
     def subscribe_to_tasks_queue(self):
+        print("Metadata extractor queue processor started")
         while True:
             try:
                 self.task_queue.getQueueAttributes().exec_command()

worker_paragraphs.py

@@ -41,6 +41,7 @@ def process(self, id, message, rc, ts):
         return True
 
     def subscribe_to_tasks_queue(self):
+        print("Paragraphs queue processor started")
         while True:
             try:
                 self.task_queue.getQueueAttributes().exec_command()

worker_translations.py

@@ -0,0 +1,70 @@
+from time import sleep
+
+import redis
+from rsmq.consumer import RedisSMQConsumer
+from rsmq import RedisSMQ, cmd
+
+from data.Translation import Translation
+from data.TranslationResponseMessage import TranslationResponseMessage
+from data.TranslationTaskMessage import TranslationTaskMessage
+
+
+class QueueProcessor:
+    def __init__(self):
+        self.task_queue = RedisSMQ(
+            host="127.0.0.1",
+            port=6379,
+            qname="translations_tasks",
+        )
+
+        self.results_queue = RedisSMQ(
+            host="127.0.0.1",
+            port=6379,
+            qname="translations_results",
+        )
+
+    def process(self, id, message, rc, ts):
+        task = TranslationTaskMessage(**message)
+        print(task.dict())
+
+        translations: list[Translation] = [self.get_translation(task, language) for language in task.languages_to]
+        response = TranslationResponseMessage(
+            **task.dict(),
+            translations=translations,
+        )
+
+        self.results_queue.sendMessage(delay=5).message(response.dict()).execute()
+        return True
+
+    def subscribe_to_tasks_queue(self):
+        print("Translation queue processor started")
+        while True:
+            try:
+                self.task_queue.getQueueAttributes().exec_command()
+                self.results_queue.getQueueAttributes().exec_command()
+
+                redis_smq_consumer = RedisSMQConsumer(
+                    qname="translations_tasks",
+                    processor=self.process,
+                    host="127.0.0.1",
+                    port=6379,
+                )
+                redis_smq_consumer.run()
+            except redis.exceptions.ConnectionError:
+                sleep(20)
+            except cmd.exceptions.QueueDoesNotExist:
+                self.task_queue.createQueue().exceptions(False).execute()
+                self.results_queue.createQueue().exceptions(False).execute()
+
+    @staticmethod
+    def get_translation(translation_task_message: TranslationTaskMessage, language: str) -> Translation:
+        if language == "error":
+            return Translation(text="", language=language, success=False, error_message="service error")
+
+        text = f"[translation for {language}] {translation_task_message.text}"
+        return Translation(text=text, language=language, success=False, error_message="")
+
+
+if __name__ == "__main__":
+    queue_processor = QueueProcessor()
+    queue_processor.subscribe_to_tasks_queue()