From 6918b909237c5c748f33c14ce24986578a1b6f00 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Wed, 12 Jun 2024 21:52:38 +0530 Subject: [PATCH 1/8] fix querry filter in list task endpoint for none type values --- pro_tes/ga4gh/tes/task_runs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pro_tes/ga4gh/tes/task_runs.py b/pro_tes/ga4gh/tes/task_runs.py index 308aec1..c4dd48e 100644 --- a/pro_tes/ga4gh/tes/task_runs.py +++ b/pro_tes/ga4gh/tes/task_runs.py @@ -272,16 +272,19 @@ def list_tasks(self, **kwargs) -> dict: ) page_token = kwargs.get("page_token") filter_dict = {} - filter_dict["user_id"] = kwargs.get("user_id") + + user_id = kwargs.get("user_id") + if user_id is not None: + filter_dict["user_id"] = user_id if page_token is not None: filter_dict["_id"] = {"$lt": ObjectId(page_token)} view = kwargs.get("view", "BASIC") projection = self._set_projection(view=view) - name_prefix: str = str(kwargs.get("name_prefix")) - + name_prefix = kwargs.get("name_prefix") if name_prefix is not None: + name_prefix: str = str(name_prefix) filter_dict["task_original.name"] = {"$regex": f"^{name_prefix}"} cursor = ( From c6ee9ddd74770cbf75b92ba6f1d6da36b65152e3 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Tue, 18 Jun 2024 13:36:15 +0530 Subject: [PATCH 2/8] removed redundant type casting --- pro_tes/ga4gh/tes/task_runs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pro_tes/ga4gh/tes/task_runs.py b/pro_tes/ga4gh/tes/task_runs.py index c4dd48e..5001fa5 100644 --- a/pro_tes/ga4gh/tes/task_runs.py +++ b/pro_tes/ga4gh/tes/task_runs.py @@ -284,7 +284,6 @@ def list_tasks(self, **kwargs) -> dict: name_prefix = kwargs.get("name_prefix") if name_prefix is not None: - name_prefix: str = str(name_prefix) filter_dict["task_original.name"] = {"$regex": f"^{name_prefix}"} cursor = ( From c0fb758db57b45e83085027e601a9276cec76282 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Sat, 22 Jun 2024 16:31:31 +0530 Subject: [PATCH 3/8] ignore collection type error --- pro_tes/utils/db.py | 51 +++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/pro_tes/utils/db.py b/pro_tes/utils/db.py index 9e8cd40..76aeee0 100644 --- a/pro_tes/utils/db.py +++ b/pro_tes/utils/db.py @@ -23,17 +23,17 @@ class DbDocumentConnector: """ def __init__( - self, - collection: Collection, - worker_id: str, + self, + collection: Collection, # type: ignore + worker_id: str, ) -> None: """Construct object instance.""" - self.collection: Collection = collection + self.collection: Collection = collection # type: ignore self.worker_id: str = worker_id def get_document( - self, - projection: Optional[Mapping] = None, + self, + projection: Optional[Mapping] = None, ) -> DbDocument: """Get document associated with task. @@ -51,7 +51,7 @@ def get_document( """ if projection is None: projection = {"_id": False} - document_unvalidated = self.collection.find_one( + document_unvalidated = self.collection.find_one( # type: ignore filter={"worker_id": self.worker_id}, projection=projection, ) @@ -65,8 +65,8 @@ def get_document( return document def update_task_state( - self, - state: str = "UNKNOWN", + self, + state: str = "UNKNOWN", ) -> None: """Update task status. @@ -80,17 +80,17 @@ def update_task_state( TesState(state) except Exception as exc: raise ValueError(f"Unknown state: {state}") from exc - self.collection.find_one_and_update( + self.collection.find_one_and_update( # type: ignore {"worker_id": self.worker_id}, {"$set": {"task.state": state}}, ) logger.info(f"[{self.worker_id}] {state}") def upsert_fields_in_root_object( - self, - root: str, - projection: Optional[Mapping] = None, - **kwargs: object, + self, + root: str, + projection: Optional[Mapping] = None, + **kwargs: object, ) -> DbDocument: """Insert or update fields in(to) the same root (object) field. @@ -106,17 +106,18 @@ def upsert_fields_in_root_object( """ if projection is None: projection = {"_id": False} - document_unvalidated = self.collection.find_one_and_update( - {"worker_id": self.worker_id}, - { - "$set": { - ".".join([root, key]): value - for (key, value) in kwargs.items() - } - }, - projection=projection, - return_document=ReturnDocument.AFTER, - ) + document_unvalidated = ( + self.collection.find_one_and_update( # type: ignore + {"worker_id": self.worker_id}, + { + "$set": { + ".".join([root, key]): value + for (key, value) in kwargs.items() + } + }, + projection=projection, + return_document=ReturnDocument.AFTER, + )) try: document: DbDocument = DbDocument(**document_unvalidated) except Exception as exc: From c710ee4ddf221017c21a374fe9f87456f3bd38e2 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Sat, 22 Jun 2024 16:52:27 +0530 Subject: [PATCH 4/8] mongodb version update --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5565bc2..ed13acc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -36,7 +36,7 @@ services: - "5672:5672" mongodb: - image: mongo:3.2 + image: mongo:3.6 restart: unless-stopped volumes: - ${PROTES_DATA_DIR:-../data/pro_tes}/db:/data/db From c065aed2e2c7ba385739d4f67cc23a8a3134c5ef Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Sun, 23 Jun 2024 16:26:52 +0530 Subject: [PATCH 5/8] remove depricated insert method with insert_one --- pro_tes/ga4gh/tes/task_runs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pro_tes/ga4gh/tes/task_runs.py b/pro_tes/ga4gh/tes/task_runs.py index 5001fa5..55fa504 100644 --- a/pro_tes/ga4gh/tes/task_runs.py +++ b/pro_tes/ga4gh/tes/task_runs.py @@ -439,7 +439,7 @@ def _write_doc_to_db( ) document.worker_id = uuid() try: - self.db_client.insert(document.dict(exclude_none=True)) + self.db_client.insert_one(document.dict(exclude_none=True)) except DuplicateKeyError: continue assert document is not None From db9020610b522d93b3a8466375c0095f7b7b1c1e Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Wed, 26 Jun 2024 10:33:02 +0530 Subject: [PATCH 6/8] removed type:ignore --- pro_tes/utils/db.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/pro_tes/utils/db.py b/pro_tes/utils/db.py index 76aeee0..e07b4b9 100644 --- a/pro_tes/utils/db.py +++ b/pro_tes/utils/db.py @@ -2,9 +2,7 @@ import logging from typing import Mapping, Optional -from pymongo.collection import ReturnDocument # type: ignore -from pymongo import collection as Collection # type: ignore - +from pymongo.collection import ReturnDocument, Collection from pro_tes.ga4gh.tes.models import DbDocument, TesState logger = logging.getLogger(__name__) @@ -23,17 +21,17 @@ class DbDocumentConnector: """ def __init__( - self, - collection: Collection, # type: ignore - worker_id: str, + self, + collection: Collection, + worker_id: str, ) -> None: """Construct object instance.""" - self.collection: Collection = collection # type: ignore + self.collection: Collection = collection self.worker_id: str = worker_id def get_document( - self, - projection: Optional[Mapping] = None, + self, + projection: Optional[Mapping] = None, ) -> DbDocument: """Get document associated with task. @@ -51,12 +49,12 @@ def get_document( """ if projection is None: projection = {"_id": False} - document_unvalidated = self.collection.find_one( # type: ignore + document_unvalidated = self.collection.find_one( filter={"worker_id": self.worker_id}, projection=projection, ) try: - document: DbDocument = DbDocument(**document_unvalidated) + document: DbDocument = DbDocument(**(document_unvalidated or {})) except Exception as exc: raise ValueError( "Database document does not conform to schema: " @@ -65,8 +63,8 @@ def get_document( return document def update_task_state( - self, - state: str = "UNKNOWN", + self, + state: str = "UNKNOWN", ) -> None: """Update task status. @@ -80,17 +78,17 @@ def update_task_state( TesState(state) except Exception as exc: raise ValueError(f"Unknown state: {state}") from exc - self.collection.find_one_and_update( # type: ignore + self.collection.find_one_and_update( {"worker_id": self.worker_id}, {"$set": {"task.state": state}}, ) logger.info(f"[{self.worker_id}] {state}") def upsert_fields_in_root_object( - self, - root: str, - projection: Optional[Mapping] = None, - **kwargs: object, + self, + root: str, + projection: Optional[Mapping] = None, + **kwargs: object, ) -> DbDocument: """Insert or update fields in(to) the same root (object) field. @@ -107,7 +105,7 @@ def upsert_fields_in_root_object( if projection is None: projection = {"_id": False} document_unvalidated = ( - self.collection.find_one_and_update( # type: ignore + self.collection.find_one_and_update( {"worker_id": self.worker_id}, { "$set": { From aed082eb4ce245bf21d92e47c0b070cc64688fd0 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Wed, 3 Jul 2024 12:58:37 +0530 Subject: [PATCH 7/8] fixed indent --- pro_tes/utils/db.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pro_tes/utils/db.py b/pro_tes/utils/db.py index e07b4b9..9a581a6 100644 --- a/pro_tes/utils/db.py +++ b/pro_tes/utils/db.py @@ -104,18 +104,19 @@ def upsert_fields_in_root_object( """ if projection is None: projection = {"_id": False} - document_unvalidated = ( - self.collection.find_one_and_update( - {"worker_id": self.worker_id}, - { - "$set": { - ".".join([root, key]): value - for (key, value) in kwargs.items() - } - }, - projection=projection, - return_document=ReturnDocument.AFTER, - )) + document_unvalidated = self.collection.find_one_and_update( + { + "worker_id": self.worker_id + }, + { + "$set": { + ".".join([root, key]): value for (key, value) in + kwargs.items() + } + }, + projection=projection, + return_document=ReturnDocument.AFTER, + ) try: document: DbDocument = DbDocument(**document_unvalidated) except Exception as exc: From 5ac0f3e4d49edc5e9143b2440c780d2f6019ffb3 Mon Sep 17 00:00:00 2001 From: Ayush5120 Date: Wed, 3 Jul 2024 13:01:02 +0530 Subject: [PATCH 8/8] fixed indent --- pro_tes/utils/db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pro_tes/utils/db.py b/pro_tes/utils/db.py index 9a581a6..511e9cf 100644 --- a/pro_tes/utils/db.py +++ b/pro_tes/utils/db.py @@ -105,9 +105,7 @@ def upsert_fields_in_root_object( if projection is None: projection = {"_id": False} document_unvalidated = self.collection.find_one_and_update( - { - "worker_id": self.worker_id - }, + {"worker_id": self.worker_id}, { "$set": { ".".join([root, key]): value for (key, value) in