support explicit ES mappings

toluaina · Jun 11, 2024 · 282bb1b · 282bb1b
1 parent c183c86
commit 282bb1b
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 15 deletions.
diff --git a/examples/book/schema.json b/examples/book/schema.json
@@ -2,6 +2,19 @@
     {
         "database": "book",
         "index": "book",
+        "mappings": {
+            "properties": {
+                "age": {
+                    "type": "integer"
+                },
+                "email": {
+                    "type": "keyword"
+                },
+                "name": {
+                    "type": "text"
+                }
+            }
+        },
         "mapping": {
             "authors": {
                 "type": "nested"
@@ -37,7 +50,14 @@
                 }
             }
         },
-        "plugins": ["Groot", "Hero", "Villain", "Geometry", "Infinity", "TextEmbedding3Small"],
+        "plugins": [
+            "Groot",
+            "Hero",
+            "Villain",
+            "Geometry",
+            "Infinity",
+            "TextEmbedding3Small"
+        ],
         "nodes": {
             "table": "book",
             "columns": [
@@ -71,7 +91,7 @@
                     },
                     "book_isbn": {
                         "type": "text",
-                        "fields":{
+                        "fields": {
                             "ngram": {
                                 "type": "text",
                                 "analyzer": "ngram_analyzer",
@@ -95,26 +115,38 @@
                         "variant": "object",
                         "type": "one_to_one",
                         "foreign_key": {
-                            "child": ["id"],
-                            "parent": ["publisher_id"]
+                            "child": [
+                                "id"
+                            ],
+                            "parent": [
+                                "publisher_id"
+                            ]
                         }
                     },
-                    "children": [
-
-                    ],
+                    "children": [],
                     "transform": {
                         "rename": {
                             "id": "publisher_id",
                             "name": "publisher_name"
-                        }, 
+                        },
                         "concat": [
                             {
-                                "columns": ["publisher_id", "publisher_name", "is_active", "foo"],
+                                "columns": [
+                                    "publisher_id",
+                                    "publisher_name",
+                                    "is_active",
+                                    "foo"
+                                ],
                                 "destination": "xyz",
                                 "delimiter": "_"
                             },
                             {
-                                "columns": ["publisher_id", "publisher_name", "is_active", "bar"],
+                                "columns": [
+                                    "publisher_id",
+                                    "publisher_name",
+                                    "is_active",
+                                    "bar"
+                                ],
                                 "destination": "abc",
                                 "delimiter": "-"
                             }
@@ -148,7 +180,9 @@
                 {
                     "table": "author",
                     "columns": [
-                        "id", "name", "date_of_birth"
+                        "id",
+                        "name",
+                        "date_of_birth"
                     ],
                     "label": "authors",
                     "relationship": {
@@ -256,4 +290,4 @@
             ]
         }
     }
-]
+]
diff --git a/pgsync/plugin.py b/pgsync/plugin.py
@@ -2,6 +2,7 @@
 
 import logging
 import os
+import sys
 import typing as t
 from abc import ABC, abstractmethod
 from importlib import import_module
@@ -43,7 +44,9 @@ def reload(self) -> None:
         self.plugins: list = []
         self._paths: list = []
         logger.debug(f"Reloading plugins from package: {self.package}")
-        self.walk(self.package)
+        # skip in test
+        if "test" not in sys.argv[0]:
+            self.walk(self.package)
 
     def walk(self, package: str) -> None:
         """Recursively walk the supplied package and fetch all plugins."""

diff --git a/pgsync/search_client.py b/pgsync/search_client.py
@@ -260,6 +260,7 @@ def _create_setting(
         tree: Tree,
         setting: t.Optional[dict] = None,
         mapping: t.Optional[dict] = None,
+        mappings: t.Optional[dict] = None,
         routing: t.Optional[str] = None,
     ) -> None:
         """Create Elasticsearch/OpenSearch setting and mapping if required."""
@@ -268,7 +269,8 @@ def _create_setting(
         if not self.__client.indices.exists(index=index):
             if setting:
                 body.update(**{"settings": {"index": setting}})
-
+            if mappings:
+                body.update(**{"mappings": {"index": mappings}})
             if mapping:
                 if "dynamic_templates" in mapping:
                     body.update(**{"mappings": mapping})

diff --git a/pgsync/sync.py b/pgsync/sync.py
@@ -82,6 +82,7 @@ def __init__(
         self.nodes: dict = doc.get("nodes", {})
         self.setting: dict = doc.get("setting")
         self.mapping: dict = doc.get("mapping")
+        self.mappings: dict = doc.get("mappings")
         self.routing: str = doc.get("routing")
         super().__init__(
             doc.get("database", self.index), verbose=verbose, **kwargs
@@ -255,6 +256,7 @@ def create_setting(self) -> None:
             self.tree,
             setting=self.setting,
             mapping=self.mapping,
+            mappings=self.mappings,
             routing=self.routing,
         )
 

diff --git a/tests/test_constants.py b/tests/test_constants.py
@@ -99,6 +99,7 @@ def test_elasticsearch_types(self):
                 "constant_keyword",
                 "date",
                 "double",
+                "dense_vector",
                 "float",
                 "geo_point",
                 "geo_shape",

diff --git a/tests/test_sync.py b/tests/test_sync.py
@@ -763,7 +763,12 @@ def test_get_doc_id(self, sync):
     def test_create_setting(self, mock_es, sync):
         sync.create_setting()
         mock_es.assert_called_once_with(
-            "testdb", ANY, setting=None, mapping=None, routing=None
+            "testdb",
+            ANY,
+            setting=None,
+            mapping=None,
+            mappings=None,
+            routing=None,
         )
 
     @patch("pgsync.sync.Sync.teardown")