Feature/enhance (#58)

* feat: support for vector; quick mode for geo && vector data for dump

---------

Co-authored-by: yongcai <[email protected]>

Author: WinChua

src/pyinnodb/cli/sql.py

@@ -96,6 +96,8 @@ def transfter(nd):
                     elif isinstance(field, MGeo):
                         d = field.build().hex()  # .zfill(50)
                         vs.append("0x" + d)
+                    elif isinstance(field, bytes):
+                        vs.append("0x"+field.hex())
                     else:
                         vs.append(repr(field))
                 values.append(f"({','.join(vs)})")

src/pyinnodb/const/dd_column_type.py

@@ -36,6 +36,7 @@ class DDColumnType(Enum):
     STRING = 29
     GEOMETRY = 30
     JSON = 31
+    VECTOR = 32
 
     def is_int_number(self):
         return self in _int_number_type
@@ -88,13 +89,16 @@ def is_big(cls, t):
     DDColumnType.JSON,
     DDColumnType.TINY_BLOB,
     DDColumnType.GEOMETRY,
+    DDColumnType.VECTOR,
 ]
 
 _big_type = [
     DDColumnType.MEDIUM_BLOB,
     DDColumnType.LONG_BLOB,
     DDColumnType.BLOB,
     DDColumnType.JSON,
+    DDColumnType.GEOMETRY,
+    DDColumnType.VECTOR,
 ]
 
 DDColConf = namedtuple("DDColConf", "type size")
@@ -132,6 +136,7 @@ class DDColConf(DDColConf, Enum):
     STRING = DDColumnType.STRING, 0
     GEOMETRY = DDColumnType.GEOMETRY, 0
     JSON = DDColumnType.JSON, 0
+    VECTOR = DDColumnType.VECTOR, 0
 
     @classmethod
     def get_col_type_conf(cls, type) -> DDColConf:

src/pyinnodb/disk_struct/index.py

@@ -109,7 +109,7 @@ class MIndexPage(CC):
     system_records: MIndexSystemRecord = cfield(MIndexSystemRecord)
 
     @classmethod
-    def default_value_parser(cls, dd_object: Table, transfter=None, hidden_col=False):
+    def default_value_parser(cls, dd_object: Table, transfter=None, hidden_col=False, quick=True):
         primary_data_layout_col = dd_object.get_disk_data_layout()
 
         def value_parser(rh: MRecordHeader, f):
@@ -227,7 +227,7 @@ def value_parser(rh: MRecordHeader, f):
                     if col.ordinal_position in null_col_data:
                         col_value = None
                     else:
-                        col_value = col.read_data(f, vs)
+                        col_value = col.read_data(f, vs, quick=quick)
                 except Exception as e:
                     print("cur before is ", cur_before, vs, col)
                     raise e

src/pyinnodb/sdi/table.py

@@ -395,7 +395,7 @@ def _read_varchar(self, stream, size):
             else:
                 return data
 
-    def read_data(self, stream, size=None):
+    def read_data(self, stream, size=None, quick=True):
         if self.name == "DB_ROLL_PTR":
             return MRollbackPointer.parse_stream(stream)
         dtype = DDColumnType(self.type)
@@ -405,6 +405,15 @@ def read_data(self, stream, size=None):
             dsize = self.size
         if dtype.is_int_number():
             return self._read_int(stream, dsize)
+        elif dtype == DDColumnType.VECTOR:
+            if quick:
+                return stream.read(dsize)
+            else:
+                vec = []
+                for i in range(int(dsize / 4)):
+                    byte_data = stream.read(4)
+                    vec.append(struct.unpack("f", byte_data)[0])
+                return vec
         elif dtype == DDColumnType.FLOAT:
             byte_data = stream.read(dsize)
             if dsize == 4:
@@ -476,9 +485,12 @@ def read_data(self, stream, size=None):
             except Exception as e:
                 return data
         elif dtype == DDColumnType.GEOMETRY:
-            data = MGeo.parse_stream(stream)
-            logging.debug("geometry data is %s, size is %d", data, dsize)
-            return data
+            if quick:
+                return stream.read(dsize)
+            else:
+                data = MGeo.parse_stream(stream)
+                logging.debug("geometry data is %s, size is %d", data, dsize)
+                return data
 
 
 decimal_leftover_part = {
@@ -846,7 +858,8 @@ def search(self, f, primary_key, hidden_col):
         else:
             primary_key = self.build_primary_key_bytes((primary_key,))
         value_parser = MIndexPage.default_value_parser(
-            self, hidden_col=hidden_col, transfter=lambda id: id
+            self, hidden_col=hidden_col, transfter=lambda id: id,
+            quick=False,
         )
 
         while first_leaf_page != 4294967295: