Merge branch 'dev' into teo/json-null-term

.github/workflows/ci-linux_mac.yml

@@ -110,6 +110,12 @@ jobs:
             core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
             core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
 
+      - name: Install Python
+        if: ${{ !inputs.manylinux }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Prevent vpckg from building debug variants
         run: python ./scripts/ci/patch_vcpkg_triplets.py
 

CMakeLists.txt

@@ -201,6 +201,9 @@ else()
     endif()
   endif()
 
+  if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL "16")
+    add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fexperimental-library>)
+  endif()
 endif()
 
 # Definitions for all targets
@@ -306,6 +309,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer/buffer_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer_list/buffer_list_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/config/config_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/context/context_api_experimental.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/context/context_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h"

examples/c_api/errors.c

@@ -31,22 +31,30 @@
  */
 
 #include <stdio.h>
-#include <tiledb/tiledb.h>
+#include <tiledb/tiledb_experimental.h>
 
-void print_error(tiledb_ctx_t* ctx);
+void print_last_error(tiledb_ctx_t* ctx);
+void print_error(tiledb_error_t* err);
 
 int main() {
   // Create TileDB context
   tiledb_ctx_t* ctx;
-  tiledb_ctx_alloc(NULL, &ctx);
+  tiledb_error_t* err;
+  int rc = tiledb_ctx_alloc_with_error(NULL, &ctx, &err);
+  if (rc == TILEDB_OK)
+    printf("Context created successfully!\n");
+  else if (rc == TILEDB_ERR) {
+    print_error(err);
+    return 1;
+  }
 
   // Create a group. The code below creates a group `my_group` and prints a
   // message because (normally) it will succeed.
-  int rc = tiledb_group_create(ctx, "my_group");
+  tiledb_group_create(ctx, "my_group");
   if (rc == TILEDB_OK)
     printf("Group created successfully!\n");
   else if (rc == TILEDB_ERR)
-    print_error(ctx);
+    print_last_error(ctx);
 
   // Create the same group again. f we attempt to create the same group
   // `my_group` as shown below, TileDB will return an error and the example
@@ -55,24 +63,28 @@ int main() {
   if (rc == TILEDB_OK)
     printf("Group created successfully!\n");
   else if (rc == TILEDB_ERR)
-    print_error(ctx);
+    print_last_error(ctx);
 
   // Clean up
   tiledb_ctx_free(&ctx);
 
   return 0;
 }
 
-void print_error(tiledb_ctx_t* ctx) {
+void print_last_error(tiledb_ctx_t* ctx) {
   // Retrieve the last error that occurred
   tiledb_error_t* err = NULL;
   tiledb_ctx_get_last_error(ctx, &err);
 
+  print_error(err);
+
+  // Clean up
+  tiledb_error_free(&err);
+}
+
+void print_error(tiledb_error_t* err) {
   // Retrieve the error message by invoking `tiledb_error_message`.
   const char* msg;
   tiledb_error_message(err, &msg);
   printf("%s\n", msg);
-
-  // Clean up
-  tiledb_error_free(&err);
 }

format_spec/FORMAT_SPEC.md

@@ -4,17 +4,18 @@ title: Format Specification
 
 **Notes:**
 
-* The current TileDB format version number is **22** (`uint32_t`).
+* The current TileDB array format version number is **22** (`uint32_t`).
+  * Other structures might be versioned separately.
 * Data written by TileDB and referenced in this document is **little-endian**
   with the following exceptions:
 
-  - [Dictionary filter](filters/dictionary_encoding.md)
+  - [Dictionary encoding filter](filters/dictionary_encoding.md)
   - RLE filter
 
 ## Table of Contents
 
 * **Array**
-   * [Format Version History](./history.md)
+   * [Format Version History](./array_format_history.md)
    * [File hierarchy](./array_file_hierarchy.md)
    * [Array Schema](./array_schema.md)
    * [Fragment](./fragment.md)

format_spec/array_file_hierarchy.md

@@ -7,6 +7,9 @@ An array is a folder with the following structure:
 ```
 my_array                                # array folder
     |_ __schema                         # array schema folder
+          |_ <timestamp_name>           # array schema files
+          |_ ...
+          |_ __enumerations             # array enumerations folder
     |_ __fragments                      # array fragments folder
           |_ <timestamped_name>         # fragment folder
           |_ ...
@@ -22,23 +25,39 @@ my_array                                # array folder
           |_ <timestamped_name>.con     # consolidated commits file
           |_ ...
           |_ <timestamped_name>.ign     # ignore file for consolidated commits file
-    |_ __fragment_meta                  
-          |_ <timestamped_name>.meta    # consol. fragment meta file
-          |_ ...                  
+    |_ __fragment_meta                  # consolidated fragment metadata folder
+          |_ <timestamped_name>.meta    # consolidated fragment meta file
+          |_ ...
     |_ __meta                           # array metadata folder
     |_ __labels                         # dimension label folder
-
+    |_ <timestamped_name>               # legacy fragment folder
+          |_ ...
+    |_ <timestamped_name>.ok            # legacy fragment write file
+    |_ <timestamped_name>.meta          # legacy consolidated fragment meta file
+    |_ __array_schema.tdb               # legacy array schema file
 ```
 
 Inside the array folder, you can find the following:
 
-* [Array schema](./array_schema.md) folder `__schema`.
-* Inside of a fragments folder, any number of [fragment folders](./fragment.md) [`<timestamped_name>`](./timestamped_name.md).
-* Inside of a commit folder, an empty file [`<timestamped_name>`](./timestamped_name.md)`.wrt` associated with every fragment folder [`<timestamped_name>`](./timestamped_name.md), where [`<timestamped_name>`](./timestamped_name.md) is common for the folder and the WRT file. This is used to indicate that fragment [`<timestamped_name>`](./timestamped_name.md) has been *committed* (i.e., its write process finished successfully) and it is ready for use by TileDB. If the WRT file does not exist, the corresponding fragment folder is ignored by TileDB during the reads.
-* Inside the same commit folder, any number of [delete commit files](./delete_commit_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.del`.
-* Inside the same commit folder, any number of [update commit files](./update_commit_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.upd`.
-* Inside the same commit folder, any number of [consolidated commits files](./consolidated_commits_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.con`.
-* Inside the same commit folder, any number of [ignore files](./ignore_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.ign`.
-* Inside of a fragment metadata folder, any number of [consolidated fragment metadata files](./consolidated_fragment_metadata_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.meta`.
-* [Array metadata](./metadata.md) folder `__meta`.
-* Inside of a labels folder, additional TileDB arrays storing dimension label data.
+* Inside of a `__schema` folder, any number of [array schema files](./array_schema.md) [`<timestamped_name>`](./timestamped_name.md).
+  * **Note**: the name does _not_ include the format version.
+  * _New in version 20_ Inside of the schema folder, an enumerations folder `__enumerations`.
+* Inside of a `__meta` folder, any number of [array metadata files](./metadata.md) [`<timestamped_name>`](./timestamped_name.md).
+* Inside of a `__fragments` folder, any number of [fragment folders](./fragment.md) [`<timestamped_name>`](./timestamped_name.md).
+* _New in version 18_ Inside of a `__labels` folder, additional TileDB arrays storing dimension label data.
+* _New in version 12_ Inside of a `__commits` folder:
+  * Any number of empty files [`<timestamped_name>`](./timestamped_name.md)`.wrt`, each associated with fragment folder [`<timestamped_name>`](./timestamped_name.md), indicating that the fragment has been *committed* (i.e., its write process finished successfully). If the WRT file does not exist, the corresponding fragment must be ignored when reading the array.
+  * Any number of [consolidated commits files](./consolidated_commits_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.con`.
+  * Any number of [ignore files](./ignore_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.ign`.
+  * _New in version 16_ Any number of [delete commit files](./delete_commit_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.del`.
+  * _New in version 16_ Any number of [update commit files](./update_commit_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.upd`.
+* _New in version 12_ Inside of a `__fragment_meta` folder, any number of [consolidated fragment metadata files](./consolidated_fragment_metadata_file.md) of the form [`<timestamped_name>`](./timestamped_name.md)`.meta`.
+
+> [!NOTE]
+> Prior to version 12, fragments, commit files, and consolidated fragment metadata were stored directly in the array folder and the extension of commit files was `.ok` instead of `.wrt`. Implementations must support arrays that contain data in both the old and the new hierarchy at the same time.
+
+> [!NOTE]
+> Prior to version 10, the array schema was stored in a single `__array_schema.tdb` file in the array folder. Implementations must support arrays that contain both `__array_schema.tdb` and schemas in the `__schema` folder at the same time. For the purpose of array schema evolution, the timestamp of `__array_schema.tdb` must be considered to be earlier than any schema in the `__schema` folder.
+
+> [!NOTE]
+> Prior to version 5, commit files were not written. Fragments of these versions are considered to be committed if their corresponding fragment metadata file exists.

format_spec/history.md → format_spec/array_format_history.md

@@ -1,8 +1,8 @@
 ---
-title: Format version history
+title: Array format version history
 ---
 
-# Format Version History
+# Array Format Version History
 
 ## Version 22
 
@@ -24,7 +24,7 @@ Introduced in TileDB 2.19
 Introduced in TileDB 2.17
 
 * Arrays can have [enumerations](./enumeration.md).
-* The bit-width reduction and positive delta filters are supported on data of date or time types.
+* The bit-width reduction and positive delta encoding filters are supported on data of date or time types.
 * The [filter pipeline options](./filter_pipeline.md#filter-options) for the double-delta filter contain the _Reinterpret datatype_ field.
 
 ## Version 19
@@ -45,7 +45,7 @@ Introduced in TileDB 2.15
 Introduced in TileDB 2.14
 
 * The _Order_ field was added to [attributes](./array_schema.md#attribute).
-* Cell offsets in dimensions or attributes of UTF-8 string type are not written in the offset tiles, if the RLE or dictionary filter exists in the filter pipeline. They are instead encoded as part of the data tile.
+* Cell offsets in dimensions or attributes of UTF-8 string type are not written in the offset tiles, if the RLE or dictionary encoding filter exists in the filter pipeline. They are instead encoded as part of the data tile.
 
 ## Version 16
 
@@ -72,7 +72,7 @@ Introduced in TileDB 2.10
 
 Introduced in TileDB 2.9
 
-* The [dictionary filter](./filters/dictionary_encoding.md) was added.
+* Cell offsets in dimensions or attributes of ASCII string type are not written in the offset tiles, if the dictionary encoding filter exists in the filter pipeline. They are instead encoded as part of the data tile.
 
 ## Version 12
 
@@ -86,7 +86,7 @@ Introduced in TileDB 2.8
 
 Introduced in TileDB 2.7
 
-* Fragment metadata contain [metadata](./fragment.md#tile-mins-maxes) (min/max value, sum, null count) for each tile.
+* Fragment metadata contain [metadata](./fragment.md#tile-mins-maxes) (min/max value, sum, null count) for data in the whole fragment and each tile.
 * The TileDB implementation has been updated to never split cells when storing them in chunks.
 
 ## Version 10
@@ -154,7 +154,7 @@ Introduced in TileDB 1.6
     * The [footer](./fragment.md#footer) and [R-Tree](./fragment.md#r-tree) structures were added.
     * The _Bounding coords_ field was removed.
     * The _MBRs_ field was removed. MBRs are now stored in the R-Tree.
-    * Structures other than the footer like tile offsets, sizes and metadata are wrapped in their own generic tiles. This allows loading them lazily and in parallel.
+    * Tile offsets and sizes are wrapped in their own generic tiles. This allows loading them lazily and in parallel.
 
 ## Version 2