now adding act runtimes for local CI testing

.github/workflows/surrealml_core_onnx_test.yml

@@ -13,37 +13,5 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11'
-
-    - name: Pre-test Setup
-      run: |
-        python3 -m venv venv
-        source venv/bin/activate
-        pip install --upgrade pip
-        pip install -r requirements.txt
-        
-        # build the local version of the core module to be loaded into python
-        echo "Building local version of core module"
-        
-        pip install .
-        export PYTHONPATH="."
-        
-        python ./tests/scripts/ci_local_build.py
-        echo "Local build complete"
-        
-        # train the models for the tests
-        python ./tests/model_builder/onnx_assets.py
-        deactivate
-
     - name: Run Core Unit Tests
-      run: cd modules/core && cargo test --features onnx-tests
+      run: cd modules/core && docker build -t rust-onnx-runtime . && docker run --rm -it rust-onnx-runtime cargo test --features onnx-tests

.github/workflows/surrealml_core_tensorflow_test.yml

@@ -13,37 +13,5 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11'
-
-    - name: Pre-test Setup
-      run: |
-        python3 -m venv venv
-        source venv/bin/activate
-        pip install --upgrade pip
-        pip install -r requirements.txt
-        
-        # build the local version of the core module to be loaded into python
-        echo "Building local version of core module"
-        
-        pip install .
-        export PYTHONPATH="."
-        
-        python ./tests/scripts/ci_local_build.py
-        echo "Local build complete"
-        
-        # train the models for the tests
-        python ./tests/model_builder/tensorflow_assets.py
-        deactivate
-
     - name: Run Core Unit Tests
-      run: cd modules/core && cargo test --features tensorflow-tests
+      run: cd modules/core && docker build -t rust-onnx-runtime . && docker run --rm -it rust-onnx-runtime cargo test --features tensorflow-tests

.github/workflows/surrealml_core_test.yml

@@ -13,47 +13,5 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11'
-
-    - name: Pre-test Setup
-      run: |
-        python3 -m venv venv
-        source venv/bin/activate
-        pip install --upgrade pip
-        pip install -r requirements.txt
-        
-        # build the local version of the core module to be loaded into python
-        echo "Building local version of core module"
-        
-        pip install .
-        export PYTHONPATH="."
-        
-        python ./tests/scripts/ci_local_build.py
-        echo "Local build complete"
-        
-        # train the models for the tests
-        python ./tests/model_builder/sklearn_assets.py
-        deactivate
-
-    - name: Run Python Unit Tests
-      run: |
-        source venv/bin/activate
-        export PYTHONPATH="."
-        python tests/unit_tests/engine/test_sklearn.py
-        deactivate
-
     - name: Run Core Unit Tests
-      run: cd modules/core && cargo test --features sklearn-tests
-
-    - name: Run HTTP Transfer Tests
-      run: cargo test
+      run: cd modules/core && docker build -t rust-onnx-runtime . && docker run --rm -it rust-onnx-runtime cargo test --features sklearn-tests

.github/workflows/surrealml_core_torch_test.yml

@@ -13,46 +13,5 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11'
-
-    - name: Pre-test Setup
-      run: |
-        python3 -m venv venv
-        source venv/bin/activate
-        pip install --upgrade pip
-        pip install -r requirements.txt
-        
-        # build the local version of the core module to be loaded into python
-        echo "Building local version of core module"
-        
-        pip install .
-        export PYTHONPATH="."
-        
-        python ./tests/scripts/ci_local_build.py
-        echo "Local build complete"
-        
-        # train the models for the tests
-        python ./tests/model_builder/torch_assets.py
-        deactivate
-
-    - name: Run Python Unit Tests
-      run: |
-        source venv/bin/activate
-        export PYTHONPATH="."
-        python tests/unit_tests/engine/test_torch.py
-        python tests/unit_tests/test_rust_adapter.py
-        python tests/unit_tests/test_surml_file.py
-        deactivate
-
     - name: Run Core Unit Tests
-      run: cd modules/core && cargo test --features torch-tests
+      run: cd modules/core && docker build -t rust-onnx-runtime . && docker run --rm -it rust-onnx-runtime cargo test --features torch-tests

modules/core/Dockerfile

@@ -0,0 +1,37 @@
+# Use an official Rust image
+FROM rust:1.83-slim
+
+# Install necessary tools
+RUN apt-get update && apt-get install -y \
+    wget \
+    build-essential \
+    libssl-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the project files into the container
+COPY . .
+
+# Download ONNX Runtime 1.20.0
+RUN wget https://github.com/microsoft/onnxruntime/releases/download/v1.20.0/onnxruntime-linux-x64-1.20.0.tgz \
+    && tar -xvf onnxruntime-linux-x64-1.20.0.tgz \
+    && mv onnxruntime-linux-x64-1.20.0 /onnxruntime
+
+# # Download ONNX Runtime 1.16.0
+# RUN wget https://github.com/microsoft/onnxruntime/releases/download/v1.16.0/onnxruntime-linux-x64-1.16.0.tgz \
+#     && tar -xvf onnxruntime-linux-x64-1.16.0.tgz \
+#     && mv onnxruntime-linux-x64-1.16.0 /onnxruntime
+
+# Set the ONNX Runtime library path
+ENV ORT_LIB_LOCATION=/onnxruntime/lib
+ENV LD_LIBRARY_PATH=$ORT_LIB_LOCATION:$LD_LIBRARY_PATH
+
+# Clean and build the Rust project
+RUN cargo clean
+RUN cargo build --features tensorflow-tests
+
+# Run the tests
+CMD ["cargo", "test", "--features", "tensorflow-tests"]

modules/core/scripts/install_onnxruntime_linux.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Variables
+ONNX_VERSION="1.20.0"
+ONNX_DOWNLOAD_URL="https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz"
+ONNX_RUNTIME_DIR="/home/maxwellflitton/Documents/github/surreal/surrealml/modules/core/target/debug/build/ort-680c63907dcb00d8/out/onnxruntime"
+ONNX_TARGET_DIR="${ONNX_RUNTIME_DIR}/onnxruntime-linux-x64-${ONNX_VERSION}"
+LD_LIBRARY_PATH_UPDATE="${ONNX_TARGET_DIR}/lib"
+
+# Step 1: Download and Extract ONNX Runtime
+echo "Downloading ONNX Runtime version ${ONNX_VERSION}..."
+wget -q --show-progress "${ONNX_DOWNLOAD_URL}" -O "onnxruntime-linux-x64-${ONNX_VERSION}.tgz"
+
+if [ $? -ne 0 ]; then
+    echo "Failed to download ONNX Runtime. Exiting."
+    exit 1
+fi
+
+echo "Extracting ONNX Runtime..."
+tar -xvf "onnxruntime-linux-x64-${ONNX_VERSION}.tgz"
+
+if [ ! -d "onnxruntime-linux-x64-${ONNX_VERSION}" ]; then
+    echo "Extraction failed. Directory not found. Exiting."
+    exit 1
+fi
+
+# Step 2: Replace Old ONNX Runtime
+echo "Replacing old ONNX Runtime..."
+mkdir -p "${ONNX_RUNTIME_DIR}"
+mv "onnxruntime-linux-x64-${ONNX_VERSION}" "${ONNX_TARGET_DIR}"
+
+if [ ! -d "${ONNX_TARGET_DIR}" ]; then
+    echo "Failed to move ONNX Runtime to target directory. Exiting."
+    exit 1
+fi
+
+# Step 3: Update LD_LIBRARY_PATH
+echo "Updating LD_LIBRARY_PATH..."
+export LD_LIBRARY_PATH="${LD_LIBRARY_PATH_UPDATE}:$LD_LIBRARY_PATH"
+
+# Step 4: Verify Library Version
+echo "Verifying ONNX Runtime version..."
+strings "${LD_LIBRARY_PATH_UPDATE}/libonnxruntime.so" | grep "VERS_${ONNX_VERSION}" > /dev/null
+
+if [ $? -ne 0 ]; then
+    echo "ONNX Runtime version ${ONNX_VERSION} not found in library. Exiting."
+    exit 1
+fi
+
+# Step 5: Install Library Globally (Optional)
+echo "Installing ONNX Runtime globally..."
+sudo cp "${LD_LIBRARY_PATH_UPDATE}/libonnxruntime.so" /usr/local/lib/
+sudo ldconfig
+
+if [ $? -ne 0 ]; then
+    echo "Failed to install ONNX Runtime globally. Exiting."
+    exit 1
+fi
+
+# Step 6: Clean and Rebuild Project
+echo "Cleaning and rebuilding project..."
+cargo clean
+cargo test --features tensorflow-tests
+
+if [ $? -eq 0 ]; then
+    echo "ONNX Runtime updated successfully, and tests passed."
+else
+    echo "ONNX Runtime updated, but tests failed. Check the logs for details."
+fi

modules/core/src/lib.rs

@@ -60,7 +60,7 @@
 //!
 //! ### Executing models
 //! We you load a `surml` file, you can execute the model with the following code:
-//! ```rust
+//! ```no_run
 //! use surrealml_core::storage::surml_file::SurMlFile;
 //! use surrealml_core::execution::compute::ModelComputation;
 //! use ndarray::ArrayD;
@@ -94,5 +94,5 @@ pub mod errors;
 
 /// Returns the version of the ONNX runtime that is used.
 pub fn onnx_runtime() -> &'static str {
-    "1.16.0"
+    "1.20.0"
 }

requirements.txt

@@ -1,2 +1,2 @@
-onnxruntime==1.17.3
-numpy==1.26.3
+onnxruntime==1.20.0
+numpy==2.2.1

scripts/run_ci_workflows/surrealml_core_onnx.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+cd ../..
+
+
+act -W .github/workflows/surrealml_core_onnx_test.yml pull_request

scripts/run_ci_workflows/surrealml_core_sklearn.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+cd ../..
+
+
+act -W .github/workflows/surrealml_core_test.yml pull_request

scripts/run_ci_workflows/surrealml_core_tensorflow.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+cd ../..
+
+
+act -W .github/workflows/surrealml_core_tensorflow_test.yml pull_request

scripts/run_ci_workflows/surrealml_core_torch.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+cd ../..
+
+
+act -W .github/workflows/surrealml_core_torch_test.yml pull_request

tests/model_builder/tensorflow_assets.py

@@ -1,4 +1,4 @@
-from tests.model_builder.utils import install_package
+from model_builder.utils import install_package
 install_package("tf2onnx==1.16.1")
 install_package("tensorflow==2.16.1")
 import os
@@ -7,7 +7,7 @@
 from surrealml.model_templates.tensorflow.tensorflow_linear import export_model_onnx as linear_tensorflow_export_model_onnx
 from surrealml.model_templates.tensorflow.tensorflow_linear import export_model_surml as linear_tensorflow_export_model_surml
 
-from tests.model_builder.utils import delete_directory, create_directory, MODEL_STASH_DIRECTORY
+from model_builder.utils import delete_directory, create_directory, MODEL_STASH_DIRECTORY
 
 
 # create the model stash directory if it does not exist