Implementation of proposed syntax changes

.github/workflows/docs.yml

@@ -0,0 +1,142 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Build docs
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    - name: Install package
+      run: pip install .[doc]
+    - name: Revert version to most recent version tag
+      run: git checkout $(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}')
+    - name: Build docs
+      run: |
+        cd docs
+        make html
+        cd ..
+    - uses: actions/upload-artifact@v4
+      with:
+        name: docs
+        path: docs/_build/html
+
+  build-new:
+    runs-on: ubuntu-latest
+    # Set up the environment so that it finds conda
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - name: Install Pandoc for NBSphinx
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pandoc
+    - name: Install Dependencies for virtual notifications in Adv.-Exec Tutorial
+      run: |
+          sudo apt update
+          sudo apt install -y xvfb libnotify-bin dbus-x11 xfce4-notifyd
+    - name: Start Virtual Display (for notifications)
+      run: |
+        Xvfb :99 &
+        export DISPLAY=:99
+        eval "$(dbus-launch --sh-syntax)"
+        echo "DISPLAY=:99" >> $GITHUB_ENV
+        echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> $GITHUB_ENV
+    - name: Start Notification Daemon (for notifications)
+      run: |
+        xfce4-notifyd &
+        sleep 2  # Give it some time to start
+    - name: Send Notification (test notifications)
+      run: |
+        notify-send "GitHub Runner Notification" "This is a test notification from GitHub Actions"
+    - name: Debug Running Processes (for notifications)
+      run: |
+        ps aux | grep notify
+        ps aux | grep xfce4-notifyd
+        dbus-monitor --session &
+        sleep 3
+    - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
+    - name: Install Minconda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-activate-base: true
+        activate-environment: ""
+    - name: Install MRtrix via Conda
+      run: |
+        conda install -c mrtrix3 mrtrix3
+        mrconvert --version
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    - name: Install package
+      run: pip install .[doc]
+    - name: Install Python3 kernel
+      run: python -m ipykernel install --user
+    - name: Build new docs
+      run: |
+        cd new-docs
+        make html
+        cd ..
+    - uses: actions/upload-artifact@v4
+      with:
+        name: new-docs
+        path: new-docs/build/html
+
+  deploy:
+    needs: [build, build-new]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download docs
+      uses: actions/download-artifact@v4
+      with:
+        name: docs
+        path: docs-build
+    - name: Download new docs
+      uses: actions/download-artifact@v4
+      with:
+        name: new-docs
+        path: docs-build/new
+    - name: Check for GHPAGES_DEPLOY_KEY token
+      id: deployable
+      # if: github.event_name == 'release'
+      env:
+        GHPAGES_DEPLOY_KEY: "${{ secrets.GHPAGES_DEPLOY_KEY }}"
+      run: if [ -n "$GHPAGES_DEPLOY_KEY" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi
+    - name: Deploy Docs to GitHub Pages
+      if: steps.deployable.outputs.DEPLOY
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GHPAGES_DEPLOY_KEY }}
+        publish_dir: docs-build

.github/workflows/testdask.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 concurrency:
@@ -18,7 +19,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12', '3.13']
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -27,7 +28,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Setup Python version ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:

.github/workflows/testpsijlocal.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 concurrency:
@@ -27,7 +28,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Setup Python version ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:

.github/workflows/testpsijslurm.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 concurrency:
@@ -24,6 +25,8 @@ jobs:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
     - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
@@ -47,7 +50,7 @@ jobs:
             docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
         fi
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra; print(pydra.__version__)'"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
         docker exec slurm bash -c "pytest --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"

.github/workflows/testpydra.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 defaults:
@@ -50,75 +51,36 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
-        install: ['wheel']
-        include:
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'sdist'
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'repo'
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'archive'
+        python-version: ['3.11', '3.12', '3.13']
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
 
     steps:
-      - name: Fetch sdist/wheel
-        uses: actions/download-artifact@v4
-        if: matrix.install == 'sdist' || matrix.install == 'wheel'
-        with:
-          name: dist
-          path: dist/
-      - name: Fetch git archive
-        uses: actions/download-artifact@v4
-        if: matrix.install == 'archive'
-        with:
-          name: archive
-          path: archive/
       - name: Fetch repository
         uses: actions/checkout@v4
-        if: matrix.install == 'repo'
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Update pip
         run: python -m pip install --upgrade pip
 
-      - name: Determine installation target
-        run: |
-          if [[ "$INSTALL" = "sdist" ]]; then
-              echo "ARCHIVE=$( ls dist/*.tar.gz )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "wheel" ]]; then
-              echo "ARCHIVE=$( ls dist/*.whl )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "archive" ]]; then
-              echo "ARCHIVE=$( ls archive/*.zip )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "repo" ]]; then
-              echo "ARCHIVE=." >> $GITHUB_ENV
-          fi
-        env:
-          INSTALL: ${{ matrix.install }}
-
       - name: Install Pydra
-        run: pip install $ARCHIVE
+        run: pip install .[test]
 
       - name: Print version
-        run: python -c "import pydra; print(pydra.__version__)"
-
-      - name: Install Pydra tests dependencies
-        run: pip install pydra[test]
+        run: python -c "import pydra.engine; print(pydra.engine.__version__)"
 
       - name: Disable etelemetry
         run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
 
       - name: Pytest
         run: |
-          pytest -vs -n auto --doctest-modules --pyargs pydra \
+          pytest -vs -n auto --doctest-modules \
                  --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
 
       - name: Upload to codecov

.github/workflows/testsingularity.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 concurrency:
@@ -13,10 +14,10 @@ concurrency:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: ['3.11', '3.12', '3.13']
       fail-fast: False
 
     steps:
@@ -64,11 +65,13 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Install pydra (test)
         run: pip install -e ".[test]"
 
 
       - name: Pytest
-        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py
+        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py
       - name: Upload to codecov
         run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testslurm.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - develop
   pull_request:
 
 concurrency:
@@ -14,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.5]
+        python-version: [3.11.5]
       fail-fast: false
     runs-on: ubuntu-latest
     env:
@@ -24,6 +25,8 @@ jobs:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
     - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
@@ -47,7 +50,7 @@ jobs:
             docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
         fi
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
         docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"

.gitignore

@@ -4,6 +4,7 @@ dist
 
 __pycache__
 *.pyc
+.python-version
 
 .ipynb_checkpoints
 .vscode/
@@ -18,6 +19,8 @@ cov.xml
 *.venv
 
 .DS_Store
+.ipynb_checkpoints
 
 # This can be generated in-tree. We never want to commit it.
 pydra/_version.py
+pydra/engine/_version.py