Use pixi for dep management (#72)

.dockerignore

@@ -3,9 +3,7 @@
 
 # except
 !wort/
-!Pipfile.lock
-!Pipfile
-!requirements.txt
+!pixi.lock
 !config/
 !wortapp.py
 !pyproject.toml

.envrc

@@ -0,0 +1,4 @@
+load_prefix ~/.pixi
+
+watch_file pixi.lock
+eval "$(pixi shell-hook)"

.gitattributes

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML

.github/workflows/check.yaml

@@ -1,4 +1,4 @@
-name: Check docker-compose can build containers
+name: Check docker compose can build containers
 on: push
 jobs:
   test:
@@ -11,7 +11,7 @@ jobs:
           mkdir -p iam
           touch iam/wort_s3.env
       - name: Build the stack
-        run: docker-compose build
+        run: docker compose build
       # TODO: check containers can start
       #- name: Bring up containers
       #  run: docker-compose up -d web worker

.gitignore

@@ -11,6 +11,7 @@ target/
 fabfile.py
 *.wasm
 **/node_modules/*
+dist/
 
 .config/
 .envrc
@@ -27,3 +28,8 @@ machine/*
 nix/*
 nocolors_600k
 package-lock.json
+
+# pixi environments
+.pixi
+
+.direnv

Dockerfile

@@ -1,89 +1,72 @@
-# Build decoct
-FROM rust:1-slim-bullseye as builder
-RUN cargo install --git https://github.com/luizirber/decoct --rev 630f5a773e6df22a33dafc3d744e39735c055328
-
-# Build worker image
-FROM python:3.10.9-slim-bullseye as base
-
-RUN groupadd user && \
-    useradd --create-home --home-dir /home/user -g user -s /bin/bash user
-
-WORKDIR /home/user
-
-#--------------------
-
-FROM base as python-deps
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends build-essential libcurl4-openssl-dev libssl-dev curl
-
-RUN python3 -m venv /home/user/.venv
-
-COPY requirements.txt ./
-
-RUN . /home/user/.venv/bin/activate && pip install --require-hashes -r requirements.txt
-
-RUN curl --output sratoolkit.tar.gz https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/3.0.2/sratoolkit.3.0.2-ubuntu64.tar.gz
-RUN tar xf sratoolkit.tar.gz
+FROM ghcr.io/prefix-dev/pixi:0.36.0 AS build
+
+COPY . /app
+WORKDIR /app
+RUN pixi run build-wheel
+RUN pixi run -e web postinstall-prod
+RUN pixi shell-hook -e web > /shell-hook-web
+RUN echo 'exec "$@"' >> /shell-hook-web
+RUN pixi run -e worker postinstall-prod
+RUN pixi shell-hook -e worker > /shell-hook-worker
+RUN echo 'exec "$@"' >> /shell-hook-worker
 
 #--------------------
 
-FROM base as web
-
-COPY --from=python-deps /home/user/.venv /home/user/.venv
+FROM ubuntu:24.04 AS web 
 
-WORKDIR /app/wort
+# only copy the production environment into prod container
+COPY --from=build /app/.pixi/envs/web /app/.pixi/envs/web
+COPY --from=build /shell-hook-web /shell-hook
 
-ADD wortapp.py pyproject.toml ./
+RUN groupadd user && \
+    useradd --create-home --home-dir /home/user -g user -s /bin/bash user
 
-COPY wort wort
-COPY config config
+COPY wortapp.py /app
+COPY config/ /app/config
 
-RUN . /home/user/.venv/bin/activate && pip install flit && FLIT_ROOT_INSTALL=1 flit install --symlink
+WORKDIR /app
 
-USER user
+#USER user 
 
-ENV PATH "/home/user/.venv/bin:$PATH"
-CMD . /home/user/.venv/bin/activate && gunicorn -b 0.0.0.0:5000 --access-logfile - "wort.app:create_app()"
+ENTRYPOINT ["/bin/bash", "/shell-hook"]
+CMD ["gunicorn", "-b", "0.0.0.0:5000", "--access-logfile", "-", "'wortapp:create_app()'"]
 
 #--------------------
 
-FROM base as worker
+FROM ubuntu:24.04 AS worker
 
-COPY --from=builder /usr/local/cargo/bin/decoct /usr/local/bin/sourmash
-COPY --from=python-deps /home/user/sratoolkit.3.0.2-ubuntu64/bin /home/user/sratoolkit.3.0.2-ubuntu64/bin
-COPY --from=python-deps /home/user/.venv /home/user/.venv
+# only copy the production environment into prod container
+COPY --from=build /app/.pixi/envs/worker /app/.pixi/envs/worker
+COPY --from=build /shell-hook-worker /shell-hook
 
-ENV PATH "/home/user/.venv/bin:$PATH"
+RUN groupadd user && \
+    useradd --create-home --home-dir /home/user -g user -s /bin/bash user
 
-ENV PATH "$PATH:/home/user/sratoolkit.3.0.2-ubuntu64/bin"
+COPY wortapp.py /app
+COPY config/ /app/config
 
-# needed for celery[sqs]
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev
+WORKDIR /app
 
-USER user
+USER user 
 
 # Configure sra-toolkit to disable cache
-RUN mkdir .ncbi
-
-RUN echo '## auto-generated configuration file - DO NOT EDIT ##''' >> .ncbi/user-settings.mkfg
-RUN echo '' >> .ncbi/user-settings.mkfg
-RUN echo '/LIBS/GUID = "7737545d-30d4-4d05-875a-2c562df521d5"' >> .ncbi/user-settings.mkfg
-RUN echo '/config/default = "false"' >> .ncbi/user-settings.mkfg
-RUN echo '/libs/cloud/accept_aws_charges = "false"' >> .ncbi/user-settings.mkfg
-RUN echo '/libs/cloud/report_instance_identity = "true"' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/apps/file/volumes/flatAd = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/apps/refseq/volumes/refseqAd = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/apps/sra/volumes/sraAd = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/apps/sraPileup/volumes/ad = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/apps/sraRealign/volumes/ad = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/ad/public/root = "."' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/default-path = "/root/ncbi"' >> .ncbi/user-settings.mkfg
-RUN echo '/repository/user/main/public/cache-disabled = "true"' >> .ncbi/user-settings.mkfg
-
-COPY wort wort
-COPY config config
+RUN mkdir ~/.ncbi
+
+RUN echo '## auto-generated configuration file - DO NOT EDIT ##''' >> ~/.ncbi/user-settings.mkfg
+RUN echo '' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/LIBS/GUID = "7737545d-30d4-4d05-875a-2c562df521d5"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/config/default = "false"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/libs/cloud/accept_aws_charges = "false"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/libs/cloud/report_instance_identity = "true"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/file/volumes/flatAd = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/refseq/volumes/refseqAd = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/sra/volumes/sraAd = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/sraPileup/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/sraRealign/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/root = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/default-path = "/root/ncbi"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/main/public/cache-disabled = "true"' >> ~/.ncbi/user-settings.mkfg
 
 ENV RAYON_NUM_THREADS 3
-CMD celery -A wort.blueprints.compute.tasks worker -Q compute_small,compute_medium,genomes --without-gossip --without-mingle --without-heartbeat -l INFO -c 1
+ENTRYPOINT ["/bin/bash", "/shell-hook"]
+CMD ["celery", "-A", "wort.blueprints.compute.tasks", "worker", "-Q", "compute_small,compute_medium,genomes", "--without-gossip", "--without-mingle", "--without-heartbeat", "-l", "INFO", "-c", "1"]

docker-compose.yml

@@ -34,7 +34,6 @@ services:
     command: >
       gunicorn -b 0.0.0.0:5000
         --access-logfile -
-        --reload
         "wort.app:create_app()"
     environment:
       PYTHONUNBUFFERED: 'true'
@@ -48,8 +47,8 @@ services:
       - redis
     env_file:
       - env.production
-    volumes:
-      - .:/app/wort
+#    volumes:
+#      - .:/app/wort
 
   redis:
     restart: always