Merge pull request #289 from nf-core/dev

Dev -> Master for release

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+{
+    "name": "nfcore",
+    "image": "nfcore/gitpod:latest",
+    "remoteUser": "gitpod",
+
+    // Configure tool-specific properties.
+    "customizations": {
+        // Configure properties specific to VS Code.
+        "vscode": {
+            // Set *default* container specific settings.json values on container create.
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/conda/bin/python",
+                "python.linting.enabled": true,
+                "python.linting.pylintEnabled": true,
+                "python.formatting.autopep8Path": "/opt/conda/bin/autopep8",
+                "python.formatting.yapfPath": "/opt/conda/bin/yapf",
+                "python.linting.flake8Path": "/opt/conda/bin/flake8",
+                "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
+                "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
+                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+            },
+
+            // Add the IDs of extensions you want installed when the container is created.
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
+        }
+    }
+}

.gitattributes

@@ -1,3 +1,4 @@
 *.config linguist-language=nextflow
+*.nf.test linguist-language=nextflow
 modules/nf-core/** linguist-generated
 subworkflows/nf-core/** linguist-generated

.github/CONTRIBUTING.md

@@ -101,3 +101,19 @@ If you are using a new feature from core Nextflow, you may bump the minimum requ
 ### Images and figures
 
 For overview images and other documents we follow the nf-core [style guidelines and examples](https://nf-co.re/developers/design_guidelines).
+
+## GitHub Codespaces
+
+This repo includes a devcontainer configuration which will create a GitHub Codespaces for Nextflow development! This is an online developer environment that runs in your browser, complete with VSCode and a terminal.
+
+To get started:
+
+- Open the repo in [Codespaces](https://github.com/nf-core/methylseq/codespaces)
+- Tools installed
+  - nf-core
+  - Nextflow
+
+Devcontainer specs:
+
+- [DevContainer config](.devcontainer/devcontainer.json)
+- [Dockerfile](.devcontainer/Dockerfile)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -42,7 +42,7 @@ body:
     attributes:
       label: System information
       description: |
-        * Nextflow version _(eg. 21.10.3)_
+        * Nextflow version _(eg. 22.10.1)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
         * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_

.github/workflows/ci.yml

@@ -4,14 +4,17 @@ on:
   push:
     branches:
       - dev
-      - dsl2
   pull_request:
   release:
     types: [published]
 
 env:
   NXF_ANSI_LOG: false
 
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }}
@@ -21,7 +24,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "21.10.3"
+          - "22.10.1"
           - "latest-everything"
         aligner:
           - bismark
@@ -30,14 +33,32 @@ jobs:
         profile: ["docker"] # TODO , "singularity", "conda"]
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: ./test-datasets/
+          key: methylseq-test-data
 
-      - uses: actions/cache@v2
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
         with:
-          path: /usr/local/bin/nextflow
-          key: ${{ runner.os }}
-          restore-keys: |
-            ${{ runner.os }}-nextflow-
+          repository: nf-core/test-datasets
+          ref: methylseq
+          path: ./test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ./test-datasets/samplesheet/*csv; do
+            sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f
+            sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
@@ -68,7 +89,11 @@ jobs:
         run: conda clean -a
 
       - name: Run nf-test
-        run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.aligner }}/*.nf.test --tap=test.tap
+        run: |
+          nf-test test \
+            --profile=test_ci,test,${{ matrix.profile }} \
+            tests/${{ matrix.aligner }}/*.nf.test \
+            --tap=test.tap
 
       - uses: pcolby/tap-summary@v1
         with:

.github/workflows/fix-linting.yml

@@ -24,7 +24,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
 
       - name: Install Prettier
         run: npm install -g prettier @prettier/plugin-php
@@ -34,9 +34,9 @@ jobs:
         id: prettier_status
         run: |
           if prettier --check ${GITHUB_WORKSPACE}; then
-            echo "::set-output name=result::pass"
+            echo "name=result::pass" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=result::fail"
+            echo "name=result::fail" >> $GITHUB_OUTPUT
           fi
 
       - name: Run 'prettier --write'

.github/workflows/linting.yml

@@ -4,6 +4,8 @@ name: nf-core linting
 # that the code meets the nf-core guidelines.
 on:
   push:
+    branches:
+      - dev
   pull_request:
   release:
     types: [published]
@@ -12,9 +14,9 @@ jobs:
   EditorConfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -25,9 +27,9 @@ jobs:
   Prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
 
       - name: Install Prettier
         run: npm install -g prettier
@@ -38,7 +40,7 @@ jobs:
   PythonBlack:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Check code lints with Black
         uses: psf/black@stable
@@ -69,12 +71,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
           architecture: "x64"
@@ -97,7 +99,7 @@ jobs:
 
       - name: Upload linting log file artifact
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: linting-logs
           path: |

.github/workflows/linting_comment.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Get PR number
         id: pr_number
-        run: echo "::set-output name=pr_number::$(cat linting-logs/PR_number.txt)"
+        run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT
 
       - name: Post PR comment
         uses: marocchino/sticky-pull-request-comment@v2

.gitignore

@@ -8,3 +8,4 @@ testing*
 *.pyc
 .vscode
 .nf-test
+test-datasets/

.prettierignore

@@ -1,5 +1,6 @@
 email_template.html
 adaptivecard.json
+slackreport.json
 .nextflow*
 work/
 data/
@@ -8,4 +9,5 @@ results/
 testing/
 testing*
 *.pyc
+bin/
 *.nf.test.snap

CHANGELOG.md

@@ -1,10 +1,26 @@
 # nf-core/methylseq
 
+## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16
+
+### Pipeline Updates
+
+- ⚙️ Dramatically increase the default process time config requests for Bismark and bwa meth alignment
+- ✨ Add a `tower.yml` file to enable Reports in Nextflow Tower
+- 🤖 GitHub Actions CI - download the test data prior to running tests
+
+### Bug fixes & refactoring
+
+- 🧹 Refactor genome indices preparation into a separate workflow
+- 🧹 Refactor subworkflow logic out of alignment subworkflows, for later sharing
+- 🐛 Fix a bug with using a local genome reference FASTA file
+- 🐛 Fix a bunch of problems in the CI tests using nf-test ([#279](https://github.com/nf-core/methylseq/pull/279))
+
 ## [v2.2.0](https://github.com/nf-core/methylseq/releases/tag/2.2.0) - 2022-11-29
 
 ### Pipeline Updates
 
-- ✨ Updated the `bismark2summary` step so that it no longer stages the aligned BAM files into the working directory. Should be much faster / cheaper for running on the cloud ([#268](https://github.com/nf-core/methylseq/pull/268))
+- ✨ Updated the `bismark2summary` step so that it no longer stages the aligned BAM files into the working directory ([#268](https://github.com/nf-core/methylseq/pull/268))
+  - Should be much faster / cheaper for running on the cloud.
 - ✨ Added ability to merge FastQ files based on shared IDs in sample sheet ([#272](https://github.com/nf-core/methylseq/pull/272))
 
 ### Bug fixes & refactoring
@@ -16,8 +32,6 @@
   - Use the existing option `--cytosine_report` to launch the new `COVERAGE2CYTOSINE` process.
   - Removed option `--cytosine_report genome_index` from the Bismark methylation extractor.
 
-### Software Updates
-
 ## [v2.1.0](https://github.com/nf-core/methylseq/releases/tag/2.1.0) - 2022-11-10
 
 ### Pipeline Updates

README.md

@@ -3,7 +3,7 @@
 [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/methylseq/results)
 [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.1343417-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.1343417)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
@@ -42,7 +42,7 @@ Choose between workflows by using `--aligner bismark` (default, uses bowtie2 for
 
 ## Quick Start
 
-1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`)
+1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`)
 
 2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_.