diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb5c273..6537a44 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,8 @@ name: Build OpenAPI docs on: - - push - # - pull_request + push: + pull_request: + types: [opened, reopened, sychronize] jobs: docs-build: runs-on: ubuntu-latest @@ -21,24 +22,5 @@ jobs: with: branch: gh-pages folder: . - deploy: - if: github.ref == 'refs/heads/master' # Only run this job on the master branch - runs-on: ubuntu-latest - steps: - - name: Setup Node.js environment - uses: actions/setup-node@v4 - with: - node-version: 14.x - # Comes with npm 6. For newer Node, encountered: https://github.com/npm/cli/issues/3359 - - run: npm install -g @redocly/openapi-cli && npm install -g redoc-cli - - run: npm install -g gh-openapi-docs - - name: Check out repository code - uses: actions/checkout@v4 - - run: gh-openapi-docs - - name: Deploy 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - with: - branch: gh-pages - folder: . # Folder containing the built documentation - target-folder: docs/index.html # Deploy to docs/index.html if needed - clean: true + target-folder: ${{ github.ref == 'refs/heads/main' && 'docs/index.html' || '' }} # Deploy to docs/index.html only if on main + clean: ${{ github.ref == 'refs/heads/main' }} # Clean only if on the main branch \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d4a9fc..e12d7da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ name: Lint and validate OpenAPI specs on: - - push - - pull_request_target + - pull_request jobs: lint: name: Lint OpenAPI definition @@ -10,17 +9,16 @@ jobs: - name: Check out head branch uses: actions/checkout@v4 - name: Run OpenAPI Lint Action - uses: nwestfall/openapi-action@v1.0.2 + uses: mhiew/redoc-lint-github-action@v4 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - file: openapi/task_execution_service.openapi.yaml + args: 'openapi/task_execution_service.openapi.yaml' diff: name: Show OpenAPI differences relative to target branch runs-on: ubuntu-latest if: ${{ github.event_name == 'pull_request' }} steps: - name: Check out head branch - uses: aactions/checkout@v4 + uses: actions/checkout@v4 with: ref: ${{ github.ref }} path: head diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b90a8fd..171be80 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,9 @@ Suggested changes to this schema can be initiated as [**Issues**](https://github Even those with write access to the main repository should in general create pull request branches within their own forks. This way when the main repository is forked again, the new fork is created with a minimum of extraneous volatile branches. -> To facilitate review of external pull requests, users are encouraged to activate [**Travis CI**](https://travis-ci.org/) to monitor the build status (documentation, Swagger UI) of their fork. By following the documentation for [deployment to GitHub Pages](https://docs.travis-ci.com/user/deployment/pages/) and adding a `$GITHUB_TOKEN` environment variable to their repo configuration, pushes to the forked repo should be viewable relative to `https://[user-or-org].github.io/workflow-execution-service-schemas/preview//`: + +> To facilitate the review of external pull requests, users are encouraged to activate [**GitHub Actions**](https://github.com/features/actions) to monitor the build status (documentation, Swagger UI) of their fork. By following the documentation for [deployment to GitHub Pages](https://docs.github.com/en/pages/getting-started-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site), pushes to the forked repository will be viewable at `https://[user-or-org].github.io/task-execution-service/preview//`: + + https://[user-or-org].github.io/task-execution-schemas/preview/\/docs/ + https://[user-or-org].github.io/task-execution-schemas/preview/\/swagger-ui/ @@ -24,11 +26,4 @@ Even those with write access to the main repository should in general create pul If a security vulnerability is identified with the specification please send an email to security-notification@ga4gh.org detailing your concerns. -## Approving Changes - -### pre-TES v1.0.0 / Testbed Voting Procedure -Changes for the release are to be approved by 2 TES specification leads. - -### post TES v1.0.0 Voting Procedure -The post v1.0.0 voting group include stakeholders, such as server and client implementors. -The membership of this group will be established as part of the v1.0.0 release. +For more information please see the [Governance](GOVERNANCE.md) documentation. diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..e0ef422 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,23 @@ +TES Governance and Process +============================== + +The Task Execution Service (TES) operates under a community-driven development model, where advancements to the specification occur exclusively through community contributions. The governance is steered by a core team of API champions that fosters collaboration and consensus among contributors. This team is tasked with overseeing the design and development processes, setting priorities, managing the release schedule, and making decisions in instances where consensus cannot be reached. + +Current API Product leads are: + +| Name | Organization | github | +|:-------------------|:-------------|:------------------ +| Kyle Ellrot | [Oregon Health and Science University](https://www.ohsu.edu/) | [kellrott](https://github.com/kellrott) | +| Venkat Malladi | [Microsoft](https://www.microsoft.com/en-us/genomics/) | [vsmalladi](https://github.com/vsmalladi) | +| Alex Kanitz | [Swiss Institute of Bioinformatics / ELIXIR Switzerland](https://www.sib.swiss/) | [uniqueg](https://github.com/uniqueg) | + + +## Voting Procedure + +Voting group include stakeholders, such as server and client implementors. + +API Product leads review PR's on a reguarly cal (and also ping the mailing list), set a deadline by which drivers (and a few key non-drivers) need to respond by. Strive to reach consensus with our drivers and community. + +Cloud Work Stream leads retain a veto if something goes off the rails. + +[GA4GH TES API mailing list](mailto:ga4gh-cloud-tes+subscribe@ga4gh.org): to stay updated about the latest news & developments around the TES API, in particular for soliciting comments on proposed spec changes and for notifications about TES subgroup meetings. diff --git a/README.md b/README.md index ff75c5a..3d29f42 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,31 @@ GA4GH Logo -![release_badge](https://img.shields.io/github/v/tag/ga4gh/task-execution-schemas) +![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg) [![release_badge](https://img.shields.io/github/v/tag/ga4gh/task-execution-schemas?label=latest%20release&logo=github&style=flat)](https://github.com/ga4gh/task-execution-schemas/releases) + +`main` branch status + +| Build | CI | Open API Validator | +|------------------------|-----------|--------------------| +| ![Build Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml/badge.svg?branch=main) | ![CI Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml/badge.svg?branch=main) | [![Open API Validator](https://img.shields.io/badge/OpenAPI-Validator-blue)](http://online.swagger.io/validator?url=https://raw.githubusercontent.com/ga4gh/task-execution-schemas/main/openapi/task_execution_service.openapi.yaml) | + +`develop` branch status + +| Build | CI | Open API Validator | +|------------------------|-----------|--------------------| +| ![Build Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml/badge.svg?branch=develop) | ![CI Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml/badge.svg?branch=develop) | [![Open API Validator](https://img.shields.io/badge/OpenAPI-Validator-blue)](https://online.swagger.io/validator?url=https://raw.githubusercontent.com/ga4gh/task-execution-schemas/develop/openapi/task_execution_service.openapi.yaml) | Task Execution Service (TES) API ====================================== -This repository is the home for the schema for the GA4GH Task Execution API. The Goal of the API is to provide a standardized way to executing batch execution tasks. +This repository is the home for the schema of the Task Execution Service (TES) API defined by the [Cloud Work Stream](https://www.ga4gh.org/work_stream/cloud/) of the [Global Alliance for Genomics and Health (GA4GH)](https://ga4gh.org/). The goal of the API is to provide a standardized way to executing batch execution tasks. -The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) is an international coalition, formed to enable the sharing of genomic and clinical data. +[GA4GH](https://ga4gh.org/) is an international coalition, formed to enable the sharing of genomic and clinical data. Cloud Work Stream ----------------- -The [Cloud Work Stream](https://ga4gh.cloud) helps the genomics and health communities take full advantage of modern cloud environments. -Our initial focus is on “bringing the algorithms to the data”, by creating standards for defining, sharing, and executing portable workflows. +The [Cloud Work Stream](https://www.ga4gh.org/work_stream/cloud/) helps the genomics and health communities take full advantage of modern cloud environments. Our initial focus is on “bringing the algorithms to the data”, by creating standards for defining, sharing, and executing portable workflows.g We work with platform development partners and industry leaders to develop standards that will facilitate interoperability. @@ -29,23 +40,22 @@ and some other logging and metadata. API Definition -------------- -See the human-readable [Reference Documentation](https://ga4gh.github.io/task-execution-schemas/docs/) -You can also explore the specification in -the [Swagger Editor](https://editor.swagger.io/?url=https://ga4gh.github.io/task-execution-schemas/openapi.yaml)** -*Manually load the JSON if working from a non-develop branch version.* Preview documentation from -the [gh-openapi-docs](https://github.com/ga4gh/gh-openapi-docs) for the development -branch [here](https://ga4gh.github.io/task-execution-schemas/preview/develop/docs/index.html) +See the human-readable [Reference Documentation](https://ga4gh.github.io/task-execution-schemas/docs/). -> All documentation and pages hosted at 'ga4gh.github.io/task-execution-schemas' reflect the latest API release from -> the `master` branch. To monitor the latest development work, add 'preview/\' to the URLs above (e.g., ' -> ga4gh.github.io/task-execution-schemas/preview/\/docs'). To view the latest *stable* -> development API specification, refer to the `develop` branch. + +You can also examine the specification in the [Swagger Editor](https://editor.swagger.io/?url=https://ga4gh.github.io/task-execution-schemas/openapi.yaml). + +*If you are using a version from a non-develop branch, please load the JSON manually.* +All documentation and pages hosted at 'ga4gh.github.io/task-execution-schemas' reflect the latest API release from +the `main` branch. To monitor the latest development work, add 'preview/\' to the URLs above (e.g., ' +ga4gh.github.io/task-execution-schemas/preview/\/docs'). To view the latest *stable* +development API specification, refer to the `develop` branch. TES Compliant Implementations ------------------------------ -> A stand-alone security review has been performed on the API. Nevertheless, any implementation that is linked to from the documentation accompanying the API is done so without any security guarantees. If you integrate this code into your application it is AT YOUR OWN RISK AND RESPONSIBILITY to arrange for an audit to ensure compliance with any applicable regulatory and security requirements, especially where personal data may be at issue. +In alignment with GA4GH policies and regulations, security reviews are conducted for each major version release of the API. However, **no security guarantees are provided for any implementation of the API, including those linked from this page or the associated documentation**. Users are advised to proceed at their own risk and should arrange for a security audit of their application to ensure compliance with relevant regulatory and security standards, particularly when handling personal data. ### Client - [cwl-tes](https://github.com/ohsu-comp-bio/cwl-tes) @@ -91,41 +101,45 @@ Here's an example of a complete task message, defining a task which calculates an MD5 checksum on an input file and uploads the output: ```JSON { - "name": "MD5 example", - "description": "Task which runs md5sum on the input file.", - "tags": { + "name": "MD5 example", + "description": "Task which runs md5sum on the input file.", + "tags": { "custom-tag": "tag-value" - }, - "inputs": [ + }, + "inputs": [ { - "name": "infile", - "description": "md5sum input file", - "url": "/path/to/input_file", - "path": "/container/input", - "type": "FILE" + "name": "infile", + "description": "md5sum input file", + "url": "/path/to/input_file", + "path": "/container/input", + "type": "FILE" } - ], - "outputs" : [ + ], + "outputs": [ { - "url" : "/path/to/output_file", - "path" : "/container/output" + "name": "outfile", + "url": "/path/to/output_file", + "path": "/container/output" } - ], - "resources" : { + ], + "resources": { "cpuCores": 1, - "ramGb": 1.0, - "diskGb": 100.0, + "ramGb": 1, + "diskGb": 100, "preemptible": false - }, - "executors" : [ + }, + "executors": [ { - "image" : "ubuntu", - "command" : ["md5sum", "/container/input"], - "stdout" : "/container/output", - "stderr" : "/container/stderr", - "workdir": "/tmp" + "image": "ubuntu", + "command": [ + "md5sum", + "/container/input" + ], + "stdout": "/container/output", + "stderr": "/container/stderr", + "workdir": "/tmp" } - ] + ] } ``` @@ -157,11 +171,14 @@ A minimal version of the same task, including only the required fields looks lik To create the task, send an HTTP POST request: ```HTTP POST /v1/tasks - -{ "id": "task-1234" } ``` The return value is a task ID. +```JSON +{ + "id": "task-1234" +} +``` ### Get a task @@ -171,12 +188,17 @@ To get a task by ID: ```HTTP GET /v1/tasks/task-1234 - -{ "id": "task-1234", "state": "RUNNING" } ``` The return value will be a minimal description of the task state. +```JSON +{ + "id": "task-1234", + "state": "RUNNING" +} +``` + To get more information, you can change the task view using the `view` URL query parameter. The `basic` view will include all task fields except a few which might be @@ -184,17 +206,138 @@ large strings (stdout/err/system logging, input parameter contents). ```HTTP GET /v1/tasks/task-1234?view=BASIC - -{ "id": "task-1234", "state": "RUNNING", "name": "MD5 example", etc... } ``` +```JSON +{ + "id": "task123", + "name": "Sample Task", + "description": "This is a sample task description.", + "state": "COMPLETED", + "inputs": [ + { + "name": "infile", + "description": "Input file for the task.", + "url": "/path/to/input_file", + "path": "/container/input", + "type": "FILE" + } + ], + "outputs": [ + { + "name": "outfile", + "url": "/path/to/output_file", + "path": "/container/output" + } + ], + "resources": { + "cpuCores": 1, + "ramGb": 2.0, + "diskGb": 10.0, + "preemptible": false + }, + "executors": [ + { + "image": "ubuntu:latest", + "command": ["command", "arg1", "arg2"], + "stdout": "/container/output", + "stderr": "/container/stderr", + "workdir": "/tmp" + } + ], + "created": "2024-10-24T12:00:00Z", + "updated": "2024-10-24T12:30:00Z" +} + +``` The `full` view includes stdout/err/system logs and full input parameters: ```HTTP GET /v1/tasks/task-1234?view=FULL +``` -{ "id": "task-1234", "state": "RUNNING", "name": "MD5 example", - "logs": [{ "stdout": "stdout content..." }], etc... } +```JSON +{ + "id": "job-0012345", + "state": "COMPLETE", + "name": "MD5 Checksum Task", + "description": "This task computes the MD5 checksum of the input file.", + "inputs": [ + { + "url": "s3://my-object-store/file1", + "path": "/data/file1" + } + ], + "outputs": [ + { + "path": "/data/outfile", + "url": "s3://my-object-store/outfile-1", + "type": "FILE" + } + ], + "resources": { + "cpu_cores": 4, + "preemptible": false, + "ram_gb": 8, + "disk_gb": 40, + "zones": "us-west-1", + "backend_parameters": { + "VmSize": "Standard_D64_v3" + }, + "backend_parameters_strict": false + }, + "executors": [ + { + "image": "ubuntu:20.04", + "command": [ + "/bin/md5", + "/data/file1" + ], + "workdir": "/data/", + "stdin": "/data/file1", + "stdout": "/tmp/stdout.log", + "stderr": "/tmp/stderr.log", + "ignore_error": true + } + ], + "volumes": [ + "/vol/A/" + ], + "tags": { + "WORKFLOW_ID": "cwl-01234", + "PROJECT_GROUP": "alice-lab" + }, + "logs": [ + { + "logs": [ + { + "start_time": "2020-10-02T10:00:00-05:00", + "end_time": "2020-10-02T11:00:00-05:00", + "stdout": "MD5 checksum calculation completed successfully.", + "stderr": "", + "exit_code": 0 + } + ], + "metadata": { + "host": "worker-001", + "slurmm_id": 123456 + }, + "start_time": "2020-10-02T10:00:00-05:00", + "end_time": "2020-10-02T11:00:00-05:00", + "outputs": [ + { + "url": "s3://my-object-store/outfile-1", + "path": "/data/outfile", + "size_bytes": 1024 + } + ], + "system_logs": [ + "Task executed successfully without any issues." + ] + } + ], + "creation_time": "2020-10-02T10:00:00-05:00" +} ``` ### List tasks @@ -204,10 +347,29 @@ To list tasks: ```HTTP GET /v1/tasks +``` + +```JSON +{ + "tasks": [ + { + "id": "job-0012345", + "state": "COMPLETE" + }, + { + "id": "job-0012346", + "state": "RUNNING" + }, + { + "id": "job-0012347", + "state": "FAILED" + } + ] +} -{ "tasks": [{ "id": "task-1234", "state": "RUNNING"}, etc...] } ``` + Similar to getting a task by ID, you may change the task view: ```HTTP GET /v1/tasks?view=BASIC @@ -222,16 +384,20 @@ To cancel a task, send an HTTP POST to the cancel endpoint: POST /v1/tasks/task-1234:cancel ``` -Possible Future Enhancements ----------------------------- - -* Integrate with GA4GH DRS to resolve input data source (possibly support for DRS URIs as permissible values of input URLs). -* Integrate with GA4GH TRS to resolve container images (possibly support for TRS URIs as permissible values of executor image names). - How to Contribute Changes ------------------------- -See [CONTRIBUTING.md](CONTRIBUTING.md). + +### Community Contributions and Spec Advancement + +The advancement of the GA4GH Task Execution Schema (TES) relies on active community engagement and contributions. While submitting issues is an effective way to report bugs or foster discussions about existing or proposed features, it is important to note that these actions alone do not lead to modifications in the specification. The most effective method for implementing changes is through the submission of a pull request (PR). + +For detailed guidance on how to contribute, please refer to the [Contributing](CONTRIBUTING.md) document. If a security issue is identified with the specification, please send an email to security-notification@ga4gh.org detailing your concerns. + + +### Governance + +The TES specification is entirely community driven, however it is overseen by a governance committee. For more information please see the [Governance](GOVERNANCE.md) documentation. diff --git a/openapi/task_execution_service.openapi.yaml b/openapi/task_execution_service.openapi.yaml index 8b98283..4a61b45 100644 --- a/openapi/task_execution_service.openapi.yaml +++ b/openapi/task_execution_service.openapi.yaml @@ -63,6 +63,10 @@ info: If TES API implementation is to be used by another website or domain it must implement Cross Origin Resource Sharing (CORS). Please refer to https://w3id.org/ga4gh/product-approval-support/cors for more information about GA4GH’s recommendations and how to implement CORS. +security: + # Optionally, this section can be removed or adjusted by implementers based on their needs. + - BearerAuth: [] # Apply globally if authentication is required + - BasicAuth: [] # Alternatively, use Basic Authentication servers: - url: /ga4gh/tes/v1 @@ -85,6 +89,9 @@ paths: application/json: schema: $ref: '#/components/schemas/tesServiceInfo' + security: + - BearerAuth: [] # Endpoint secured with Bearer token + - BasicAuth: [] # Alternatively, secured with Basic Auth /tasks: get: tags: @@ -179,6 +186,11 @@ paths: application/json: schema: $ref: '#/components/schemas/tesListTasksResponse' + + security: + - BearerAuth: [] # Endpoint secured with Bearer token + - BasicAuth: [] # Alternatively, secured with Basic Auth + post: tags: - TaskService @@ -201,6 +213,9 @@ paths: schema: $ref: '#/components/schemas/tesCreateTaskResponse' x-codegen-request-body-name: body + security: + - BearerAuth: [] # Endpoint secured with Bearer token + - BasicAuth: [] # Alternatively, secured with Basic Auth /tasks/{id}: get: tags: @@ -224,6 +239,9 @@ paths: application/json: schema: $ref: '#/components/schemas/tesTask' + security: + - BearerAuth: [] # Endpoint secured with Bearer token + - BasicAuth: [] # Alternatively, secured with Basic Auth /tasks/{id}:cancel: post: tags: @@ -245,7 +263,20 @@ paths: application/json: schema: $ref: '#/components/schemas/tesCancelTaskResponse' + security: + - BearerAuth: [] # Endpoint secured with Bearer token + - BasicAuth: [] # Alternatively, secured with Basic Auth components: + securitySchemes: + BearerAuth: + type: http + scheme: bearer + bearerFormat: JWT # optional, can specify format if using JWT tokens + + BasicAuth: + type: http + scheme: basic + parameters: view: name: view