Taskfile.yml

---
version: 3

env:
  GORACE: halt_on_error=1,history_size=2

vars:
  INTEGRATIONTIME: 30m
  BENCHTIME: 5s
  FUZZTIME: 15s
  FUZZCORPUS: ../fuzz-corpus
  RACEFLAG: -race={{and (ne OS "windows") (ne ARCH "arm")}}
  UNIXSOCKETFLAG: -target-unix-socket={{ne OS "windows"}}
  BUILDTAGS: ferretdb_debug,ferretdb_tigris,ferretdb_hana
  SERVICES: postgres postgres_secured tigris tigris1 tigris2 tigris3 tigris4 mongodb mongodb_secured jaeger

tasks:
  # invoked when `task` is run without arguments
  default:
    deps: [all]

  all:
    desc: "Generate, format, build, test and lint code and documentation"
    cmds:
      - task: gen
      - task: build-host
      - task: test
      - task: lint
      - task: security
      - task: docs

  init-tools:
    dir: tools
    cmds:
      - go mod tidy
      - go mod verify
      - go generate -x

  init-integration:
    dir: integration
    cmds:
      - go mod tidy
      - go mod verify

  init:
    desc: "Install development tools"
    deps: [gen-version, init-tools, init-integration]
    cmds:
      - go mod tidy
      - go mod verify

  env-reset:
    desc: "Clean ALL Go and Docker data (caches, images, volumes), and reset environment"
    cmds:
      - task: env-down
      - cmd: docker buildx prune --all
        ignore_error: true # build container may not exist
      - docker system prune --all --volumes
      - bin/golangci-lint{{exeExt}} cache clean
      - go clean -cache -testcache -modcache -fuzzcache
      - rm -fr .cache .task tools/.task integration/.task tmp
      - task: env-pull
      - task: init

  env-up-detach:
    cmds:
      - >
        docker compose up --always-recreate-deps --force-recreate --remove-orphans --renew-anon-volumes --detach
        --build
        {{.SERVICES}}

  env-up-detach-offline:
    cmds:
      - >
        docker compose up --always-recreate-deps --force-recreate --remove-orphans --renew-anon-volumes --detach
        {{.SERVICES}}

  env-setup:
    deps: [gen-version]
    cmds:
      - go run {{.RACEFLAG}} ./cmd/envtool

  env-logs:
    cmds:
      - docker compose ps --all
      - docker compose logs --follow

  env-up:
    desc: "Start development environment"
    deps: [env-up-detach, env-setup]
    cmds:
      - task: env-logs

  env-up-offline:
    deps: [env-up-detach-offline, env-setup]
    cmds:
      - task: env-logs

  env-pull:
    desc: "Pull development environment's Docker images"
    cmds:
      - docker compose build --pull

  env-down:
    desc: "Stop development environment"
    cmds:
      - docker compose down --remove-orphans --volumes

  env-data:
    desc: "Fill `test` database with data for experiments"
    cmds:
      - bin/task{{exeExt}} -d integration env-data

  gen-version:
    run: once
    cmds:
      - go generate -x ./build/version

  build-host:
    desc: "Build bin/ferretdb{{exeExt}} on the host for development"
    run: once
    deps: [gen-version]
    cmds:
      - go test -c -o=bin/ferretdb{{exeExt}} {{.RACEFLAG}} -tags=ferretdb_testcover,{{.BUILDTAGS}} -coverpkg=./... ./cmd/ferretdb

  gen:
    desc: "Generate (and format) code"
    cmds:
      - go generate -x ./...
      - bin/task{{exeExt}} -d integration integration-gen
      - task: fmt

  fmt:
    desc: "Format code"
    cmds:
      - bin/goimports{{exeExt}} -format-only -local=github.com/FerretDB/FerretDB -w .
      - bin/gofumpt{{exeExt}} -w .

  test:
    desc: "Run all unit and integration tests in parallel"
    deps: [test-unit, test-integration]

  test-unit-short:
    desc: "Run short unit tests (with caching)"
    cmds:
      - go test -short {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverprofile=cover.txt -coverpkg=./... ./...
      - bin/task{{exeExt}} -d tools tools-test

  test-unit:
    desc: "Run all unit tests"
    cmds:
      - go test -count=1 {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverprofile=cover.txt -coverpkg=./... ./...
      - go test -count=1 {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -bench=. -benchtime=1x ./...
      - bin/task{{exeExt}} -d tools tools-test

  test-integration:
    desc: "Run integration tests for `pg` and `tigris` in parallel"
    deps:
      - test-integration-pg
      - test-integration-tigris
      - test-integration-mongodb
      # no test-integration-hana

  test-integration-pg:
    desc: "Run integration tests for `pg` handler"
    dir: integration
    cmds:
      - >
        go test -count=1 -timeout={{.INTEGRATIONTIME}} {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverpkg=../...
        -coverprofile=integration-pg.txt .
        -target-backend=ferretdb-pg
        -target-tls
        -postgresql-url=postgres://username@127.0.0.1:5432/ferretdb
        -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem'

  test-integration-tigris:
    desc: "Run integration tests for `tigris` handler"
    dir: integration
    cmds:
      - >
        go test -count=1 -timeout={{.INTEGRATIONTIME}} {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverpkg=../...
        -coverprofile=integration-tigris.txt .
        -target-backend=ferretdb-tigris
        {{.UNIXSOCKETFLAG}}
        -tigris-urls=127.0.0.1:8081,127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093,127.0.0.1:8094
        -compat-url=mongodb://127.0.0.1:47017/

  test-integration-hana:
    desc: "Run integration tests for `hana` handler"
    dir: integration
    cmds:
      - >
        go test -count=1 -timeout={{.INTEGRATIONTIME}} {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverpkg=../...
        -coverprofile=integration-hana.txt .
        -target-backend=ferretdb-hana
        {{.UNIXSOCKETFLAG}}
        -hana-url=$FERRETDB_HANA_URL
        -compat-url=mongodb://127.0.0.1:47017/

  test-integration-mongodb:
    desc: "Run integration tests for MongoDB"
    dir: integration
    cmds:
      - >
        go test -count=1 -timeout={{.INTEGRATIONTIME}} {{.RACEFLAG}} -tags={{.BUILDTAGS}} -shuffle=on -coverpkg=../...
        -coverprofile=integration-mongodb.txt .
        -target-url=mongodb://127.0.0.1:47017/
        -target-backend=mongodb

  bench-short:
    desc: "Benchmark for about 25 seconds (with default BENCHTIME)"
    cmds:
      - go test -list='Benchmark.*' ./...
      - echo 'Running five functions for {{.BENCHTIME}} each...'
      - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/bson/                  | tee -a new.txt
      - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/bson/                  | tee -a new.txt
      - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/handlers/pg/pjson/     | tee -a new.txt
      - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/handlers/pg/pjson/     | tee -a new.txt
      - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/handlers/tigris/tjson/ | tee -a new.txt
      - bin/benchstat{{exeExt}} old.txt new.txt

  # That's not quite correct: https://github.com/golang/go/issues/15513
  # But good enough for us.
  fuzz-init:
    deps: [gen-version]
    cmds:
      - go test -count=0 ./...

  # Those commands should still run tests (i.e., should not have -run=XXX flags)
  # to fill seed corpus for fuzz tests that use WriteSeedCorpusFile (e.g., FuzzHandler).
  fuzz:
    desc: "Fuzz for about 2 minutes (with default FUZZTIME)"
    cmds:
      - go test -list='Fuzz.*' ./...
      - echo 'Running eight functions for {{.FUZZTIME}} each...'
      - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/bson/
      - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/bson/
      - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/handlers/pg/pjson/
      - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/handlers/pg/pjson/
      - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/handlers/tigris/tjson/
      - go test -fuzz=FuzzMsg      -fuzztime={{.FUZZTIME}} ./internal/wire/
      - go test -fuzz=FuzzQuery    -fuzztime={{.FUZZTIME}} ./internal/wire/
      - go test -fuzz=FuzzReply    -fuzztime={{.FUZZTIME}} ./internal/wire/

  fuzz-corpus:
    desc: "Sync seed and generated fuzz corpora with FUZZCORPUS"
    cmds:
      - go run {{.RACEFLAG}} ./cmd/fuzztool corpus generated {{.FUZZCORPUS}}
      - go run {{.RACEFLAG}} ./cmd/fuzztool corpus seed {{.FUZZCORPUS}}
      - go run {{.RACEFLAG}} ./cmd/fuzztool corpus {{.FUZZCORPUS}} generated

  run:
    desc: "Run FerretDB with `pg` handler"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=:27017
        --proxy-addr=127.0.0.1:47017
        --mode=diff-normal
        --handler=pg
        --postgresql-url=postgres://username@127.0.0.1:5432/ferretdb
        --test-records-dir=tmp/records

  run-tigris:
    desc: "Run FerretDB with `tigris` handler"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=:27017
        --proxy-addr=127.0.0.1:47017
        --mode=diff-normal
        --handler=tigris
        --tigris-url=127.0.0.1:8081
        --test-records-dir=tmp/records

  # set FERRETDB_HANA_URL environment variable to use it
  run-hana:
    desc: "Run FerretDB with `hana` handler"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=:27017
        --proxy-addr=127.0.0.1:47017
        --mode=diff-normal
        --handler=hana
        --test-records-dir=tmp/records

  run-secured:
    desc: "Run FerretDB with `pg` handler (TLS, auth required)"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=''
        --listen-tls=:27018
        --listen-tls-cert-file=./build/certs/server-cert.pem
        --listen-tls-key-file=./build/certs/server-key.pem
        --listen-tls-ca-file=./build/certs/rootCA-cert.pem
        --proxy-addr=127.0.0.1:47017
        --mode=diff-normal
        --handler=pg
        --postgresql-url=postgres://127.0.0.1:5433/ferretdb
        --test-records-dir=tmp/records

  # set FERRETDB_TIGRIS_CLIENT_ID and FERRETDB_TIGRIS_CLIENT_SECRET environment variables to use it
  run-tigris-secured:
    desc: "Run FerretDB with `tigris` handler (TLS, auth required; SaaS)"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=''
        --listen-tls=:27018
        --listen-tls-cert-file=./build/certs/server-cert.pem
        --listen-tls-key-file=./build/certs/server-key.pem
        --listen-tls-ca-file=./build/certs/rootCA-cert.pem
        --proxy-addr=127.0.0.1:47017
        --mode=diff-normal
        --handler=tigris
        --tigris-url=api.preview.tigrisdata.cloud
        --test-records-dir=tmp/records

  run-proxy:
    desc: "Run FerretDB in diff-proxy mode"
    deps: [build-host]
    cmds:
      - >
        bin/ferretdb{{exeExt}} -test.coverprofile=cover.txt --
        --listen-addr=:27017
        --proxy-addr=127.0.0.1:47017
        --mode=diff-proxy
        --handler=pg
        --postgresql-url=postgres://username@127.0.0.1:5432/ferretdb
        --test-records-dir=tmp/records

  lint:
    desc: "Run linters"
    cmds:
      - bin/golangci-lint{{exeExt}} run --fix --config=.golangci.yml
      - bin/golangci-lint{{exeExt}} run --fix --config=.golangci-new.yml
      - bin/go-consistent{{exeExt}} -pedantic ./cmd/... ./internal/... ./build/... ./ferretdb/...
      - bin/go-sumtype{{exeExt}} ./...
      - go vet -vettool=./bin/checkswitch{{exeExt}} ./...
      - bin/task{{exeExt}} -d integration integration-lint
      - bin/task{{exeExt}} -d tools tools-lint
    sources:
      - "**/*.go"
      - "**/go.mod"
      - "**/*.yml"

  security:
    desc: "Run security scanners"
    cmds:
      # don't run them in parallel via `deps` because that breaks terminal output
      - task: security-trivy
      - task: security-govulncheck

  security-trivy:
    cmds:
      - >
        docker compose run --rm trivy filesystem .
        --secret-config=./build/trivy-secret.yml
        --ignorefile=./build/.trivyignore
        --cache-dir=./.cache/trivy
        --exit-code=1

  security-govulncheck:
    cmds:
      - bin/govulncheck{{exeExt}} -v -test ./...
      - bin/task{{exeExt}} -d integration integration-security

  godocs:
    desc: "Serve godoc documentation"
    cmds:
      - cmd: open http://127.0.0.1:6060/pkg/github.com/FerretDB/FerretDB/?m=all
        ignore_error: true # open might not work on that platform
      - bin/godoc{{exeExt}} -http=127.0.0.1:6060

  psql:
    desc: "Run psql"
    cmds:
      - docker compose exec -e PGPASSWORD=password postgres psql -U username -d ferretdb

  mongosh:
    desc: "Run MongoDB shell (`mongosh`)"
    cmds:
      - >
        docker compose exec mongodb mongosh
        --verbose --eval 'disableTelemetry()' --shell
        'mongodb://host.docker.internal:27017/?heartbeatFrequencyMS=300000'

  mongosh-secured:
    desc: "Run MongoDB shell (`mongosh`) with TLS and auth"
    cmds:
      - >
        docker compose exec mongodb mongosh
        --verbose --eval 'disableTelemetry()' --shell
        'mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem'

  testjs:
    desc: "Run legacy MongoDB shell (`mongo`) with test.js script"
    cmds:
      - >
        docker compose run --rm legacy-mongo-shell
        'mongodb://host.docker.internal:{{.PORT | default 27017}}/'
        /legacy-mongo-shell/test.js

  docker-init:
    cmds:
      - docker buildx create --driver=docker-container --name=ferretdb --bootstrap=true --use=false
      - docker buildx ls

  docker-build:
    deps: [gen-version]
    cmds:
      - >
        docker buildx build --builder=ferretdb
        --file=build/docker/{{.FILE}}.Dockerfile
        --build-arg=LABEL_VERSION={{.VERSION}}
        --build-arg=LABEL_COMMIT={{.COMMIT}}
        --platform={{default "linux" .PLATFORM}}
        {{range splitList "," .DOCKER_IMAGES}}--tag={{trim .}} {{end -}}
        --output={{.OUTPUT}}
        .
    vars:
      VERSION:
        sh: cat build/version/version.txt
      COMMIT:
        sh: cat build/version/commit.txt

  # do not change that target much; see "Building and packaging" in README.md
  build-release:
    desc: "Build bin/ferretdb (production build)"
    deps: [gen-version]
    cmds:
      - echo 'bin' > build/version/package.txt
      - task: docker-build
        vars:
          FILE: production
          PLATFORM: linux/amd64 # TODO https://github.com/FerretDB/FerretDB/issues/2171
          DOCKER_IMAGES: ignored
          OUTPUT: type=local,dest=tmp/build
      - mv tmp/build/ferretdb bin/ferretdb

  docker-local:
    desc: "Build `ferretdb-local` Docker image for the host platform"
    cmds:
      - echo 'docker-local' > build/version/package.txt
      - task: docker-build
        vars:
          FILE: development
          DOCKER_IMAGES: ferretdb-local
          OUTPUT: type=docker

  # TODO https://github.com/FerretDB/FerretDB/issues/2212
  docker-all-in-one-push:
    cmds:
      - echo 'docker-all-in-one' > build/version/package.txt
      - task: docker-build
        vars:
          FILE: all-in-one
          PLATFORM: linux/amd64,linux/arm64 # no mongosh for arm/v7
          OUTPUT: type=image,push=true

  # TODO https://github.com/FerretDB/FerretDB/issues/2212
  docker-development-push:
    cmds:
      - echo 'docker' > build/version/package.txt
      - task: docker-build
        vars:
          FILE: development
          PLATFORM: linux/amd64,linux/arm64,linux/arm/v7
          OUTPUT: type=image,push=true

  # TODO https://github.com/FerretDB/FerretDB/issues/2212
  docker-production-push:
    cmds:
      - echo 'docker' > build/version/package.txt
      - task: docker-build
        vars:
          FILE: production
          PLATFORM: linux/amd64,linux/arm64,linux/arm/v7
          OUTPUT: type=image,push=true

  packages:
    cmds:
      - task: packages-deb
      - task: packages-rpm
      # TODO https://github.com/FerretDB/FerretDB/issues/2172

  packages-deb:
    cmds:
      - echo 'deb' > build/version/package.txt
      - mkdir -p tmp/debs
      - task: docker-build
        vars:
          FILE: production
          PLATFORM: linux/amd64 # TODO https://github.com/FerretDB/FerretDB/issues/2171
          DOCKER_IMAGES: ignored
          OUTPUT: type=local,dest=tmp/build
      - cmd: bin/nfpm{{exeExt}} package --config=build/nfpm.yml --packager=deb --target=tmp/debs/ferretdb.deb
        env:
          ARCH: amd64
          VERSION:
            sh: cat build/version/version.txt
      - docker compose run --rm ubuntu /bin/sh -c 'dpkg -i /debs/ferretdb.deb && ferretdb --version'

  packages-rpm:
    cmds:
      - echo 'rpm' > build/version/package.txt
      - mkdir -p tmp/rpms
      - task: docker-build
        vars:
          FILE: production
          PLATFORM: linux/amd64 # TODO https://github.com/FerretDB/FerretDB/issues/2171
          DOCKER_IMAGES: ignored
          OUTPUT: type=local,dest=tmp/build
      - cmd: bin/nfpm{{exeExt}} package --config=build/nfpm.yml --packager=rpm --target=tmp/rpms/ferretdb.rpm
        env:
          ARCH: amd64
          VERSION:
            sh: cat build/version/version.txt
      - docker compose run --rm ubi /bin/sh -c 'rpm -i /rpms/ferretdb.rpm && ferretdb --version'

  docs:
    desc: "Format, lint and build documentation"
    deps: [docs-fmt]
    cmds:
      - docker compose run --rm docusaurus-docs build

  blog:
    desc: "Format, lint and build blog"
    deps: [docs-fmt]
    cmds:
      - docker compose run --rm docusaurus-blog build

  # see https://github.com/DavidAnson/markdownlint-cli2#command-line for the reason we use double-quotes
  docs-fmt:
    desc: "Format and lint documentation"
    cmds:
      - docker compose run --rm textlint --fix --config build/.textlintrc "**/*.md" ".github/**/*.md"
      - docker compose run --rm markdownlint "**/*.md" "#CHANGELOG.md"

  docs-dev:
    desc: "Start documentation development server"
    cmds:
      - docker compose run --rm --service-ports docusaurus-docs start --host=0.0.0.0

  docs-cloudflare-preview:
    cmds:
      # for debugging
      - >
        docker compose run --rm
        --entrypoint='/bin/sh -c "git config --global --add safe.directory /workdir && git status"'
        wrangler
      - docker compose run --rm wrangler pages deployment create --project-name=docs website/build

  blog-dev:
    desc: "Start blog development server"
    cmds:
      - docker compose run --rm --service-ports docusaurus-blog start --host=0.0.0.0 --port=3001