From c3a51a38c556bfb1e8a0038e93301af82463201f Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 6 Feb 2025 17:15:39 +0000 Subject: [PATCH] Deps management improv --- .devcontainer/Dockerfile | 12 +++++ .devcontainer/devcontainer.json | 52 ++++++++++--------- .devcontainer/post_create_setup.sh | 11 ++++ .github/workflows/ci.yml | 4 +- .github/workflows/long-test.yml | 2 +- .github/workflows/release.yml | 6 +-- scripts/install-azure-linux-deps.sh | 24 --------- scripts/install-azure-linux-dev.sh | 0 scripts/setup-ci-basic.sh | 35 +++++++++++++ ...ure-linux-extended.sh => setup-ci-full.sh} | 11 ++-- scripts/setup-dev.sh | 9 ++++ 11 files changed, 108 insertions(+), 58 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100755 .devcontainer/post_create_setup.sh delete mode 100755 scripts/install-azure-linux-deps.sh create mode 100644 scripts/install-azure-linux-dev.sh create mode 100755 scripts/setup-ci-basic.sh rename scripts/{install-azure-linux-extended.sh => setup-ci-full.sh} (74%) create mode 100755 scripts/setup-dev.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000000..13ad49107f5a --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +ARG BASE_IMAGE=mcr.microsoft.com/azurelinux/base/core:3.0 +FROM ${BASE_IMAGE} + +# ? +RUN gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY \ + && tdnf -y update + +# VSCode server dependency +RUN tdnf -y install awk diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b5be8162b195..706ad301e0fa 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,31 +1,35 @@ { "name": "CCF Development Environment", - "image": "ghcr.io/microsoft/ccf/ci/default:latest", + "build": { + "dockerfile": "Dockerfile", + "context": "." + }, "runArgs": [], - "extensions": [ - "eamodio.gitlens", - "esbenp.prettier-vscode", - "me-dutour-mathieu.vscode-github-actions", - "ms-azure-devops.azure-pipelines", - "ms-python.black-formatter", - "ms-python.python", - "ms-vscode.cpptools" - ], - "settings": { - "python.defaultInterpreterPath": "python3", - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "waderyan.gitblame", + "esbenp.prettier-vscode", + "github.vscode-github-actions", + "ms-python.black-formatter", + "ms-python.python", + "ms-vscode.cpptools", + "github.copilot" + ] }, - "python.formatting.provider": "none", - "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.fixAll": true + "settings": { + "python.defaultInterpreterPath": "python3", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "python.formatting.provider": "none", + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll": true + } } }, - "features": { - "ghcr.io/devcontainers/features/docker-from-docker:1": { - "version": "latest" - } - } + "postCreateCommand": "./.devcontainer/post_create_setup.sh" } diff --git a/.devcontainer/post_create_setup.sh b/.devcontainer/post_create_setup.sh new file mode 100755 index 000000000000..852694c66e0e --- /dev/null +++ b/.devcontainer/post_create_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +./scripts/setup-ci-basic.sh +./scripts/setup-ci-full.sh +./scripts/setup-dev.sh + +git config --global --add safe.directory /workspaces/CCF \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67d070e70615..12d8a8726deb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: shell: bash run: | set -ex - ./scripts/install-azure-linux-deps.sh + ./scripts/setup-ci-basic.sh - name: "Build Debug" run: | @@ -148,5 +148,5 @@ jobs: # Unit tests ./tests.sh --output-on-failure -L unit -j$(nproc --all) # All other acceptably fast tests, which are now supported on Mariner. - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|protocolstest|suite|unit|lts_compatibility" + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit|lts" shell: bash diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 8d435e90eb3b..b066a9f06237 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -303,7 +303,7 @@ jobs: shell: bash run: | set -ex - ./scripts/install-azure-linux-deps.sh + ./scripts/setup-ci-basic.sh - name: "Build Debug (Long Test)" run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8af240ef2b6..9840349889bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,10 +105,8 @@ jobs: shell: bash run: | set -ex - ./scripts/install-azure-linux-deps.sh - ./scripts/install-azure-linux-extended.sh - # For packaging and release tests - tdnf -y install rpm-build + ./scripts/setup-ci-basic.sh + ./scripts/setup-ci-full.sh - name: "Build Release ${{ matrix.platform.name }}" shell: bash diff --git a/scripts/install-azure-linux-deps.sh b/scripts/install-azure-linux-deps.sh deleted file mode 100755 index f610712c2919..000000000000 --- a/scripts/install-azure-linux-deps.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -set -ex - -# Source control -tdnf -y install ca-certificates git - -# Build tools -tdnf -y install build-essential clang cmake ninja-build which - -# libc++ -tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - -# Dependencies -tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - -# Test dependencies -tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect procps - -# Install CDDL via rubygems -tdnf -y install rubygems -gem install cddl diff --git a/scripts/install-azure-linux-dev.sh b/scripts/install-azure-linux-dev.sh new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/setup-ci-basic.sh b/scripts/setup-ci-basic.sh new file mode 100755 index 000000000000..2e217ca27854 --- /dev/null +++ b/scripts/setup-ci-basic.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +# Source control +tdnf -y install \ + git \ + ca-certificates + +# To build CCF +tdnf -y install \ + build-essential \ + clang \ + cmake \ + ninja-build \ + which \ + openssl-devel \ + libuv-devel \ + nghttp2-devel \ + curl-devel \ + libarrow-devel \ + parquet-libs-devel + +# To run standard tests +tdnf -y install \ + lldb \ + expect \ + npm \ + jq + +# Extra-dependency for CDDL schema checker +tdnf -y install rubygems +gem install cddl diff --git a/scripts/install-azure-linux-extended.sh b/scripts/setup-ci-full.sh similarity index 74% rename from scripts/install-azure-linux-extended.sh rename to scripts/setup-ci-full.sh index 7e7d9f8fd38c..29c0d514160d 100755 --- a/scripts/install-azure-linux-extended.sh +++ b/scripts/setup-ci-full.sh @@ -6,11 +6,16 @@ set -ex H2SPEC_VERSION="v2.6.0" -# protocoltest dependency -tdnf install -y bind-utils +# Release (extended) tests +tdnf -y install procps -# protocoltest dependency +# protocoltest +tdnf install -y bind-utils curl -L --output h2spec_linux_amd64.tar.gz https://github.com/summerwind/h2spec/releases/download/$H2SPEC_VERSION/h2spec_linux_amd64.tar.gz tar -xvf h2spec_linux_amd64.tar.gz mkdir /opt/h2spec mv h2spec /opt/h2spec/h2spec +rm h2spec_linux_amd64.tar.gz + +# For packaging +tdnf -y install rpm-build diff --git a/scripts/setup-dev.sh b/scripts/setup-dev.sh new file mode 100755 index 000000000000..5491c7a11521 --- /dev/null +++ b/scripts/setup-dev.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +tdnf -y install \ + vim \ + shellcheck