From fb4a8454d9dab4c743e0041dba955d952770e98f Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 23 Sep 2024 14:36:49 +0300 Subject: [PATCH 1/3] Copy the Github Actions workflows from the main repo Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-rust-audit.yml | 69 ++++++ .github/workflows/test-lang-rust-ci-ARM.yml | 71 ++++++ .github/workflows/test-lang-rust-ci.yml | 247 ++++++++++++++++++++ .github/workflows/test-lang-rust-clippy.yml | 50 ++++ 4 files changed, 437 insertions(+) create mode 100644 .github/workflows/test-lang-rust-audit.yml create mode 100644 .github/workflows/test-lang-rust-ci-ARM.yml create mode 100644 .github/workflows/test-lang-rust-ci.yml create mode 100644 .github/workflows/test-lang-rust-clippy.yml diff --git a/.github/workflows/test-lang-rust-audit.yml b/.github/workflows/test-lang-rust-audit.yml new file mode 100644 index 0000000..0649ae9 --- /dev/null +++ b/.github/workflows/test-lang-rust-audit.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rust Security Audit +on: + workflow_dispatch: + push: + branches: [ main ] + paths: + - .github/workflows/test-lang-rust-audit.yml + - '**/Cargo.toml' + - Cargo.lock + - deny.toml + pull_request: + branches: [ main ] + paths: + - .github/workflows/test-lang-rust-audit.yml + - '**/Cargo.toml' + - Cargo.lock + - deny.toml + +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Dependency Review + if: github.event_name == 'pull_request' + uses: actions/dependency-review-action@v4 + - name: Install Cargo Audit + run: cargo install cargo-audit + - name: Audit + run: cargo audit + + deny: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Cargo Deny + run: cargo install cargo-deny + - name: Check + run: cargo deny check + \ No newline at end of file diff --git a/.github/workflows/test-lang-rust-ci-ARM.yml b/.github/workflows/test-lang-rust-ci-ARM.yml new file mode 100644 index 0000000..fe2ec39 --- /dev/null +++ b/.github/workflows/test-lang-rust-ci-ARM.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Rust Continuous Integration on ARM' +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + arm64: + name: Rust on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Cargo + uses: actions/cache@v4 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: ${{ runner.os }}-target-arm64-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-stable- + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: stable + components: rustfmt + targets: x86_64-unknown-linux-gnu + + - name: Build + run: | + set -x + ./build.sh test diff --git a/.github/workflows/test-lang-rust-ci.yml b/.github/workflows/test-lang-rust-ci.yml new file mode 100644 index 0000000..9bbd2ba --- /dev/null +++ b/.github/workflows/test-lang-rust-ci.yml @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rust Continuous Integration +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ci: + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - 'stable' + - 'beta' + - 'nightly' + - '1.73.0' # MSRV + target: + - x86_64-unknown-linux-gnu + - wasm32-unknown-unknown + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Cargo + uses: actions/cache@v4 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: ${{ runner.os }}-target-cache1-${{ hashFiles('**/Cargo.lock') }} + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: ${{ matrix.rust }} + components: rustfmt + targets: ${{ matrix.target }} + + - name: Cache cargo-rdme + if: matrix.rust == 'stable' && matrix.target == 'x86_64-unknown-linux-gnu' + uses: actions/cache@v4 + with: + path: ~/.cargo-${{ matrix.rust }}/cargo-rdme + key: cargo-rdme- + + # Check if the doc cumment in avro/src/lib.rs and avro/README.md are in sync. + - name: Run cargo-rdme + # The result is environment independent so one test pattern is enough. + if: matrix.rust == 'stable' && matrix.target == 'x86_64-unknown-linux-gnu' + run: | + cargo install --root ~/.cargo-${{ matrix.rust }}/cargo-rdme --locked cargo-rdme + export PATH=$PATH:~/.cargo-${{ matrix.rust }}/cargo-rdme/bin + cargo rdme --check + + - name: Rust Format + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo fmt --all -- --check + + - name: Rust Build + run: cargo build --all-features --all-targets + + - name: Rust Test + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo test --all-features --target ${{ matrix.target }} + + - name: Rust Test AVRO-3549 + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo test --target ${{ matrix.target }} test_avro_3549_read_not_enabled_codec + + # because of https://github.com/rust-lang/cargo/issues/6669 + - name: Rust Test docs + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo test --doc + + interop: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: stable + + - name: Cache Cargo + uses: actions/cache@v4 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: ${{ runner.os }}-target-cache1-${{ hashFiles('**/Cargo.lock') }} + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-stable-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache Local Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: 'Setup Temurin JDK 8, 11, 17 & 21' + uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4.2.2 + with: + distribution: 'temurin' + java-version: | + 8 + 11 + 17 + 21 + + - name: 'Setup Maven 3.9.6' + uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5 + with: + maven-version: 3.9.6 + + - name: Install Java Avro for Interop Test + working-directory: . + run: mvn -B install -PskipQuality + + - name: Create Interop Data Directory + working-directory: . + run: mkdir -p build/interop/data + + - name: Generate Interop Resources + working-directory: lang/java/avro + run: mvn -B -P interop-data-generate generate-resources + + - name: Generate interop data + run: ./build.sh interop-data-generate + + - name: Rust reads interop files created by Java and Rust + run: ./build.sh interop-data-test + + - uses: shogo82148/actions-setup-perl@v1 + with: + perl-version: 5.32 + + - name: Install Dependencies + run: | + sudo apt-get -qqy install --no-install-recommends libcompress-raw-zlib-perl \ + libcpan-uploader-perl \ + libencode-perl \ + libio-string-perl \ + libjansson-dev \ + libjson-xs-perl \ + libmodule-install-perl \ + libmodule-install-readmefrompod-perl \ + libobject-tiny-perl \ + libsnappy-dev \ + libtest-exception-perl \ + libtest-pod-perl + cpanm --mirror https://www.cpan.org/ install Compress::Zstd \ + Error::Simple \ + Module::Install::Repository \ + Object::Tiny \ + Regexp::Common \ + Try::Tiny \ + inc::Module::Install + + + - name: Perl reads interop files created by Java and Rust + working-directory: lang/perl + run: ./build.sh interop-data-test + + web-assembly: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: stable + targets: wasm32-unknown-unknown + + - name: Cache Cargo + uses: actions/cache@v4 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: ${{ runner.os }}-target-cache1-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-stable-${{ hashFiles('**/Cargo.lock') }} + + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + + - name: Build the Web Assembly demo app + run: wasm-pack build wasm-demo + + - name: Test the Web Assembly demo app + run: RUST_BACKTRACE=1 wasm-pack test --headless --firefox wasm-demo diff --git a/.github/workflows/test-lang-rust-clippy.yml b/.github/workflows/test-lang-rust-clippy.yml new file mode 100644 index 0000000..2b44ed4 --- /dev/null +++ b/.github/workflows/test-lang-rust-clippy.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rust Clippy Check +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + clippy_check: + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - 'stable' + - '1.73.0' # MSRV + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: ${{ matrix.rust }} + components: clippy + - run: cargo clippy --all-features --all-targets -- -Dclippy::all -Dunused_imports From b831b99f846eca40bf5731efd8f044f6c96ab66b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 23 Sep 2024 14:47:39 +0300 Subject: [PATCH 2/3] Git clone the main repo for the interop tests Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-rust-ci.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-lang-rust-ci.yml b/.github/workflows/test-lang-rust-ci.yml index 9bbd2ba..1dfd42e 100644 --- a/.github/workflows/test-lang-rust-ci.yml +++ b/.github/workflows/test-lang-rust-ci.yml @@ -116,6 +116,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Checkout other SDKs + uses: actions/checkout@v4 + with: + repository: apache/avro + path: main-repo + - name: Rust Toolchain uses: dtolnay/rust-toolchain@nightly with: @@ -154,25 +160,27 @@ jobs: 17 21 - - name: 'Setup Maven 3.9.6' + - name: 'Setup Maven 3.9.9' uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5 with: - maven-version: 3.9.6 + maven-version: 3.9.9 - name: Install Java Avro for Interop Test - working-directory: . + working-directory: main-repo run: mvn -B install -PskipQuality - name: Create Interop Data Directory - working-directory: . + working-directory: main-repo run: mkdir -p build/interop/data - name: Generate Interop Resources - working-directory: lang/java/avro + working-directory: main-repo/lang/java/avro run: mvn -B -P interop-data-generate generate-resources - name: Generate interop data - run: ./build.sh interop-data-generate + run: | + ln -s main-repo/build build + ./build.sh interop-data-generate - name: Rust reads interop files created by Java and Rust run: ./build.sh interop-data-test @@ -205,7 +213,7 @@ jobs: - name: Perl reads interop files created by Java and Rust - working-directory: lang/perl + working-directory: main-repo/lang/perl run: ./build.sh interop-data-test web-assembly: From 82e3acf64372dedd3237a1a25e35aed10995b343 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 23 Sep 2024 15:00:49 +0300 Subject: [PATCH 3/3] Read the INTEROP_ROOT_FOLDER from an env var Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-rust-ci.yml | 6 ++---- avro/examples/generate_interop_data.rs | 14 +++++++++----- avro/examples/test_interop_data.rs | 3 ++- .../test_interop_single_object_encoding.rs | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-lang-rust-ci.yml b/.github/workflows/test-lang-rust-ci.yml index 1dfd42e..13e6f34 100644 --- a/.github/workflows/test-lang-rust-ci.yml +++ b/.github/workflows/test-lang-rust-ci.yml @@ -178,12 +178,10 @@ jobs: run: mvn -B -P interop-data-generate generate-resources - name: Generate interop data - run: | - ln -s main-repo/build build - ./build.sh interop-data-generate + run: INTEROP_ROOT_FOLDER="main-repo" ./build.sh interop-data-generate - name: Rust reads interop files created by Java and Rust - run: ./build.sh interop-data-test + run: INTEROP_ROOT_FOLDER="main-repo" ./build.sh interop-data-test - uses: shogo82148/actions-setup-perl@v1 with: diff --git a/avro/examples/generate_interop_data.rs b/avro/examples/generate_interop_data.rs index 29d50a1..c13dc06 100644 --- a/avro/examples/generate_interop_data.rs +++ b/avro/examples/generate_interop_data.rs @@ -76,11 +76,15 @@ fn create_datum(schema: &Schema) -> Record { } fn main() -> Result<(), Box> { - let schema_str = std::fs::read_to_string("../../share/test/schemas/interop.avsc") - .expect("Unable to read the interop Avro schema"); + let interop_root_folder: String = std::env::var("INTEROP_ROOT_FOLDER")?; + let schema_str = std::fs::read_to_string(format!( + "{}/share/test/schemas/interop.avsc", + &interop_root_folder + )) + .expect("Unable to read the interop Avro schema"); let schema = Schema::parse_str(schema_str.as_str())?; - let data_folder = "../../build/interop/data"; - std::fs::create_dir_all(data_folder)?; + let data_folder = format!("{interop_root_folder}/build/interop/data"); + std::fs::create_dir_all(&data_folder)?; for codec in Codec::iter() { let codec_name = <&str>::from(codec); @@ -90,7 +94,7 @@ fn main() -> Result<(), Box> { format!("_{codec_name}") }; - let file_name = format!("{data_folder}/rust{suffix}.avro"); + let file_name = format!("{}/rust{suffix}.avro", &data_folder); let output_file = std::fs::File::create(&file_name)?; let mut writer = Writer::with_codec(&schema, BufWriter::new(output_file), codec); diff --git a/avro/examples/test_interop_data.rs b/avro/examples/test_interop_data.rs index 39c97d0..7833b4f 100644 --- a/avro/examples/test_interop_data.rs +++ b/avro/examples/test_interop_data.rs @@ -27,7 +27,8 @@ fn main() -> Result<(), Box> { let mut expected_user_metadata: HashMap> = HashMap::new(); expected_user_metadata.insert("user_metadata".to_string(), b"someByteArray".to_vec()); - let data_dir = std::fs::read_dir("../../build/interop/data/") + let interop_root_folder: String = std::env::var("INTEROP_ROOT_FOLDER")?; + let data_dir = std::fs::read_dir(format!("{interop_root_folder}/build/interop/data/")) .expect("Unable to list the interop data directory"); let mut errors = Vec::new(); diff --git a/avro/examples/test_interop_single_object_encoding.rs b/avro/examples/test_interop_single_object_encoding.rs index ef13465..61b1e6f 100644 --- a/avro/examples/test_interop_single_object_encoding.rs +++ b/avro/examples/test_interop_single_object_encoding.rs @@ -16,14 +16,17 @@ // under the License. use apache_avro::{schema::AvroSchema, types::Value}; - -const RESOURCES_FOLDER: &str = "../../share/test/data/messageV1"; +use std::error::Error; struct InteropMessage; impl AvroSchema for InteropMessage { fn get_schema() -> apache_avro::Schema { - let schema = std::fs::read_to_string(format!("{RESOURCES_FOLDER}/test_schema.avsc")) + let interop_root_folder: String = std::env::var("INTEROP_ROOT_FOLDER") + .expect("INTEROP_ROOT_FOLDER env var should be set"); + let resource_folder = format!("{interop_root_folder}/share/test/data/messageV1"); + + let schema = std::fs::read_to_string(format!("{resource_folder}/test_schema.avsc")) .expect("File should exist with schema inside"); apache_avro::Schema::parse_str(schema.as_str()) .expect("File should exist with schema inside") @@ -48,11 +51,16 @@ impl From for Value { } } -fn main() { - let single_object = std::fs::read(format!("{RESOURCES_FOLDER}/test_message.bin")) +fn main() -> Result<(), Box> { + let interop_root_folder: String = std::env::var("INTEROP_ROOT_FOLDER")?; + let resource_folder = format!("{interop_root_folder}/share/test/data/messageV1"); + + let single_object = std::fs::read(format!("{resource_folder}/test_message.bin")) .expect("File with single object not found or error occurred while reading it."); test_write(&single_object); test_read(single_object); + + Ok(()) } fn test_write(expected: &[u8]) {