diff --git a/Cargo.lock b/Cargo.lock index 3d321eacd..ddf028c23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,17 @@ version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -277,6 +288,84 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-extra" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" +dependencies = [ + "axum", + "axum-core", + "bytes", + "fastrand", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "multer", + "pin-project-lite", + "serde", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -2019,6 +2108,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.9.5" @@ -2557,6 +2652,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -2783,6 +2884,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2821,6 +2932,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.2.0", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "native-tls" version = "0.2.12" @@ -3057,6 +3185,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ota-dev-server" +version = "0.3.3" +dependencies = [ + "axum", + "axum-extra", + "local-ip-address", + "tokio", + "tower 0.4.13", + "tower-http", + "tracing", + "tracing-subscriber", +] + [[package]] name = "overload" version = "0.1.1" @@ -3860,6 +4002,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_plain" version = "1.0.2" @@ -4521,6 +4673,68 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4533,10 +4747,23 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "tracing-core" version = "0.1.33" @@ -4569,6 +4796,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -4644,6 +4872,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "unicase" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/Cargo.toml b/Cargo.toml index 90f82adff..0f140a9dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "micro-rdk-server", "micro-rdk-ffi", "examples/modular-drivers", + "etc/ota-dev-server", ] default-members = [ @@ -118,7 +119,11 @@ syn = "2.0.90" tempfile = "3.14.0" test-log = "0.2.16" thiserror = "2.0.4" -tokio = { version = "1.42.0", default-features = false } +tokio = { version = "1.42.0", features = ["full"] } +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.5.0", features = ["fs", "trace"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } trackable = "1.3.0" uuid = "1.11.0" version-compare = "0.2" diff --git a/Makefile b/Makefile index c84efc0d5..f30870a52 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,10 @@ upload: cargo-ver cargo +esp espflash flash --package micro-rdk-server --monitor --partition-table micro-rdk-server/esp32/partitions.csv --baud 460800 -f 80mhz --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort test: - cargo test -p micro-rdk --lib --features native + cargo test -p micro-rdk --lib --features native,ota clippy-native: - cargo clippy -p micro-rdk --no-deps --features native -- -Dwarnings + cargo clippy -p micro-rdk --no-deps --features native,ota -- -Dwarnings clippy-esp32: cargo +esp clippy -p micro-rdk --features esp32,ota --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort -- -Dwarnings @@ -99,11 +99,34 @@ doc-open: size: find . -name "esp-build.map" -exec ${IDF_PATH}/tools/idf_size.py {} \; -build-esp32-bin: - cargo +esp espflash save-image --package micro-rdk-server --merge --chip esp32 target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin -T micro-rdk-server/esp32/partitions.csv -s 4mb --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release +build-esp32-bin: build-esp32-ota + cargo +esp espflash save-image \ + --skip-update-check \ + --package=micro-rdk-server \ + --features=ota \ + --chip=esp32 \ + --bin=micro-rdk-server-esp32 \ + --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ + --target=xtensa-esp32-espidf \ + -Zbuild-std=std,panic_abort --release \ + --flash-size=8mb \ + --merge \ + target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin build-esp32-ota: - cargo +esp espflash save-image --package micro-rdk-server --features=ota --chip=esp32 ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin --bin=micro-rdk-server-esp32 --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release + cargo +esp espflash save-image \ + --skip-update-check \ + --package=micro-rdk-server \ + --features=ota \ + --chip=esp32 \ + --bin=micro-rdk-server-esp32 \ + --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ + --target=xtensa-esp32-espidf \ + -Zbuild-std=std,panic_abort --release \ + target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin + +serve-ota: build-esp32-ota + cargo r --package ota-dev-server flash-esp32-bin: ifneq (,$(wildcard ./target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin)) diff --git a/OTA.md b/OTA.md new file mode 100644 index 000000000..8be7648e0 --- /dev/null +++ b/OTA.md @@ -0,0 +1,118 @@ +# Over-The-Air (OTA) Updates + + +**OTA is in active development. Breaking changes should be expected. Check this document frequently for updates.**** + + +## Workflow + +In [app.viam.com](app.viam.com), add the following to the `services` array; you can alternatively add a `generic` service then edit it to match the following + +### OTA Service Config + +```json +{ + "name": "OTA", + "namespace": "rdk", + "type": "generic", + "model": "ota_service", + "attributes": { + "url": , + "version": + } +} +``` + + +In the `url` field, enter the url where a firmware will be downloaded from + - if using a local endpoint on the same network, remember to use the proper private ip address + - for cloud-hosted resources, embedded auth in the url is easiest, we don't support tokens yet + + +The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. +After successfully applying the new firmware, this `version` will be stored in NVS. +This value is compared to that of the latest machine config from app.viam.com and will trigger the update process. + + +## Requirements + +- an esp32 WROVER-E with 8MB or more of flash memory +- a partition table (ex `partitions.csv`) with `otadata`, `ota_0`, and `ota_1` partitions + + +## Primer + +Consider firmware built with the following partition table: + +``` +# ESP-IDF Partition Table + +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, 0x9000, 0x6000, +otadata, data, ota, 0xF000, 0x2000, +phy_init, data, phy, 0x11000, 0x1000, +ota_0, app, ota_0, , 0x377000, +ota_1, app, ota_1, , 0x377000, +``` + +The `otadata` partition contains information on which OTA partition to boot from and the states of the `ota_*` partitions. + +The terms 'firmware' or 'binary' can be a bit generic. +In this section, we will refer to two types of binaries that can be built. +1. a Merged Image +2. an [Application (App) Image](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/app_image_format.html) + +> Note: in the absence of a `factory` partition `ota_0` fills the same initial role. + +## Full Build + +If a device is built with the above partition table, the `make build-esp32-bin` command creates a Merged Image that includes +- the bootloader +- the partition table mapping (`partitions.csv`) +- populated partitions (with partition headers) according to the mapping + +The command `make flash-esp32-bin` writes this entire Merged Image to the device's flash memory. + +This is the build workflow which must be used to: +- flash a new device for the first time +- update a device's partition table + - for example, make a device capable of OTA. + +**This is not the build that should be hosted at the `url` in the service config.** +You can confirm this by using `ls -l` in your build directory to compare the size of the binary to your partition table; the Merged Image will about the size of the full partition table, `8MB` in this example. + +## OTA Build + +The `make build-esp32-ota` command produces an App Image (described above), which internally consists of: +- the type-specific partition header, `esp_app_desc_t** +- the application image that contains the program instructions + +**This app image is what you must host, see [Firmware Hosting Options](#firmware-hosting-options).** + +This build must be within the size limits of the smallest `ota_*` partition in a device's *current* partition table. +This document assumes the user is using our included partition tables; should the final image be larger than the capacity of the ota partitions, the build will fail indicating so. + +To update a device's partition table, use the method in the [Full Build](#full-build) workflow. + + +## Firmware Hosting Options +### Local + - use `make serve-dev-ota` to create a local endpoint for serving the ota app image + - the command will build the ota firmware first before serving the url + +### Cloud + +The OTA Service in the micro-rdk currently supports **only HTTP/2**, this means that the hosting platform must support HTTP/2 connections. + +While not all blob storage platform support HTTP/2, many offer Content Delivery Network (CDN) solutions that do. + +We don't currently support authentication tokens in the [OTA Service Config](#ota-service-config), so if permissions are required to access the endpoint they must be embedded in the URL as query params. + +## Related Links + +> Links may point to latest branches of documentation to reduce chances of dead links; reference the appropriate version if available. + +- [Over The Air Updates](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/ota.html) - Espressif +- [Partition Tables](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/partition-tables.html) - Espressif +- [App Image Format](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/app_image_format.html) - Espressif diff --git a/etc/ota-dev-server/Cargo.toml b/etc/ota-dev-server/Cargo.toml new file mode 100644 index 000000000..93504b95a --- /dev/null +++ b/etc/ota-dev-server/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ota-dev-server" +authors.workspace = true +description.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true +rust-version.workspace = true + +[dependencies] +axum = {version = "0.7.9", features = ["http2"]} +axum-extra = "0.9.6" +local-ip-address.workspace = true +tokio.workspace = true +tower.workspace = true +tower-http.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/etc/ota-dev-server/src/main.rs b/etc/ota-dev-server/src/main.rs new file mode 100644 index 000000000..0c1bb4a9c --- /dev/null +++ b/etc/ota-dev-server/src/main.rs @@ -0,0 +1,37 @@ +use axum::Router; +use local_ip_address::local_ip; +use std::net::SocketAddr; +use tower_http::{services::ServeDir, trace::TraceLayer}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +const MICRO_RDK_OTA_BIN: &str = "micro-rdk-server-esp32-ota.bin"; +const TARGET_DIR: &str = "target/xtensa-esp32-espidf"; + +#[tokio::main] +async fn main() { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + format!("{}=debug,tower_http=debug", env!("CARGO_CRATE_NAME")).into() + }), + ) + .with(tracing_subscriber::fmt::layer()) + .init(); + + tokio::join!(serve(using_serve_dir(), 3001),); +} + +fn using_serve_dir() -> Router { + Router::new().nest_service("/", ServeDir::new(TARGET_DIR)) +} + +async fn serve(app: Router, port: u16) { + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); + // get private address + let local = local_ip().unwrap(); + tracing::info!("serving ota partition: \n\n\thttp://{local}:{port}/{MICRO_RDK_OTA_BIN}"); + axum::serve(listener, app.layer(TraceLayer::new_for_http())) + .await + .unwrap(); +}