From c455f0a1f6631f764e0d88a09033caa670207042 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Mon, 9 Dec 2024 17:37:22 -0500 Subject: [PATCH 01/14] first draft --- OTA.md | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 OTA.md diff --git a/OTA.md b/OTA.md new file mode 100644 index 00000000..a4fcd468 --- /dev/null +++ b/OTA.md @@ -0,0 +1,88 @@ +# Over-The-Air (OTA) Updates + + +> OTA is experimental and in active development. Breaking changes should be expected often. Check this document frequently for updates + + +## Workflow + +In app.viam, add the following to the `services` array; you can alternatively add a `generic` service then edit it to match the following + +```json + { + "name": "OTA", + "namespace": "rdk", + "type": "generic", + "model": "ota_service", + "attributes": { + "url": "", + "version": "" + } + } +``` + + +In the `url` field, enter the url where a firmware will be downloaded from + - if using a local endpoint on the same network, remember to use the proper private ip address + - for cloud-hosted resources, embedded auth in the url is easiest, we don't support jwt yet + + +The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. After successfully applying the new firmware, this `version` will be stored in NVS. This values is compared to that of the latest machine config from app.viam and will trigger the update process. + + +## Requirements + +- An esp32 wrover-e with 8MB of flash memory. + +## Build Process + +## Primer + +``` +# ESP-IDF Partition Table +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x4000, +otadata, data, ota, 0xd000, 0x2000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 1M, +ota_0, app, ota_0, 0x110000, 1M, +ota_1, app, ota_1, 0x210000, 1M, + +``` + +The terms 'firmware' or 'binary' can be a bit generic. +In this section, we will refer to two types of binaries that can be built. +1. a Merged Binary +2. an App Image + +## Full Build + +If a device is built with the above partition table, the `make build-esp32-bin` command creates a Merged Binary that includes +- the bootloader +- the partition table mapping (`partitions.csv`) +- populated partitions (with partition headers) according to the mapping + +The command `make flash-esp32-bin` writes this entire merged binary to the device's flash memory. + +This is the build workflow which must be used if you want to update a device's partition table; for example, to make a device capable of OTA. + +**This is not the build that should be hosted at the `url` in the service config.** +You can confirm this by using `ls -l` in your build directory to compare the size of the binary to your partition table. + +### OTA Build +The `ota` build consists of *only*: +- the type-specific partition header, `esp_app_desc_t` +- the application image that contains the program instructions + +This build must be within the size limits for the `ota0` and `ota1` partitions specified by a device's *current* partition table. + +To update a device's partition table, use the method in the Full Build workflow. + + +## Firmware Upload and Hosting Options +### Local +- use `make serve-ota` to point to create a local endpoint serving the ota build. Ensure the link includes the host's private address in the url. +### Cloud +- if using a cloud hosting solution, generate a `url` for downloading your firmware that includes auth embedded in the url if possible. + +## Internals From dbcd06d1b45e06577af60cfa030217254d4c8140 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Mon, 9 Dec 2024 17:38:57 -0500 Subject: [PATCH 02/14] update build commands to use ota --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c84efc0d..c303fb53 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,10 @@ upload: cargo-ver cargo +esp espflash flash --package micro-rdk-server --monitor --partition-table micro-rdk-server/esp32/partitions.csv --baud 460800 -f 80mhz --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort test: - cargo test -p micro-rdk --lib --features native + cargo test -p micro-rdk --lib --features native,ota clippy-native: - cargo clippy -p micro-rdk --no-deps --features native -- -Dwarnings + cargo clippy -p micro-rdk --no-deps --features native,ota -- -Dwarnings clippy-esp32: cargo +esp clippy -p micro-rdk --features esp32,ota --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort -- -Dwarnings @@ -99,8 +99,8 @@ doc-open: size: find . -name "esp-build.map" -exec ${IDF_PATH}/tools/idf_size.py {} \; -build-esp32-bin: - cargo +esp espflash save-image --package micro-rdk-server --merge --chip esp32 target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin -T micro-rdk-server/esp32/partitions.csv -s 4mb --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release +build-esp32-bin: build-esp32-ota + cargo +esp espflash save-image --package micro-rdk-server --features=ota --merge --chip esp32 target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin -T micro-rdk-server/esp32/ota_8mb_partitions.csv -s 8mb --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release build-esp32-ota: cargo +esp espflash save-image --package micro-rdk-server --features=ota --chip=esp32 ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin --bin=micro-rdk-server-esp32 --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release From 10ab511f2203cd050b63efde4103b9ea1c17b153 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Tue, 10 Dec 2024 15:39:40 -0500 Subject: [PATCH 03/14] add local ota server for testing --- Cargo.toml | 1 + Makefile | 3 +++ OTA.md | 8 +++++--- etc/ota-server/Cargo.toml | 19 +++++++++++++++++++ etc/ota-server/src/main.rs | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 etc/ota-server/Cargo.toml create mode 100644 etc/ota-server/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 90f82adf..99f70e7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "micro-rdk-server", "micro-rdk-ffi", "examples/modular-drivers", + "etc/ota-server", ] default-members = [ diff --git a/Makefile b/Makefile index c303fb53..c2740e8f 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,9 @@ build-esp32-bin: build-esp32-ota build-esp32-ota: cargo +esp espflash save-image --package micro-rdk-server --features=ota --chip=esp32 ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin --bin=micro-rdk-server-esp32 --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release +serve-ota: build-esp32-ota + cargo r --package ota-server + flash-esp32-bin: ifneq (,$(wildcard ./target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin)) espflash write-bin 0x0 ./target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin --baud 460800 && sleep 2 && espflash monitor diff --git a/OTA.md b/OTA.md index a4fcd468..ff27821c 100644 --- a/OTA.md +++ b/OTA.md @@ -24,7 +24,7 @@ In app.viam, add the following to the `services` array; you can alternatively ad In the `url` field, enter the url where a firmware will be downloaded from - if using a local endpoint on the same network, remember to use the proper private ip address - - for cloud-hosted resources, embedded auth in the url is easiest, we don't support jwt yet + - for cloud-hosted resources, embedded auth in the url is easiest, we don't support tokens yet The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. After successfully applying the new firmware, this `version` will be stored in NVS. This values is compared to that of the latest machine config from app.viam and will trigger the update process. @@ -80,8 +80,10 @@ To update a device's partition table, use the method in the Full Build workflow. ## Firmware Upload and Hosting Options -### Local -- use `make serve-ota` to point to create a local endpoint serving the ota build. Ensure the link includes the host's private address in the url. +- Local + - Use `make serve-ota` to point to create a local endpoint serving the ota build. + - the command will build the ota firmware first before serving the url + ### Cloud - if using a cloud hosting solution, generate a `url` for downloading your firmware that includes auth embedded in the url if possible. diff --git a/etc/ota-server/Cargo.toml b/etc/ota-server/Cargo.toml new file mode 100644 index 00000000..00e8214e --- /dev/null +++ b/etc/ota-server/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ota-server" +authors.workspace = true +description.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true +rust-version.workspace = true + +[dependencies] +axum = {version = "0.7.9", features = ["http2"]} +axum-extra = "0.9.6" +local-ip-address.workspace = true +tokio = { version = "1.0", features = ["full"] } +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.5.0", features = ["fs", "trace"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/etc/ota-server/src/main.rs b/etc/ota-server/src/main.rs new file mode 100644 index 00000000..081883cd --- /dev/null +++ b/etc/ota-server/src/main.rs @@ -0,0 +1,35 @@ +use axum::Router; +use local_ip_address::local_ip; +use std::net::SocketAddr; +use tower_http::{services::ServeDir, trace::TraceLayer}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +#[tokio::main] +async fn main() { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + format!("{}=debug,tower_http=debug", env!("CARGO_CRATE_NAME")).into() + }), + ) + .with(tracing_subscriber::fmt::layer()) + .init(); + + tokio::join!(serve(using_serve_dir(), 3001),); +} + +fn using_serve_dir() -> Router { + // serve the file in the "assets" directory under `/assets` + Router::new().nest_service("/", ServeDir::new("../../target/xtensa-esp32-espidf")) +} + +async fn serve(app: Router, port: u16) { + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); + // get private address + let local = local_ip().unwrap(); + tracing::info!("serving ota partition at `http://{local}:{port}/micro-rdk-server-esp32-ota.bin`"); + axum::serve(listener, app.layer(TraceLayer::new_for_http())) + .await + .unwrap(); +} From af3e08b36acef4c8b65162363e360cf3830df964 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Tue, 10 Dec 2024 16:26:21 -0500 Subject: [PATCH 04/14] small changes --- etc/ota-server/src/main.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/etc/ota-server/src/main.rs b/etc/ota-server/src/main.rs index 081883cd..1d6efb1e 100644 --- a/etc/ota-server/src/main.rs +++ b/etc/ota-server/src/main.rs @@ -1,9 +1,12 @@ use axum::Router; use local_ip_address::local_ip; use std::net::SocketAddr; -use tower_http::{services::ServeDir, trace::TraceLayer}; +use tower_http::{services::ServeFile, trace::TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +const MICRO_RDK_OTA_BIN: &str = "micro-rdk-server-esp32-ota.bin"; +const TARGET_DIR: &str = "../../target/xtensa-esp32-espidf"; + #[tokio::main] async fn main() { tracing_subscriber::registry() @@ -19,8 +22,10 @@ async fn main() { } fn using_serve_dir() -> Router { - // serve the file in the "assets" directory under `/assets` - Router::new().nest_service("/", ServeDir::new("../../target/xtensa-esp32-espidf")) + Router::new().nest_service( + "/", + ServeFile::new(format!("{TARGET_DIR}/{MICRO_RDK_OTA_BIN}")), + ) } async fn serve(app: Router, port: u16) { @@ -28,7 +33,7 @@ async fn serve(app: Router, port: u16) { let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); // get private address let local = local_ip().unwrap(); - tracing::info!("serving ota partition at `http://{local}:{port}/micro-rdk-server-esp32-ota.bin`"); + tracing::info!("serving ota partition: \n\n\thttp://{local}:{port}/{MICRO_RDK_OTA_BIN}"); axum::serve(listener, app.layer(TraceLayer::new_for_http())) .await .unwrap(); From a6bdcc20897bdab1c70e751bb07e1258dceb6318 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Tue, 10 Dec 2024 16:32:02 -0500 Subject: [PATCH 05/14] undo changes --- etc/ota-server/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ota-server/src/main.rs b/etc/ota-server/src/main.rs index 1d6efb1e..52bb5509 100644 --- a/etc/ota-server/src/main.rs +++ b/etc/ota-server/src/main.rs @@ -1,7 +1,7 @@ use axum::Router; use local_ip_address::local_ip; use std::net::SocketAddr; -use tower_http::{services::ServeFile, trace::TraceLayer}; +use tower_http::{services::ServeDir, trace::TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; const MICRO_RDK_OTA_BIN: &str = "micro-rdk-server-esp32-ota.bin"; @@ -24,7 +24,7 @@ async fn main() { fn using_serve_dir() -> Router { Router::new().nest_service( "/", - ServeFile::new(format!("{TARGET_DIR}/{MICRO_RDK_OTA_BIN}")), + ServeDir::new(TARGET_DIR), ) } From 040524a786fa46b5dc0469e40e5d3fe732dfebf8 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Wed, 11 Dec 2024 11:42:36 -0500 Subject: [PATCH 06/14] fix targetdir when serving from workspace root --- etc/ota-server/src/main.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/etc/ota-server/src/main.rs b/etc/ota-server/src/main.rs index 52bb5509..0c1bb4a9 100644 --- a/etc/ota-server/src/main.rs +++ b/etc/ota-server/src/main.rs @@ -5,7 +5,7 @@ use tower_http::{services::ServeDir, trace::TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; const MICRO_RDK_OTA_BIN: &str = "micro-rdk-server-esp32-ota.bin"; -const TARGET_DIR: &str = "../../target/xtensa-esp32-espidf"; +const TARGET_DIR: &str = "target/xtensa-esp32-espidf"; #[tokio::main] async fn main() { @@ -22,10 +22,7 @@ async fn main() { } fn using_serve_dir() -> Router { - Router::new().nest_service( - "/", - ServeDir::new(TARGET_DIR), - ) + Router::new().nest_service("/", ServeDir::new(TARGET_DIR)) } async fn serve(app: Router, port: u16) { From e3e06d316acc5f645879a02e7803dfbdbd86373c Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Wed, 11 Dec 2024 11:44:50 -0500 Subject: [PATCH 07/14] update cargo lock --- Cargo.lock | 234 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3d321eac..21708201 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,17 @@ version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -277,6 +288,84 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-extra" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" +dependencies = [ + "axum", + "axum-core", + "bytes", + "fastrand", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "multer", + "pin-project-lite", + "serde", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -2019,6 +2108,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.9.5" @@ -2557,6 +2652,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -2783,6 +2884,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2821,6 +2932,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.2.0", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "native-tls" version = "0.2.12" @@ -3057,6 +3185,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ota-server" +version = "0.3.3" +dependencies = [ + "axum", + "axum-extra", + "local-ip-address", + "tokio", + "tower 0.4.13", + "tower-http", + "tracing", + "tracing-subscriber", +] + [[package]] name = "overload" version = "0.1.1" @@ -3860,6 +4002,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_plain" version = "1.0.2" @@ -4521,6 +4673,68 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4533,10 +4747,23 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "tracing-core" version = "0.1.33" @@ -4569,6 +4796,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -4644,6 +4872,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "unicase" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" + [[package]] name = "unicode-ident" version = "1.0.14" From 2a02ececc0ad6f07d852f99e3121989db7502ac3 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Wed, 18 Dec 2024 17:09:10 -0500 Subject: [PATCH 08/14] add doc suggestions, TODO update ota-server name,deps --- Makefile | 24 ++++++++++++++-- OTA.md | 86 +++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index c2740e8f..7d4d420e 100644 --- a/Makefile +++ b/Makefile @@ -100,10 +100,30 @@ size: find . -name "esp-build.map" -exec ${IDF_PATH}/tools/idf_size.py {} \; build-esp32-bin: build-esp32-ota - cargo +esp espflash save-image --package micro-rdk-server --features=ota --merge --chip esp32 target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin -T micro-rdk-server/esp32/ota_8mb_partitions.csv -s 8mb --bin micro-rdk-server-esp32 --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release + cargo +esp espflash save-image \ + --skip-update-check \ + --package=micro-rdk-server \ + --features=ota \ + --chip=esp32 \ + --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ + --flash-size=8mb \ + --bin=micro-rdk-server-esp32 \ + --target=xtensa-esp32-espidf \ + -Zbuild-std=std,panic_abort --release \ + target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin \ + --merge build-esp32-ota: - cargo +esp espflash save-image --package micro-rdk-server --features=ota --chip=esp32 ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin --bin=micro-rdk-server-esp32 --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv --target=xtensa-esp32-espidf -Zbuild-std=std,panic_abort --release + cargo +esp espflash save-image \ + --skip-update-check \ + --package=micro-rdk-server \ + --features=ota \ + --chip=esp32 \ + --bin=micro-rdk-server-esp32 \ + --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ + --target=xtensa-esp32-espidf \ + -Zbuild-std=std,panic_abort --release \ + ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin serve-ota: build-esp32-ota cargo r --package ota-server diff --git a/OTA.md b/OTA.md index ff27821c..54e44c61 100644 --- a/OTA.md +++ b/OTA.md @@ -1,23 +1,25 @@ # Over-The-Air (OTA) Updates -> OTA is experimental and in active development. Breaking changes should be expected often. Check this document frequently for updates +**OTA is in active development. Breaking changes should be expected. Check this document frequently for updates.**** ## Workflow -In app.viam, add the following to the `services` array; you can alternatively add a `generic` service then edit it to match the following +In [app.viam.com](app.viam.com), add the following to the `services` array; you can alternatively add a `generic` service then edit it to match the following + +### OTA Service Config ```json - { - "name": "OTA", + { + "name": "OTA", "namespace": "rdk", "type": "generic", "model": "ota_service", "attributes": { - "url": "", - "version": "" - } + "url": , + "version": + } } ``` @@ -27,33 +29,41 @@ In the `url` field, enter the url where a firmware will be downloaded from - for cloud-hosted resources, embedded auth in the url is easiest, we don't support tokens yet -The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. After successfully applying the new firmware, this `version` will be stored in NVS. This values is compared to that of the latest machine config from app.viam and will trigger the update process. +The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. +After successfully applying the new firmware, this `version` will be stored in NVS. +This values is compared to that of the latest machine config from app.viam.com and will trigger the update process. ## Requirements -- An esp32 wrover-e with 8MB of flash memory. +- an esp32 WROVER-E with 8MB or more of flash memory +- a partition table (ex `partitions.csv`) with `otadata`, `ota_0`, and `ota_1` partitions -## Build Process ## Primer +Consider firmware built with the following partition table: + ``` # ESP-IDF Partition Table -# Name, Type, SubType, Offset, Size, Flags -nvs, data, nvs, 0x9000, 0x4000, -otadata, data, ota, 0xd000, 0x2000, -phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 1M, -ota_0, app, ota_0, 0x110000, 1M, -ota_1, app, ota_1, 0x210000, 1M, +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, 0x9000, 0x6000, +otadata, data, ota, 0xF000, 0x2000, +phy_init, data, phy, 0x11000, 0x1000, +ota_0, app, ota_0, , 0x377000, +ota_1, app, ota_1, , 0x377000, ``` +The `otadata` partition contains information on which OTA partition to boot from and the states of the `ota_*` partitions. + The terms 'firmware' or 'binary' can be a bit generic. In this section, we will refer to two types of binaries that can be built. -1. a Merged Binary -2. an App Image +1. a Merged Image +2. an [Application (App) Image](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/app_image_format.html) + +> Note: in the absence of a `factory` partition `ota_0` fills the same initial role. ## Full Build @@ -61,30 +71,42 @@ If a device is built with the above partition table, the `make build-esp32-bin` - the bootloader - the partition table mapping (`partitions.csv`) - populated partitions (with partition headers) according to the mapping - +/ The command `make flash-esp32-bin` writes this entire merged binary to the device's flash memory. -This is the build workflow which must be used if you want to update a device's partition table; for example, to make a device capable of OTA. +This is the build workflow which must be used to: +- flash a new device for the first time +- update a device's partition table + - for example, make a device capable of OTA. **This is not the build that should be hosted at the `url` in the service config.** -You can confirm this by using `ls -l` in your build directory to compare the size of the binary to your partition table. +You can confirm this by using `ls -l** in your build directory to compare the size of the binary to your partition table. -### OTA Build -The `ota` build consists of *only*: -- the type-specific partition header, `esp_app_desc_t` +## OTA Build +The `ota` build produces an app image (described above), which internally consists of: +- the type-specific partition header, `esp_app_desc_t** - the application image that contains the program instructions -This build must be within the size limits for the `ota0` and `ota1` partitions specified by a device's *current* partition table. +**This app image is what you must host, see [Firmware Hosting Options](#firmware-hosting-options).** + +This build must be within the size limits of the smallest `ota_*` partition in a device's *current* partition table. + +In this document's example, both the `ota_0` and `ota_1` partitions are ~3.4 MB. +The command underlying `make build-esp32-ota` takes the partition table as input so it **should** fail, additionally, target devices must be programmed to verify at runtime that the assertion holds. -To update a device's partition table, use the method in the Full Build workflow. +To update a device's partition table, use the method in the [Full Build](#full-build) workflow. -## Firmware Upload and Hosting Options -- Local - - Use `make serve-ota` to point to create a local endpoint serving the ota build. +## Firmware Hosting Options +### Local + - use `make serve-dev-ota` to create a local endpoint for serving the ota app image - the command will build the ota firmware first before serving the url ### Cloud -- if using a cloud hosting solution, generate a `url` for downloading your firmware that includes auth embedded in the url if possible. -## Internals +The OTA Service in the micro-rdk currently supports **only HTTP/2**, this means that the hosting platform must support HTTP/2 connections. + +While not all blob storage platform support HTTP/2, many offer Content Delivery Network (CDN) solutions that do. + +We don't currently support tokens in the [OTA Service Config](#ota-service-config), so if permissions are required to access the endpoint they must be embedded in the URL itself. + From 2d2bee125b646abb3805683c3134e47d02a4136d Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Thu, 19 Dec 2024 10:17:16 -0500 Subject: [PATCH 09/14] more doc suggestions --- Makefile | 7 +++---- OTA.md | 13 ++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 7d4d420e..c6f6a42f 100644 --- a/Makefile +++ b/Makefile @@ -105,13 +105,12 @@ build-esp32-bin: build-esp32-ota --package=micro-rdk-server \ --features=ota \ --chip=esp32 \ - --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ - --flash-size=8mb \ --bin=micro-rdk-server-esp32 \ + --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ --target=xtensa-esp32-espidf \ -Zbuild-std=std,panic_abort --release \ - target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin \ --merge + target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin build-esp32-ota: cargo +esp espflash save-image \ @@ -123,7 +122,7 @@ build-esp32-ota: --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ --target=xtensa-esp32-espidf \ -Zbuild-std=std,panic_abort --release \ - ./target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin + target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin serve-ota: build-esp32-ota cargo r --package ota-server diff --git a/OTA.md b/OTA.md index 54e44c61..d6190855 100644 --- a/OTA.md +++ b/OTA.md @@ -67,12 +67,12 @@ In this section, we will refer to two types of binaries that can be built. ## Full Build -If a device is built with the above partition table, the `make build-esp32-bin` command creates a Merged Binary that includes +If a device is built with the above partition table, the `make build-esp32-bin` command creates a Merged Image that includes - the bootloader - the partition table mapping (`partitions.csv`) - populated partitions (with partition headers) according to the mapping -/ -The command `make flash-esp32-bin` writes this entire merged binary to the device's flash memory. + +The command `make flash-esp32-bin` writes this entire Merged Image to the device's flash memory. This is the build workflow which must be used to: - flash a new device for the first time @@ -83,16 +83,15 @@ This is the build workflow which must be used to: You can confirm this by using `ls -l** in your build directory to compare the size of the binary to your partition table. ## OTA Build -The `ota` build produces an app image (described above), which internally consists of: + +The `make build-esp32-ota` command produces an App Image (described above), which internally consists of: - the type-specific partition header, `esp_app_desc_t** - the application image that contains the program instructions **This app image is what you must host, see [Firmware Hosting Options](#firmware-hosting-options).** This build must be within the size limits of the smallest `ota_*` partition in a device's *current* partition table. - -In this document's example, both the `ota_0` and `ota_1` partitions are ~3.4 MB. -The command underlying `make build-esp32-ota` takes the partition table as input so it **should** fail, additionally, target devices must be programmed to verify at runtime that the assertion holds. +This document assumes the user is using our included partition tables; should the final image be larger than the capacity of the ota partitions, the build will fail indicating so. To update a device's partition table, use the method in the [Full Build](#full-build) workflow. From 2b8a68d32ea8d594179a7c4ff1995ce91200049a Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Thu, 19 Dec 2024 10:19:43 -0500 Subject: [PATCH 10/14] service json formatting --- OTA.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OTA.md b/OTA.md index d6190855..2914cc62 100644 --- a/OTA.md +++ b/OTA.md @@ -13,12 +13,12 @@ In [app.viam.com](app.viam.com), add the following to the `services` array; you ```json { "name": "OTA", - "namespace": "rdk", - "type": "generic", - "model": "ota_service", - "attributes": { - "url": , - "version": + "namespace": "rdk", + "type": "generic", + "model": "ota_service", + "attributes": { + "url": , + "version": } } ``` From 5baea093b139a8b799670317cc3ab80dba2e1dc4 Mon Sep 17 00:00:00 2001 From: Matthew J Perez <41358385+mattjperez@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:23:53 -0500 Subject: [PATCH 11/14] fmt using github editor --- OTA.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/OTA.md b/OTA.md index 2914cc62..cff23e5b 100644 --- a/OTA.md +++ b/OTA.md @@ -11,16 +11,16 @@ In [app.viam.com](app.viam.com), add the following to the `services` array; you ### OTA Service Config ```json - { - "name": "OTA", - "namespace": "rdk", - "type": "generic", - "model": "ota_service", - "attributes": { - "url": , - "version": - } - } +{ + "name": "OTA", + "namespace": "rdk", + "type": "generic", + "model": "ota_service", + "attributes": { + "url": , + "version": + } +} ``` From b7840b753a49a54c41fcd41f7455dbc1d3b29e54 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Thu, 19 Dec 2024 10:38:12 -0500 Subject: [PATCH 12/14] update ota-server to ota-dev-server --- Cargo.lock | 2 +- Cargo.toml | 8 ++++++-- Makefile | 2 +- etc/{ota-server => ota-dev-server}/Cargo.toml | 12 ++++++------ etc/{ota-server => ota-dev-server}/src/main.rs | 0 5 files changed, 14 insertions(+), 10 deletions(-) rename etc/{ota-server => ota-dev-server}/Cargo.toml (54%) rename etc/{ota-server => ota-dev-server}/src/main.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 21708201..ddf028c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3186,7 +3186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] -name = "ota-server" +name = "ota-dev-server" version = "0.3.3" dependencies = [ "axum", diff --git a/Cargo.toml b/Cargo.toml index 99f70e7b..0f140a9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ "micro-rdk-server", "micro-rdk-ffi", "examples/modular-drivers", - "etc/ota-server", + "etc/ota-dev-server", ] default-members = [ @@ -119,7 +119,11 @@ syn = "2.0.90" tempfile = "3.14.0" test-log = "0.2.16" thiserror = "2.0.4" -tokio = { version = "1.42.0", default-features = false } +tokio = { version = "1.42.0", features = ["full"] } +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.5.0", features = ["fs", "trace"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } trackable = "1.3.0" uuid = "1.11.0" version-compare = "0.2" diff --git a/Makefile b/Makefile index c6f6a42f..94c4d16c 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ build-esp32-ota: target/xtensa-esp32-espidf/micro-rdk-server-esp32-ota.bin serve-ota: build-esp32-ota - cargo r --package ota-server + cargo r --package ota-dev-server flash-esp32-bin: ifneq (,$(wildcard ./target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin)) diff --git a/etc/ota-server/Cargo.toml b/etc/ota-dev-server/Cargo.toml similarity index 54% rename from etc/ota-server/Cargo.toml rename to etc/ota-dev-server/Cargo.toml index 00e8214e..93504b95 100644 --- a/etc/ota-server/Cargo.toml +++ b/etc/ota-dev-server/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ota-server" +name = "ota-dev-server" authors.workspace = true description.workspace = true edition.workspace = true @@ -12,8 +12,8 @@ rust-version.workspace = true axum = {version = "0.7.9", features = ["http2"]} axum-extra = "0.9.6" local-ip-address.workspace = true -tokio = { version = "1.0", features = ["full"] } -tower = { version = "0.4", features = ["util"] } -tower-http = { version = "0.5.0", features = ["fs", "trace"] } -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tokio.workspace = true +tower.workspace = true +tower-http.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/etc/ota-server/src/main.rs b/etc/ota-dev-server/src/main.rs similarity index 100% rename from etc/ota-server/src/main.rs rename to etc/ota-dev-server/src/main.rs From 18b1bb198d57ae4820ce210a7f2b798357015d85 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Thu, 19 Dec 2024 10:49:58 -0500 Subject: [PATCH 13/14] additional links --- OTA.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/OTA.md b/OTA.md index cff23e5b..ab60270d 100644 --- a/OTA.md +++ b/OTA.md @@ -109,3 +109,10 @@ While not all blob storage platform support HTTP/2, many offer Content Delivery We don't currently support tokens in the [OTA Service Config](#ota-service-config), so if permissions are required to access the endpoint they must be embedded in the URL itself. +## Related Links + +> Links may point to latest branches of documentation to reduce chances of dead links; reference the appropriate version if available. + +- [Over The Air Updates](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/ota.html) - Espressif +- [Partition Tables](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/partition-tables.html) - Espressif +- [App Image Format](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/app_image_format.html) - Espressif From 146b257ddeb2668bd42b306d9ab1d6e542261643 Mon Sep 17 00:00:00 2001 From: Matthew J Perez Date: Thu, 19 Dec 2024 11:29:29 -0500 Subject: [PATCH 14/14] add suggestions, put --flash-size back --- Makefile | 3 ++- OTA.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 94c4d16c..f30870a5 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,8 @@ build-esp32-bin: build-esp32-ota --partition-table=micro-rdk-server/esp32/ota_8mb_partitions.csv \ --target=xtensa-esp32-espidf \ -Zbuild-std=std,panic_abort --release \ - --merge + --flash-size=8mb \ + --merge \ target/xtensa-esp32-espidf/micro-rdk-server-esp32.bin build-esp32-ota: diff --git a/OTA.md b/OTA.md index ab60270d..8be7648e 100644 --- a/OTA.md +++ b/OTA.md @@ -31,7 +31,7 @@ In the `url` field, enter the url where a firmware will be downloaded from The `version` field is equivalent to a `tag` and can be any arbitrary string of up to 128 characters. After successfully applying the new firmware, this `version` will be stored in NVS. -This values is compared to that of the latest machine config from app.viam.com and will trigger the update process. +This value is compared to that of the latest machine config from app.viam.com and will trigger the update process. ## Requirements @@ -80,7 +80,7 @@ This is the build workflow which must be used to: - for example, make a device capable of OTA. **This is not the build that should be hosted at the `url` in the service config.** -You can confirm this by using `ls -l** in your build directory to compare the size of the binary to your partition table. +You can confirm this by using `ls -l` in your build directory to compare the size of the binary to your partition table; the Merged Image will about the size of the full partition table, `8MB` in this example. ## OTA Build @@ -107,7 +107,7 @@ The OTA Service in the micro-rdk currently supports **only HTTP/2**, this means While not all blob storage platform support HTTP/2, many offer Content Delivery Network (CDN) solutions that do. -We don't currently support tokens in the [OTA Service Config](#ota-service-config), so if permissions are required to access the endpoint they must be embedded in the URL itself. +We don't currently support authentication tokens in the [OTA Service Config](#ota-service-config), so if permissions are required to access the endpoint they must be embedded in the URL as query params. ## Related Links