From 0ac1fea6c68b31d87d68ef06b05667d56d87fd55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 27 Sep 2026 23:56:31 +0000 Subject: [PATCH 01/18] feat(mqtt): implement wildcard subscriptions and enhance QoS handling --- docs/design/055-wildcard-inbound-links.md | 66 ++++++++++++++++++----- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 1daf340c..fa8b158e 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -191,9 +191,17 @@ impl AimDb { -> DbResult; } +#[non_exhaustive] +pub struct Subscription { + pub filter: Arc, + /// Config of every link this filter stands for: itself, identical + /// topics and the filters it covers (§5.7). + pub links: Vec>, +} + impl Router { /// Subscription filters with any filter covered by another removed. - pub fn covering_resource_ids(&self) -> Vec>; + pub fn subscriptions(&self) -> Vec; } pub fn pump_source_with(db: &AimDb, scheme: &str, src: impl Source + 'static, @@ -206,6 +214,10 @@ pub fn pump_source_with(db: &AimDb, scheme: &str, src: impl Source + 'static, cannot disagree. The spike replaced the separate `RouterBuilder::from_routes(..)` calls in `native.rs` and `embedded/mod.rs::inbound_topics`. +- The router keeps its grammar (covering needs it, §5.7) and each link's + config, so a connector can read per-link subscribe options. Core does not + interpret the config. A `Router` built with `Router::new` reports + `links: []`; `Route` and `collect_inbound_routes` are unchanged. - **Every in-tree connector moves to `inbound_router`** in the same change: MQTT with `&MqttGrammar`; KNX, the WebSocket server and client, and core's AimX session client (TCP, UDS, serial) with `&ExactGrammar`. Only then is a `{…}` link on those @@ -236,8 +248,11 @@ Errors name the record key and URL: - a multi-level capture or wildcard where the grammar forbids it; - a level `classify` rejects (`a+` in MQTT, `$*` in Zenoh); - any `{…}` on a connector whose grammar has `supports_patterns() == false`; -- patterns returned by a `TopicResolverFn` (018), compiled and checked like - URL patterns. +- patterns returned by a `TopicResolverFn` (018). They go through every + check in this section, including the `.key(..)` capture check: the router + looks up the key's capture slot in the resolved pattern, and a missing + capture is an error rather than a keyed link without keys (§5.6). Both + paths share one parser for the `{…}` syntax. ### 5.4 Matching @@ -315,6 +330,7 @@ pub struct KeyId(NonZeroU16); // Option is 2 bytes; .index() is 0-based #[serde(default, skip_serializing_if = "Option::is_none")] pub inbound_keys: Option, +#[non_exhaustive] pub struct InboundKeysInfo { pub captures: Vec, // one per keyed link pub capacity: u16, @@ -337,15 +353,25 @@ the connector, so it is uncontended. including the parent level (`a/#` matches `a`) and only last, a leading wildcard does not match a `$…` topic, and a wildcard must be a whole level. -- Both backends subscribe `covering_resource_ids()` of +- Both backends subscribe `subscriptions()` of `db.inbound_router("mqtt", &MqttGrammar)` and route with `pump_source_with(.., &MqttGrammar)`. - **Covering set.** A filter is left out when another matches every topic it matches (`sensors/kitchen/temp` under `sensors/+/temp`; `a/+/b` under `a/#`). Required for backend parity (§3.1). The router still fans each - message out to every route. Both backends subscribe at a fixed QoS 1 - today; once per-link subscribe QoS is honoured, a covering filter takes - the highest QoS of the filters it covers. + message out to every route. +- A wildcard covers a literal level only where `wildcard_matches` allows + it. `#` and `+/x` do not cover `$SYS/x`: the broker never delivers `$…` + topics to a leading wildcard, so dropping `$SYS/x` would silence that + link. `sensors/+` does cover `sensors/$x` (the rule is level 0 only). A + wildcard in the covered filter is covered by one at the same level, + because `wildcard_matches` does not depend on the wildcard kind. +- **Subscribe QoS.** Each filter is subscribed at the highest `qos` among + its `links` (set by `with_qos`), default 1. A subscriber receives + `min(publish, subscribe)` QoS, so every covered link gets at least what + it asked for. The embedded backend caps at 1 (mountain-mqtt rejects QoS 2 + subscriptions) and `warn_unsupported_qos` names each inbound route that + asks for 2, once at build. - Outbound links reject patterns at `build()`: you cannot publish to a filter. @@ -361,7 +387,12 @@ literals: serde form is backward compatible. Mark both `#[non_exhaustive]` in the same change, so later fields are not -breaking. +breaking. The attribute itself also breaks struct literals and exhaustive +destructuring outside the crate, so it ships in the same breaking change as +the fields. The new `InboundKeysInfo` is `#[non_exhaustive]` from the start. + +One behaviour change: inbound `with_qos` takes effect. Both backends ignored +it and subscribed at QoS 1. ## 6. Guidance for pattern records @@ -431,9 +462,7 @@ for the whole ingest call, exactly as `Router::route` does today. Because ## 9. Open questions -- **Per-link subscribe QoS.** Both backends subscribe at QoS 1. When - per-link QoS is honoured, confirm the covering filter's QoS rule (§5.7) in - the parity test. +None. ## 10. Acceptance criteria @@ -443,13 +472,22 @@ for the whole ingest call, exactly as `Router::route` does today. Because `ExactGrammar` routers unchanged. 2. `build()` rejects each §5.3 build-time error with the record key; `inbound_router` rejects each connector-build error, including a pattern - on an `ExactGrammar` connector and an invalid resolver-returned pattern. + on an `ExactGrammar` connector, an invalid resolver-returned pattern, and + a resolver-returned pattern without the keyed capture (the error names + the record and the resolved topic). 3. Covering-set unit tests: `sensors/+/temp` covers `sensors/kitchen/temp`; `a/#` covers `a` and `a/+/b`; `a/**/b` covers `a/*/b`; unrelated filters - are all kept. + are all kept. Hidden levels: `#` and `+/x` do not cover `$SYS/x`; + `$SYS/#` covers `$SYS/x`; `#` covers `+/x`; `sensors/+` covers + `sensors/$x`; with the Zenoh-style grammar, `a/*` does not cover `a/@x` + and `a/**` does not cover `a/@x/y`. `subscriptions()` groups each + filter's link config. 4. Parity test, both backends against one broker: a pattern link beside a covered exact link subscribes only the covering filter; each record - receives the message once; capture and key reach the deserializer. + receives the message once; capture and key reach the deserializer. With + `with_qos(2)` on the covered link, the native backend subscribes the + covering filter at QoS 2 (granted QoS in the `SubAck`) and the embedded + backend at QoS 1. Unit test: the highest `qos` wins, the default is 1. 5. End-to-end test: two devices on one pattern record get distinct `KeyId`s; `inbound_key_name` resolves them; a table of capacity 2 drops the third device and counts it; two keyed links on one record share keys; From 9026ffbf05cdc98b3684c5bcc7538ea910cbd118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 00:24:10 +0000 Subject: [PATCH 02/18] docs(design): 055 moves topic matching into connectors --- docs/design/055-wildcard-inbound-links.md | 164 +++++++++++++--------- 1 file changed, 98 insertions(+), 66 deletions(-) diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index fa8b158e..9f9df020 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -1,12 +1,13 @@ # 055 — Wildcard inbound links -**Status:** 📝 Proposed — validated by a spike (§3), 2026-09-27 +**Status:** 📝 Proposed — validated by a spike (§3), 2026-09-27; matching +moved into connectors (§3.3), 2026-09-28 **Scope:** inbound links whose topic is a pattern: matching in `aimdb-core`'s router, the matched topic and captures passed to a match-aware deserializer, optional per-record key interning surfaced in -record metadata, a grammar trait implemented by connectors, and the MQTT -grammar in `aimdb-mqtt-connector`. `RuntimeContext` and `IngestFn` are +record metadata, a grammar trait through which each connector matches its +own topics, and the MQTT grammar in `aimdb-mqtt-connector`. `RuntimeContext` and `IngestFn` are untouched; §5.8 lists the two public structs that gain fields. **Independent of** [054](./054-zero-alloc-connector-boundary.md). This @@ -95,7 +96,8 @@ tests. Everything below was measured on that spike, not estimated. the covering set (§5.7) the backends would disagree. 2. **The grammar is a trait.** Zenoh allows `**` anywhere and hides verbatim `@…` chunks from wildcards at any level. A data struct with - tokens and a "must be last" rule cannot say that (§5.1). + tokens and a "must be last" rule cannot say that (§5.1). §3.3 moved + the matcher itself behind the trait. 3. **Fewer checks run at `build()`.** "Capture shares a level with text" needs the separator, and "multi-level capture must be last" depends on the grammar. Both run when the connector builds (§5.3). @@ -114,6 +116,19 @@ tests. Everything below was measured on that spike, not estimated. path; the dropped counter is `AtomicU32` (thumbv7em has no 64-bit atomics). +### 3.3 After the spike: connectors own matching + +The spike kept one matcher in core and asked the grammar about single +levels. That put every protocol's rules in core: backtracking over `**` +and hidden `@…` chunks for Zenoh, which has no connector yet, next to the +far simpler MQTT rules. Matching now belongs to the connector (§5.1): core +parses the `{…}` syntax, numbers the captures and routes; the grammar +compiles each pattern into a matcher and decides covering. + +The trait moves from one call per wildcard level to one call per pattern +route. The routing times in §3.1 were measured with core's matcher; +criterion 6 measures the allocations again on the new shape. + ## 4. User API ```rust @@ -144,42 +159,59 @@ builder.configure::("sensors.readings", |reg| { ### 5.1 The grammar is a connector-supplied trait Wildcard syntax is protocol-specific and the router is protocol-agnostic. -Core owns one matcher; a grammar only answers questions about single levels: +Core owns the `{…}` syntax and the capture numbering; the connector owns +levels, wildcards and matching: ```rust // aimdb-core -pub enum LevelKind { Literal, Single, Multi, Invalid(&'static str) } +pub const MAX_CAPTURES: usize = 8; +/// Byte range of each capture in the topic, indexed by capture number. +pub type Spans = [(u16, u16); MAX_CAPTURES]; + +/// A topic with its `{…}` syntax checked. +pub struct TopicPattern<'a> { /* the topic, split into parts */ } +pub enum PatternPart<'a> { Text(&'a str), Capture { name: &'a str, multi: bool } } + +impl<'a> TopicPattern<'a> { + pub fn parse(topic: &'a str) -> Result; + pub fn as_str(&self) -> &'a str; + pub fn parts(&self) -> &[PatternPart<'a>]; + pub fn has_captures(&self) -> bool; +} pub trait TopicGrammar: Send + Sync { - /// `false`: every `{…}` link on this connector is an error. - fn supports_patterns(&self) -> bool { true } - fn separator(&self) -> char; - /// What a hand-written level is (`+` → Single, `a+` → Invalid, …). - fn classify(&self, level: &str) -> LevelKind; - /// Tokens used to render `{name}` / `{name..}` for the subscription. - fn single_token(&self) -> &str; - fn multi_token(&self) -> &str; - /// Zenoh `a/**/b`: true. MQTT `#`: false (last only). - fn multi_anywhere(&self) -> bool { false } - /// Whether a wildcard at level `index` may match `level`. - /// MQTT: not a leading `$…`. Zenoh: not a verbatim `@…` chunk. - fn wildcard_matches(&self, index: usize, level: &str) -> bool { true } + /// Compile one pattern. Captures are numbered in `parts()` order. + fn compile(&self, pattern: &TopicPattern<'_>) + -> Result, String>; + /// Whether filter `a` matches every topic filter `b` matches (§5.7). + fn covers(&self, a: &str, b: &str) -> bool { a == b } +} + +pub trait TopicFilter: Send + Sync { + /// What the connector subscribes (`sensors/+/temp`). + fn filter(&self) -> &str; + /// Matches only `filter()` itself; the router compares strings. + fn is_literal(&self) -> bool; + /// Match `topic`, writing capture `i`'s byte range to `spans[i]`. + fn matches(&self, topic: &str, spans: &mut Spans) -> bool; } /// KNX, WebSocket, AimX session connectors: no wildcards. pub struct ExactGrammar; // aimdb-mqtt-connector -pub struct MqttGrammar; // '/', "+", "#", `$` hidden at level 0 +pub struct MqttGrammar; // '/', "+", "#" last only, `$` hidden at level 0 ``` - Grammars are passed as `&'static dyn TopicGrammar` (unit structs: - `&MqttGrammar`). `Router` stays non-generic and the per-match dynamic - calls are one `separator()` at compile time and one - `wildcard_matches()` per wildcard level; §3.1 shows no measurable cost. -- Features outside the trait are rejected by `classify` with a reason, e.g. - Zenoh's `$*` sub-chunk wildcards. Supporting them later is a trait - addition, not a change to the router. + `&MqttGrammar`). `Router` stays non-generic; each pattern route costs one + dynamic `matches()` call per message. +- The router never passes a topic longer than `u16::MAX` bytes, so spans + fit. +- `ExactGrammar` rejects every pattern with captures and compiles the rest + to string equality. +- A second grammar needs no change to core. Zenoh (053) can build on the + `zenoh` crate's own key-expression inclusion for `covers`. ### 5.2 One inbound router per connector @@ -244,10 +276,10 @@ Errors name the record key and URL: **At connector build** (`inbound_router` / `pump_source_with`, returning `DbResult`): everything that needs the grammar: -- a capture sharing a level with text (`sensors/dev-{id}`); -- a multi-level capture or wildcard where the grammar forbids it; -- a level `classify` rejects (`a+` in MQTT, `$*` in Zenoh); -- any `{…}` on a connector whose grammar has `supports_patterns() == false`; +- whatever `TopicGrammar::compile` rejects. For MQTT: a capture sharing a + level with text (`sensors/dev-{id}`), a multi-level capture or `#` that + is not last, a wildcard that is not a whole level (`a+`); +- any `{…}` on an `ExactGrammar` connector; - patterns returned by a `TopicResolverFn` (018). They go through every check in this section, including the `.key(..)` capture check: the router looks up the key's capture slot in the resolved pattern, and a missing @@ -256,15 +288,12 @@ Errors name the record key and URL: ### 5.4 Matching -Each pattern route is compiled once into levels: `Literal`, `Single` or -`Multi`, each wildcard optionally carrying a capture slot. +Each pattern route is compiled once into a `TopicFilter`. A filter whose +`is_literal()` is true becomes an exact route. `Router::route` checks exact routes as today, then pattern routes in -registration order. The matcher walks the topic in place, recording capture -positions as byte ranges in a fixed `[(u16, u16); 8]`. A `Multi` level that -is last takes the rest in one step; one followed by more levels (Zenoh) -backtracks over split points. A topic matching several routes is delivered -to each, as today. +registration order, calling `matches()` with a `Spans` on its stack. A topic +matching several routes is delivered to each, as today. Routes are scanned linearly. At 64 routes a pattern route costs about 60 ns over an exact one (§3.1). An index can come later if a benchmark asks for it. @@ -360,12 +389,12 @@ the connector, so it is uncontended. it matches (`sensors/kitchen/temp` under `sensors/+/temp`; `a/+/b` under `a/#`). Required for backend parity (§3.1). The router still fans each message out to every route. -- A wildcard covers a literal level only where `wildcard_matches` allows - it. `#` and `+/x` do not cover `$SYS/x`: the broker never delivers `$…` - topics to a leading wildcard, so dropping `$SYS/x` would silence that - link. `sensors/+` does cover `sensors/$x` (the rule is level 0 only). A - wildcard in the covered filter is covered by one at the same level, - because `wildcard_matches` does not depend on the wildcard kind. +- `MqttGrammar::covers` compares level by level. A wildcard covers a + literal level only where it may match it: `#` and `+/x` do not cover + `$SYS/x`, because the broker never delivers `$…` topics to a leading + wildcard, so dropping `$SYS/x` would silence that link. `sensors/+` + does cover `sensors/$x` (the rule is level 0 only). `+` covers `+`, and + `#` covers `+` and `#`. - **Subscribe QoS.** Each filter is subscribed at the highest `qos` among its `links` (set by `with_qos`), default 1. A subscriber receives `min(publish, subscribe)` QoS, so every covered link gets at least what @@ -434,30 +463,34 @@ for the whole ingest call, exactly as `Router::route` does today. Because 3. **Grammar as a data struct** (separator, tokens, a `hidden` function). Fits MQTT but cannot express Zenoh's mid-pattern `**` or per-level hidden chunks (§3.2). The trait costs nothing measurable. -4. **Change `IngestFn` to take the topic.** Clean, but breaking; 054 is the +4. **One matcher in core, a per-level grammar trait** (the spike's shape, + §3.3). Core would carry every protocol's matching rules, Zenoh's + backtracking included, and a Zenoh connector could not reuse the + `zenoh` crate's own key-expression logic. +5. **Change `IngestFn` to take the topic.** Clean, but breaking; 054 is the breaking window that removes the need. -5. **Match carried in `RuntimeContext` (`ctx.inbound_match()`).** Reaches +6. **Match carried in `RuntimeContext` (`ctx.inbound_match()`).** Reaches plain `with_deserializer` users, but costs one allocation per message, lets the match outlive the message through a cloned context, and breaks when 054 turns the topic into a borrow. -6. **Records created per new topic at runtime.** Per-publisher buffers and +7. **Records created per new topic at runtime.** Per-publisher buffers and AimX addresses, but it is the post-`run()` registration problem, far larger than this. -7. **Positional captures only (`+` → index 0).** Indices shift when a pattern +8. **Positional captures only (`+` → index 0).** Indices shift when a pattern changes; names cost nothing at runtime. -8. **All checks at `build()`** by declaring grammars on the builder. Adds a +9. **All checks at `build()`** by declaring grammars on the builder. Adds a registration step for every connector; connector-build errors are early enough. -9. **Subscribing every filter as written.** Duplicates on MQTT 5 but not - 3.1.1 (§3.1), so the backends would disagree. **Rejecting overlaps** - instead rules out a legitimate layout. -10. **Delivering unkeyed messages when the key table is full.** Every +10. **Subscribing every filter as written.** Duplicates on MQTT 5 but not + 3.1.1 (§3.1), so the backends would disagree. **Rejecting overlaps** + instead rules out a legitimate layout. +11. **Delivering unkeyed messages when the key table is full.** Every consumer of a keyed link would have to handle `key() == None`. -11. **One key table per link.** Overlapping `KeyId`s on a record with two +12. **One key table per link.** Overlapping `KeyId`s on a record with two keyed links (§3.2). -12. **Reserving the key table's full capacity.** 67 KB for 1,024 keys up +13. **Reserving the key table's full capacity.** 67 KB for 1,024 keys up front; lazy growth measured the same per message. -13. **An index over pattern routes.** Not needed at the measured cost; +14. **An index over pattern routes.** Not needed at the measured cost; revisit with a benchmark. ## 9. Open questions @@ -466,22 +499,21 @@ None. ## 10. Acceptance criteria -1. Matcher unit tests: the MQTT §4.7 cases; captures at first, middle and - last level; `{name..}` matching zero levels; a Zenoh-style test grammar - with `a/**/b`, a mid-pattern `{path..}` and verbatim `@` chunks; - `ExactGrammar` routers unchanged. +1. Core: `TopicPattern::parse` accepts and rejects the §5.3 syntax; + `ExactGrammar` routers unchanged. `MqttGrammar`: the §4.7 cases; + captures at first, middle and last level; `{name..}` matching zero + levels. 2. `build()` rejects each §5.3 build-time error with the record key; `inbound_router` rejects each connector-build error, including a pattern on an `ExactGrammar` connector, an invalid resolver-returned pattern, and a resolver-returned pattern without the keyed capture (the error names the record and the resolved topic). -3. Covering-set unit tests: `sensors/+/temp` covers `sensors/kitchen/temp`; - `a/#` covers `a` and `a/+/b`; `a/**/b` covers `a/*/b`; unrelated filters - are all kept. Hidden levels: `#` and `+/x` do not cover `$SYS/x`; - `$SYS/#` covers `$SYS/x`; `#` covers `+/x`; `sensors/+` covers - `sensors/$x`; with the Zenoh-style grammar, `a/*` does not cover `a/@x` - and `a/**` does not cover `a/@x/y`. `subscriptions()` groups each - filter's link config. +3. `MqttGrammar::covers`: `sensors/+/temp` covers `sensors/kitchen/temp`; + `a/#` covers `a` and `a/+/b`; unrelated filters do not cover. Hidden + levels: `#` and `+/x` do not cover `$SYS/x`; `$SYS/#` covers `$SYS/x`; + `#` covers `+/x`; `sensors/+` covers `sensors/$x`. Core, with a stub + grammar: `subscriptions()` drops covered filters, keeps unrelated ones, + and groups each filter's link config. 4. Parity test, both backends against one broker: a pattern link beside a covered exact link subscribes only the covering filter; each record receives the message once; capture and key reach the deserializer. With From be5f70b52c22834583156239bf849094cbb88b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 00:24:22 +0000 Subject: [PATCH 03/18] feat(core): topic pattern syntax and grammar traits (055) --- aimdb-core/src/lib.rs | 7 + aimdb-core/src/topic_pattern.rs | 251 ++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 aimdb-core/src/topic_pattern.rs diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index dbae34e6..c731fc9d 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -94,6 +94,7 @@ pub mod router; #[cfg(feature = "connector-session")] pub mod session; pub mod signal; +pub mod topic_pattern; pub mod transform; pub mod transport; pub mod typed_api; @@ -167,6 +168,12 @@ pub use connector::{ // Router exports for connector implementations pub use router::{Route, Router, RouterBuilder}; +// Topic grammar for connectors with wildcard subscriptions +pub use topic_pattern::{ + ExactGrammar, PatternError, PatternPart, Spans, TopicFilter, TopicGrammar, TopicPattern, + MAX_CAPTURES, +}; + // Record identification exports pub use record_id::{RecordId, RecordKey, StringKey}; diff --git a/aimdb-core/src/topic_pattern.rs b/aimdb-core/src/topic_pattern.rs new file mode 100644 index 00000000..ff6b0613 --- /dev/null +++ b/aimdb-core/src/topic_pattern.rs @@ -0,0 +1,251 @@ +//! Inbound topic patterns: `{name}` captures one level, `{name..}` the rest. +//! Matching belongs to the connector's [`TopicGrammar`]. + +use alloc::{boxed::Box, format, string::String, vec::Vec}; +use core::fmt; + +/// Most captures one pattern may name. +pub const MAX_CAPTURES: usize = 8; + +/// Byte range of each capture in the topic. +pub type Spans = [(u16, u16); MAX_CAPTURES]; + +/// Invalid `{…}` syntax. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PatternError(String); + +impl fmt::Display for PatternError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// One piece of a [`TopicPattern`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatternPart<'a> { + Text(&'a str), + /// `{name}`, or `{name..}` when `multi`. + Capture { + name: &'a str, + multi: bool, + }, +} + +/// A topic with valid `{…}` syntax. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TopicPattern<'a> { + topic: &'a str, + parts: Vec>, +} + +impl<'a> TopicPattern<'a> { + /// Rejects unbalanced braces, names outside `[A-Za-z0-9_]+`, duplicates + /// and more than [`MAX_CAPTURES`] captures. + pub fn parse(topic: &'a str) -> Result { + let mut parts = Vec::new(); + let mut captures = 0; + let mut rest = topic; + + while let Some(open) = rest.find(['{', '}']) { + if rest[open..].starts_with('}') { + return Err(PatternError(format!("unbalanced '}}' in '{topic}'"))); + } + let after = &rest[open + 1..]; + let close = match after.find(['{', '}']) { + Some(i) if after[i..].starts_with('}') => i, + _ => return Err(PatternError(format!("unbalanced '{{' in '{topic}'"))), + }; + let body = &after[..close]; + let (name, multi) = match body.strip_suffix("..") { + Some(name) => (name, true), + None => (body, false), + }; + if name.is_empty() { + return Err(PatternError(format!("empty capture name in '{topic}'"))); + } + if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(PatternError(format!( + "capture name '{name}' in '{topic}' may only contain A-Z, a-z, 0-9 and _" + ))); + } + if parts + .iter() + .any(|p| matches!(p, PatternPart::Capture { name: n, .. } if *n == name)) + { + return Err(PatternError(format!( + "capture '{name}' appears twice in '{topic}'" + ))); + } + if captures == MAX_CAPTURES { + return Err(PatternError(format!( + "'{topic}' has more than {MAX_CAPTURES} captures" + ))); + } + if open > 0 { + parts.push(PatternPart::Text(&rest[..open])); + } + parts.push(PatternPart::Capture { name, multi }); + captures += 1; + rest = &after[close + 1..]; + } + if !rest.is_empty() { + parts.push(PatternPart::Text(rest)); + } + + Ok(Self { topic, parts }) + } + + pub fn as_str(&self) -> &'a str { + self.topic + } + + /// Text and captures; captures are numbered in this order. + pub fn parts(&self) -> &[PatternPart<'a>] { + &self.parts + } + + pub fn has_captures(&self) -> bool { + self.parts + .iter() + .any(|p| matches!(p, PatternPart::Capture { .. })) + } +} + +/// A connector's wildcard rules. +pub trait TopicGrammar: Send + Sync { + fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String>; + + /// Whether `a` matches every topic `b` matches. May be conservative. + fn covers(&self, a: &str, b: &str) -> bool { + a == b + } +} + +/// One compiled pattern. +pub trait TopicFilter: Send + Sync { + /// What to subscribe (`sensors/+/temp`). + fn filter(&self) -> &str; + + /// Matches only [`filter`](Self::filter) itself. + fn is_literal(&self) -> bool; + + /// Writes capture `i` to `spans[i]`. Topics never exceed `u16::MAX` bytes. + fn matches(&self, topic: &str, spans: &mut Spans) -> bool; +} + +/// No wildcards: string equality, captures rejected. +#[derive(Debug, Clone, Copy, Default)] +pub struct ExactGrammar; + +impl TopicGrammar for ExactGrammar { + fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String> { + if pattern.has_captures() { + return Err(format!( + "'{}': this connector does not support topic patterns", + pattern.as_str() + )); + } + Ok(Box::new(ExactFilter(pattern.as_str().into()))) + } +} + +struct ExactFilter(Box); + +impl TopicFilter for ExactFilter { + fn filter(&self) -> &str { + &self.0 + } + + fn is_literal(&self) -> bool { + true + } + + fn matches(&self, topic: &str, _spans: &mut Spans) -> bool { + topic == &*self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::string::ToString; + + fn parts(topic: &str) -> Vec> { + TopicPattern::parse(topic).unwrap().parts().to_vec() + } + + #[test] + fn parse_splits_text_and_captures() { + use PatternPart::{Capture, Text}; + assert_eq!( + parts("a/{x}/b/{rest..}"), + [ + Text("a/"), + Capture { + name: "x", + multi: false + }, + Text("/b/"), + Capture { + name: "rest", + multi: true + }, + ] + ); + assert_eq!( + parts("{x}"), + [Capture { + name: "x", + multi: false + }] + ); + assert_eq!(parts("a/+/#"), [Text("a/+/#")]); + assert!(!TopicPattern::parse("a/+/#").unwrap().has_captures()); + assert!(TopicPattern::parse("dev-{id}").unwrap().has_captures()); + } + + #[test] + fn parse_errors() { + for (topic, needle) in [ + ("a/{x", "unbalanced '{'"), + ("a/x}", "unbalanced '}'"), + ("a/{{x}}", "unbalanced '{'"), + ("a/{}", "empty capture name"), + ("a/{..}", "empty capture name"), + ("a/{x-y}", "may only contain"), + ("a/{x}/{x..}", "appears twice"), + ("{a}/{b}/{c}/{d}/{e}/{f}/{g}/{h}/{i}", "more than 8"), + ] { + let err = TopicPattern::parse(topic).unwrap_err().to_string(); + assert!(err.contains(needle), "'{topic}': {err}"); + } + assert!(TopicPattern::parse("{a}/{b}/{c}/{d}/{e}/{f}/{g}/{h}").is_ok()); + } + + #[test] + fn exact_grammar_compares_strings() { + let filter = ExactGrammar + .compile(&TopicPattern::parse("a/+/#").unwrap()) + .unwrap(); + let mut spans = [(0, 0); MAX_CAPTURES]; + assert!(filter.is_literal()); + assert_eq!(filter.filter(), "a/+/#"); + assert!(filter.matches("a/+/#", &mut spans)); + assert!(!filter.matches("a/b/c", &mut spans)); + } + + #[test] + fn exact_grammar_rejects_captures() { + let err = ExactGrammar + .compile(&TopicPattern::parse("a/{x}").unwrap()) + .err() + .unwrap(); + assert!(err.contains("does not support topic patterns"), "{err}"); + } + + #[test] + fn default_covers_is_equality() { + assert!(ExactGrammar.covers("a/b", "a/b")); + assert!(!ExactGrammar.covers("a/+", "a/b")); + } +} From 1f11b51ccae541d8d7a66a646406bab3b913119e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 00:35:21 +0000 Subject: [PATCH 04/18] feat(core): per-record inbound key table (055) --- aimdb-core/src/inbound_key.rs | 150 ++++++++++++++++++++++ aimdb-core/src/lib.rs | 6 + docs/design/055-wildcard-inbound-links.md | 4 +- 3 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 aimdb-core/src/inbound_key.rs diff --git a/aimdb-core/src/inbound_key.rs b/aimdb-core/src/inbound_key.rs new file mode 100644 index 00000000..4baf5204 --- /dev/null +++ b/aimdb-core/src/inbound_key.rs @@ -0,0 +1,150 @@ +//! Keys: small integers standing for the values of one capture, one table per +//! record. + +use alloc::{sync::Arc, vec::Vec}; +use core::num::NonZeroU16; +use core::sync::atomic::{AtomicU32, Ordering}; + +use hashbrown::HashMap; + +#[cfg(feature = "std")] +type Mutex = std::sync::Mutex; +#[cfg(not(feature = "std"))] +type Mutex = spin::Mutex; + +#[cfg(feature = "std")] +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} +#[cfg(not(feature = "std"))] +fn lock(m: &Mutex) -> spin::MutexGuard<'_, T> { + m.lock() +} + +/// A capture value's key, assigned the first time the value is seen. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct KeyId(NonZeroU16); + +impl KeyId { + /// 0-based, for indexing per-key state. + pub fn index(self) -> usize { + usize::from(self.0.get()) - 1 + } +} + +/// A record's key table. Grows as values arrive, up to `capacity`. +pub(crate) struct KeyTable { + capacity: NonZeroU16, + keys: Mutex, + dropped: AtomicU32, +} + +#[derive(Default)] +struct Keys { + by_name: HashMap, KeyId>, + names: Vec>, +} + +impl KeyTable { + pub(crate) fn new(capacity: NonZeroU16) -> Self { + Self { + capacity, + keys: Mutex::new(Keys::default()), + dropped: AtomicU32::new(0), + } + } + + /// The key for `name`, assigned if new. `None` when the table is full; + /// the caller drops the message and it is counted. + pub(crate) fn key(&self, name: &str) -> Option { + let mut keys = lock(&self.keys); + if let Some(&id) = keys.by_name.get(name) { + return Some(id); + } + let Some(id) = u16::try_from(keys.names.len() + 1) + .ok() + .filter(|&n| n <= self.capacity.get()) + .and_then(NonZeroU16::new) + .map(KeyId) + else { + let _ = self + .dropped + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1)); + return None; + }; + let name: Arc = name.into(); + keys.names.push(name.clone()); + keys.by_name.insert(name, id); + Some(id) + } + + /// The value `key` stands for. + pub(crate) fn name(&self, key: KeyId) -> Option> { + lock(&self.keys).names.get(key.index()).cloned() + } + + pub(crate) fn capacity(&self) -> u16 { + self.capacity.get() + } + + pub(crate) fn assigned(&self) -> usize { + lock(&self.keys).names.len() + } + + /// Messages turned away because the table was full. + pub(crate) fn dropped(&self) -> u32 { + self.dropped.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn table(capacity: u16) -> KeyTable { + KeyTable::new(NonZeroU16::new(capacity).unwrap()) + } + + #[test] + fn keys_are_assigned_in_order_and_stable() { + let t = table(4); + let a = t.key("kitchen").unwrap(); + let b = t.key("hall").unwrap(); + assert_eq!((a.index(), b.index()), (0, 1)); + assert_eq!(t.key("kitchen"), Some(a)); + assert_eq!(t.assigned(), 2); + assert_eq!(t.name(a).as_deref(), Some("kitchen")); + assert_eq!(t.name(b).as_deref(), Some("hall")); + } + + #[test] + fn full_table_drops_new_values_and_keeps_known_ones() { + let t = table(2); + let a = t.key("a").unwrap(); + t.key("b").unwrap(); + assert_eq!(t.key("c"), None); + assert_eq!(t.key("d"), None); + assert_eq!(t.key("a"), Some(a)); + assert_eq!((t.capacity(), t.assigned(), t.dropped()), (2, 2, 2)); + } + + #[test] + fn unknown_key_has_no_name() { + let big = table(8); + let foreign = big.key("x").and(big.key("y")).unwrap(); + assert_eq!(table(8).name(foreign), None); + } + + #[test] + fn full_u16_capacity_does_not_overflow() { + let t = table(u16::MAX); + for i in 0..u16::MAX { + assert_eq!( + t.key(&alloc::format!("{i}")).map(KeyId::index), + Some(usize::from(i)) + ); + } + assert_eq!(t.key("one more"), None); + assert_eq!(t.dropped(), 1); + } +} diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index c731fc9d..4128acce 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -85,6 +85,9 @@ mod error; pub mod executor; pub mod extensions; pub mod graph; +// Used by the router once pattern routes land. +#[allow(dead_code)] +mod inbound_key; #[cfg(feature = "observability")] pub mod profiling; pub mod record_id; @@ -174,6 +177,9 @@ pub use topic_pattern::{ MAX_CAPTURES, }; +// Keys assigned to capture values of keyed inbound links +pub use inbound_key::KeyId; + // Record identification exports pub use record_id::{RecordId, RecordKey, StringKey}; diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 9f9df020..5ffec2ee 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -340,8 +340,8 @@ pub struct KeyId(NonZeroU16); // Option is 2 bytes; .index() is 0-based `temp/{dev}` and `hum/{id}` gets the same key. Keyed links on one record must give the same capacity. - The table is `HashMap, KeyId>` (hashbrown) plus - `Vec>` for the reverse direction, under one `spin::Mutex`. Both - dependencies are already in `aimdb-core`. + `Vec>` for the reverse direction, under one mutex (`std` or + `spin`, as elsewhere in `aimdb-core`). - **It grows as keys arrive**; capacity is a limit, not a reservation. Memory is about 66 bytes per key plus the name (§3.1). - A new value costs one allocation (its name); a known value costs none. From d08e4345821031437332168006d6ebf056661c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 00:54:17 +0000 Subject: [PATCH 05/18] feat(core): pattern routes, TopicMatch and subscriptions in Router (055) --- aimdb-core/src/connector.rs | 7 + aimdb-core/src/lib.rs | 6 +- aimdb-core/src/router.rs | 351 +++++++++++++++++++++- aimdb-core/src/topic_pattern.rs | 43 +++ aimdb-mqtt-connector/src/link_ext.rs | 4 +- docs/design/055-wildcard-inbound-links.md | 41 +-- 6 files changed, 411 insertions(+), 41 deletions(-) diff --git a/aimdb-core/src/connector.rs b/aimdb-core/src/connector.rs index fa4c521a..67fa738a 100644 --- a/aimdb-core/src/connector.rs +++ b/aimdb-core/src/connector.rs @@ -515,6 +515,13 @@ impl ConnectorLink { /// captured) for context-aware deserializers. pub type IngestFn = Arc Result<(), String> + Send + Sync>; +/// Fused ingest callback of a pattern route: also receives the match. +pub type MatchIngestFn = Arc< + dyn Fn(&crate::RuntimeContext, &crate::TopicMatch<'_>, &[u8]) -> Result<(), String> + + Send + + Sync, +>; + /// Type alias for ingest factory callback (alloc feature) /// /// Takes the live [`AimDb`] and returns the fused [`IngestFn`]. This allows diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 4128acce..eda282d0 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -162,7 +162,7 @@ pub use profiling::{ pub use connector::TopicProvider; pub use connector::TopicResolverFn; pub use connector::{ConnectorLink, ConnectorUrl, LinkAddress, SerializeError}; -pub use connector::{IngestFactoryFn, IngestFn}; +pub use connector::{IngestFactoryFn, IngestFn, MatchIngestFn}; pub use connector::{ SerializedPayload, SerializedReader, SerializedSource, SerializedValue, SerializedValueInto, SourceFactoryFn, @@ -173,8 +173,8 @@ pub use router::{Route, Router, RouterBuilder}; // Topic grammar for connectors with wildcard subscriptions pub use topic_pattern::{ - ExactGrammar, PatternError, PatternPart, Spans, TopicFilter, TopicGrammar, TopicPattern, - MAX_CAPTURES, + ExactGrammar, PatternError, PatternPart, Spans, TopicFilter, TopicGrammar, TopicMatch, + TopicPattern, MAX_CAPTURES, }; // Keys assigned to capture values of keyed inbound links diff --git a/aimdb-core/src/router.rs b/aimdb-core/src/router.rs index 00ae2f21..b1351195 100644 --- a/aimdb-core/src/router.rs +++ b/aimdb-core/src/router.rs @@ -11,9 +11,13 @@ //! - DDS: Routes topics to records //! - Shared Memory: Routes segment names to records -use alloc::{string::String, sync::Arc, vec::Vec}; +use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; -use crate::connector::IngestFn; +use crate::connector::{IngestFn, MatchIngestFn}; +use crate::inbound_key::{KeyId, KeyTable}; +use crate::topic_pattern::{ + ExactGrammar, Spans, TopicFilter, TopicGrammar, TopicMatch, MAX_CAPTURES, +}; /// A single routing entry /// @@ -62,13 +66,86 @@ pub struct Route { /// - **Shmem**: `segment_name` (e.g., "temperature_buffer") pub struct Router { /// List of all registered routes - routes: Vec, + routes: Vec, + /// Decides covering in [`subscriptions`](Self::subscriptions). + grammar: &'static dyn TopicGrammar, +} + +/// A route as the router runs it. An exact route is one whose filter matches +/// only itself. +pub(crate) struct CompiledRoute { + filter: Arc, + /// `None`: compare `filter` as a string. + matcher: Option>, + /// Capture names by number. + names: Box<[Box]>, + /// Key table and the capture number whose value is keyed. + key: Option<(Arc, usize)>, + ingest: MatchIngestFn, +} + +impl CompiledRoute { + /// A route comparing `resource_id` as a string. + pub(crate) fn exact(resource_id: Arc, ingest: IngestFn) -> Self { + Self { + filter: resource_id, + matcher: None, + names: Box::new([]), + key: None, + ingest: Arc::new(move |ctx, _m, payload| ingest(ctx, payload)), + } + } + + /// A route matching through `filter`. + #[allow(dead_code)] + pub(crate) fn pattern( + filter: Box, + names: Box<[Box]>, + key: Option<(Arc, usize)>, + ingest: MatchIngestFn, + ) -> Self { + Self { + filter: filter.filter().into(), + matcher: (!filter.is_literal()).then_some(filter), + names, + key, + ingest, + } + } + + fn matches(&self, topic: &str, spans: &mut Spans) -> bool { + match &self.matcher { + None => *self.filter == *topic, + Some(m) => topic.len() <= usize::from(u16::MAX) && m.matches(topic, spans), + } + } + + /// The message's key; `None` when the key table is full. + fn key(&self, topic: &str, spans: &Spans) -> Option> { + let Some((table, capture)) = &self.key else { + return Some(None); + }; + let &(start, end) = spans.get(*capture)?; + let value = topic.get(usize::from(start)..usize::from(end))?; + table.key(value).map(Some) + } } impl Router { /// Create a new router with the given routes pub fn new(routes: Vec) -> Self { - Self { routes } + Self { + routes: routes + .into_iter() + .map(|r| CompiledRoute::exact(r.resource_id, r.ingest)) + .collect(), + grammar: &ExactGrammar, + } + } + + #[allow(dead_code)] + pub(crate) fn compiled(grammar: &'static dyn TopicGrammar, routes: Vec) -> Self { + Self { routes, grammar } } /// Route a message to the appropriate record(s) @@ -101,10 +178,16 @@ impl Router { // Linear search through all routes // Note: Multiple routes may match the same resource_id (different types) + let mut spans: Spans = [(0, 0); MAX_CAPTURES]; for route in &self.routes { - if route.resource_id.as_ref() == resource_id { + if route.matches(resource_id, &mut spans) { matched = true; - match (route.ingest)(ctx, payload) { + let Some(key) = route.key(resource_id, &spans) else { + log_debug!("Key table full, dropped message on '{}'", resource_id); + continue; + }; + let m = TopicMatch::new(resource_id, &route.names, &spans, key); + match (route.ingest)(ctx, &m, payload) { Ok(()) => { routed = true; @@ -149,7 +232,7 @@ impl Router { /// Useful for subscribing at the protocol level (e.g., MQTT SUBSCRIBE). /// Returns unique resource IDs (deduplicated even if multiple routes per resource). pub fn resource_ids(&self) -> Vec> { - let mut ids: Vec> = self.routes.iter().map(|r| r.resource_id.clone()).collect(); + let mut ids: Vec> = self.routes.iter().map(|r| r.filter.clone()).collect(); // Deduplicate by converting to strings for comparison ids.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref())); @@ -158,6 +241,23 @@ impl Router { ids } + /// Filters to subscribe: [`resource_ids`](Self::resource_ids) without + /// the filters another one covers. + pub fn subscriptions(&self) -> Vec> { + let ids = self.resource_ids(); + let g = self.grammar; + ids.iter() + .enumerate() + .filter(|&(i, a)| { + // Of two filters covering each other, the first one stays. + !ids.iter() + .enumerate() + .any(|(j, b)| j != i && g.covers(b, a) && (j < i || !g.covers(a, b))) + }) + .map(|(_, a)| a.clone()) + .collect() + } + /// Get the number of routes in this router pub fn route_count(&self) -> usize { self.routes.len() @@ -376,4 +476,241 @@ mod tests { // Ingest failures are logged, not propagated. router.route("err/resource", b"dummy", &test_ctx()).unwrap(); } + + // ---- pattern routes --------------------------------------------------- + + use crate::topic_pattern::{PatternPart, TopicPattern}; + use core::num::NonZeroU16; + use std::sync::Mutex; + + /// `/`-separated levels; `+` and `{name}` match one level. + struct Plus; + + struct PlusFilter { + filter: String, + /// Per level: `None` for a wildcard. + levels: Vec>, + /// Per level: capture number. + captures: Vec>, + } + + impl TopicGrammar for Plus { + fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String> { + let mut filter = String::new(); + let mut captures = Vec::new(); + for part in pattern.parts() { + match part { + PatternPart::Text(t) => filter.push_str(t), + PatternPart::Capture { .. } => { + filter.push('+'); + captures.push(filter.split('/').count() - 1); + } + } + } + let levels: Vec> = filter + .split('/') + .map(|l| (l != "+").then(|| l.to_string())) + .collect(); + let captures = (0..levels.len()) + .map(|i| captures.iter().position(|&c| c == i)) + .collect(); + Ok(Box::new(PlusFilter { + filter, + levels, + captures, + })) + } + + fn covers(&self, a: &str, b: &str) -> bool { + a.split('/').count() == b.split('/').count() + && a.split('/') + .zip(b.split('/')) + .all(|(x, y)| x == "+" || x == y) + } + } + + impl TopicFilter for PlusFilter { + fn filter(&self) -> &str { + &self.filter + } + fn is_literal(&self) -> bool { + self.levels.iter().all(Option::is_some) + } + fn matches(&self, topic: &str, spans: &mut Spans) -> bool { + if topic.split('/').count() != self.levels.len() { + return false; + } + let mut start = 0; + for (i, level) in topic.split('/').enumerate() { + match &self.levels[i] { + Some(lit) if lit != level => return false, + _ => {} + } + if let Some(c) = self.captures[i] { + spans[c] = (start as u16, (start + level.len()) as u16); + } + start += level.len() + 1; + } + true + } + } + + type Seen = Arc>, Option)>>>; + + /// Records the topic, the named captures and the key index. + fn recording(seen: &Seen, names: &'static [&'static str]) -> MatchIngestFn { + let seen = seen.clone(); + Arc::new(move |_ctx, m, _payload| { + let caps = names.iter().map(|n| m.get(n).map(String::from)).collect(); + seen.lock() + .unwrap() + .push((m.topic().to_string(), caps, m.key().map(|k| k.index()))); + Ok(()) + }) + } + + fn pattern_route( + topic: &str, + ingest: MatchIngestFn, + key: Option<(Arc, usize)>, + ) -> CompiledRoute { + let pattern = TopicPattern::parse(topic).unwrap(); + let names = pattern + .parts() + .iter() + .filter_map(|p| match p { + PatternPart::Capture { name, .. } => Some(Box::from(*name)), + PatternPart::Text(_) => None, + }) + .collect(); + CompiledRoute::pattern(Plus.compile(&pattern).unwrap(), names, key, ingest) + } + + fn exact_route(topic: &str, ingest: IngestFn) -> CompiledRoute { + CompiledRoute::exact(Arc::from(topic), ingest) + } + + #[test] + fn pattern_route_receives_topic_and_captures() { + let seen: Seen = Default::default(); + let router = Router::compiled( + &Plus, + vec![pattern_route( + "{site}/+/{dev}", + recording(&seen, &["site", "dev", "nope"]), + None, + )], + ); + let ctx = test_ctx(); + router.route("vienna/x/k1", b"", &ctx).unwrap(); + router.route("vienna/k1", b"", &ctx).unwrap(); + + assert_eq!( + *seen.lock().unwrap(), + vec![( + "vienna/x/k1".to_string(), + vec![Some("vienna".into()), Some("k1".into()), None], + None + )] + ); + } + + #[test] + fn exact_and_pattern_routes_both_receive_a_message() { + let exact = Arc::new(AtomicUsize::new(0)); + let seen: Seen = Default::default(); + let router = Router::compiled( + &Plus, + vec![ + exact_route("s/kitchen/t", counting_ingest(exact.clone())), + pattern_route("s/{d}/t", recording(&seen, &["d"]), None), + pattern_route("s/kitchen/t", recording(&seen, &[]), None), + ], + ); + router.route("s/kitchen/t", b"", &test_ctx()).unwrap(); + + assert_eq!(exact.load(Ordering::SeqCst), 1); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + // A literal pattern route still receives the topic. + assert_eq!(seen[1].0, "s/kitchen/t"); + } + + #[test] + fn keyed_route_assigns_keys_and_drops_when_full() { + let seen: Seen = Default::default(); + let table = Arc::new(KeyTable::new(NonZeroU16::new(2).unwrap())); + let router = Router::compiled( + &Plus, + vec![pattern_route( + "s/{d}/t", + recording(&seen, &["d"]), + Some((table.clone(), 0)), + )], + ); + let ctx = test_ctx(); + for topic in ["s/a/t", "s/b/t", "s/a/t", "s/c/t"] { + router.route(topic, b"", &ctx).unwrap(); + } + + let keys: Vec<_> = seen.lock().unwrap().iter().map(|s| s.2).collect(); + assert_eq!(keys, [Some(0), Some(1), Some(0)]); + assert_eq!(table.dropped(), 1); + } + + #[test] + fn overlong_topics_skip_pattern_routes() { + let seen: Seen = Default::default(); + let router = Router::compiled( + &Plus, + vec![pattern_route("{x}", recording(&seen, &[]), None)], + ); + let topic = "x".repeat(usize::from(u16::MAX) + 1); + router.route(&topic, b"", &test_ctx()).unwrap(); + assert!(seen.lock().unwrap().is_empty()); + } + + fn subscriptions(router: &Router) -> Vec { + router + .subscriptions() + .iter() + .map(|s| s.to_string()) + .collect() + } + + #[test] + fn subscriptions_drop_covered_filters() { + let noop = || counting_ingest(Arc::new(AtomicUsize::new(0))); + let seen: Seen = Default::default(); + let router = Router::compiled( + &Plus, + vec![ + exact_route("s/kitchen/t", noop()), + exact_route("other/x", noop()), + exact_route("s/+/t", noop()), + pattern_route("s/{d}/t", recording(&seen, &[]), None), + ], + ); + assert_eq!(subscriptions(&router), ["other/x", "s/+/t"]); + } + + #[test] + fn plain_router_subscribes_each_id_once() { + let noop = || counting_ingest(Arc::new(AtomicUsize::new(0))); + let router = Router::new(vec![ + Route { + resource_id: Arc::from("a"), + ingest: noop(), + }, + Route { + resource_id: Arc::from("a"), + ingest: noop(), + }, + Route { + resource_id: Arc::from("b"), + ingest: noop(), + }, + ]); + assert_eq!(subscriptions(&router), ["a", "b"]); + } } diff --git a/aimdb-core/src/topic_pattern.rs b/aimdb-core/src/topic_pattern.rs index ff6b0613..96cae6c6 100644 --- a/aimdb-core/src/topic_pattern.rs +++ b/aimdb-core/src/topic_pattern.rs @@ -4,6 +4,8 @@ use alloc::{boxed::Box, format, string::String, vec::Vec}; use core::fmt; +use crate::inbound_key::KeyId; + /// Most captures one pattern may name. pub const MAX_CAPTURES: usize = 8; @@ -111,6 +113,47 @@ impl<'a> TopicPattern<'a> { } } +/// The topic a pattern route matched, borrowed for one ingest call. +#[derive(Debug, Clone, Copy)] +pub struct TopicMatch<'a> { + topic: &'a str, + names: &'a [Box], + spans: &'a Spans, + key: Option, +} + +impl<'a> TopicMatch<'a> { + pub(crate) fn new( + topic: &'a str, + names: &'a [Box], + spans: &'a Spans, + key: Option, + ) -> Self { + Self { + topic, + names, + spans, + key, + } + } + + pub fn topic(&self) -> &'a str { + self.topic + } + + /// The value of capture `name`. + pub fn get(&self, name: &str) -> Option<&'a str> { + let i = self.names.iter().position(|n| &**n == name)?; + let &(start, end) = self.spans.get(i)?; + self.topic.get(usize::from(start)..usize::from(end)) + } + + /// `Some` iff the link is keyed. + pub fn key(&self) -> Option { + self.key + } +} + /// A connector's wildcard rules. pub trait TopicGrammar: Send + Sync { fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String>; diff --git a/aimdb-mqtt-connector/src/link_ext.rs b/aimdb-mqtt-connector/src/link_ext.rs index bf81659f..b7ff54c3 100644 --- a/aimdb-mqtt-connector/src/link_ext.rs +++ b/aimdb-mqtt-connector/src/link_ext.rs @@ -28,8 +28,8 @@ use core::fmt::Debug; pub trait MqttLinkExt: Sized { /// Sets the MQTT Quality of Service level (0, 1, or 2). /// - /// Outbound: the publish QoS. Inbound: the subscribe QoS. Defaults to - /// QoS 1 when unset (the connectors' own default). + /// Outbound: the publish QoS, defaulting to 1. Inbound: not applied yet; + /// both backends subscribe at QoS 1. fn with_qos(self, qos: u8) -> Self; } diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 5ffec2ee..8eff3b72 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -64,6 +64,8 @@ Two facts make a small change sufficient: - Making the match visible to plain `with_deserializer` closures (§5.5). - Releasing or reusing keys. A key lives as long as the process. - An index (trie) over pattern routes. Routes are scanned linearly (§5.4). +- Inbound subscribe QoS. Both MQTT backends subscribe at QoS 1 and ignore + `with_qos` on inbound links, today and after this design (§5.7). ## 3. Evaluation @@ -223,17 +225,9 @@ impl AimDb { -> DbResult; } -#[non_exhaustive] -pub struct Subscription { - pub filter: Arc, - /// Config of every link this filter stands for: itself, identical - /// topics and the filters it covers (§5.7). - pub links: Vec>, -} - impl Router { - /// Subscription filters with any filter covered by another removed. - pub fn subscriptions(&self) -> Vec; + /// Filters to subscribe, with any filter covered by another removed. + pub fn subscriptions(&self) -> Vec>; } pub fn pump_source_with(db: &AimDb, scheme: &str, src: impl Source + 'static, @@ -246,10 +240,8 @@ pub fn pump_source_with(db: &AimDb, scheme: &str, src: impl Source + 'static, cannot disagree. The spike replaced the separate `RouterBuilder::from_routes(..)` calls in `native.rs` and `embedded/mod.rs::inbound_topics`. -- The router keeps its grammar (covering needs it, §5.7) and each link's - config, so a connector can read per-link subscribe options. Core does not - interpret the config. A `Router` built with `Router::new` reports - `links: []`; `Route` and `collect_inbound_routes` are unchanged. +- The router keeps its grammar, which covering needs (§5.7). `Route` and + `collect_inbound_routes` are unchanged. - **Every in-tree connector moves to `inbound_router`** in the same change: MQTT with `&MqttGrammar`; KNX, the WebSocket server and client, and core's AimX session client (TCP, UDS, serial) with `&ExactGrammar`. Only then is a `{…}` link on those @@ -395,12 +387,9 @@ the connector, so it is uncontended. wildcard, so dropping `$SYS/x` would silence that link. `sensors/+` does cover `sensors/$x` (the rule is level 0 only). `+` covers `+`, and `#` covers `+` and `#`. -- **Subscribe QoS.** Each filter is subscribed at the highest `qos` among - its `links` (set by `with_qos`), default 1. A subscriber receives - `min(publish, subscribe)` QoS, so every covered link gets at least what - it asked for. The embedded backend caps at 1 (mountain-mqtt rejects QoS 2 - subscriptions) and `warn_unsupported_qos` names each inbound route that - asks for 2, once at build. +- **Subscribe QoS** stays 1 for every filter, as today. Covering drops a + filter only where another delivers the same messages at the same QoS, so + nothing is lost. Inbound `with_qos` stays unapplied; its doc says so. - Outbound links reject patterns at `build()`: you cannot publish to a filter. @@ -420,9 +409,6 @@ breaking. The attribute itself also breaks struct literals and exhaustive destructuring outside the crate, so it ships in the same breaking change as the fields. The new `InboundKeysInfo` is `#[non_exhaustive]` from the start. -One behaviour change: inbound `with_qos` takes effect. Both backends ignored -it and subscribed at QoS 1. - ## 6. Guidance for pattern records A pattern record is one interleaved stream: its latest value is whichever @@ -512,14 +498,11 @@ None. `a/#` covers `a` and `a/+/b`; unrelated filters do not cover. Hidden levels: `#` and `+/x` do not cover `$SYS/x`; `$SYS/#` covers `$SYS/x`; `#` covers `+/x`; `sensors/+` covers `sensors/$x`. Core, with a stub - grammar: `subscriptions()` drops covered filters, keeps unrelated ones, - and groups each filter's link config. + grammar: `subscriptions()` drops covered filters and keeps unrelated + ones. 4. Parity test, both backends against one broker: a pattern link beside a covered exact link subscribes only the covering filter; each record - receives the message once; capture and key reach the deserializer. With - `with_qos(2)` on the covered link, the native backend subscribes the - covering filter at QoS 2 (granted QoS in the `SubAck`) and the embedded - backend at QoS 1. Unit test: the highest `qos` wins, the default is 1. + receives the message once; capture and key reach the deserializer. 5. End-to-end test: two devices on one pattern record get distinct `KeyId`s; `inbound_key_name` resolves them; a table of capacity 2 drops the third device and counts it; two keyed links on one record share keys; From cca20b68277379f97384cb34370086a910aaf1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 01:12:14 +0000 Subject: [PATCH 06/18] feat(core): with_match_deserializer and .key() on inbound links (055) --- aimdb-core/src/builder.rs | 19 +- aimdb-core/src/connector.rs | 31 ++ aimdb-core/src/lib.rs | 2 +- aimdb-core/src/typed_api.rs | 444 ++++++++++++++++++---- docs/design/055-wildcard-inbound-links.md | 12 +- 5 files changed, 417 insertions(+), 91 deletions(-) diff --git a/aimdb-core/src/builder.rs b/aimdb-core/src/builder.rs index 80710d55..5c7bc48b 100644 --- a/aimdb-core/src/builder.rs +++ b/aimdb-core/src/builder.rs @@ -1173,7 +1173,8 @@ impl AimDb { /// Vector of tuples: (topic, ingest) /// /// The topic is resolved dynamically if a `TopicResolverFn` is configured, - /// otherwise the static topic from the URL is used. + /// otherwise the static topic from the URL is used. Links whose topic has + /// `{…}` captures are skipped with a warning. pub fn collect_inbound_routes( &self, scheme: &str, @@ -1192,8 +1193,22 @@ impl AimDb { // Resolve topic: dynamic (from resolver) or static (from URL) let topic = link.resolve_topic(); + if crate::TopicPattern::parse(&topic).is_ok_and(|p| p.has_captures()) { + log_warn!( + "Skipping inbound link '{}': this connector does not support topic patterns", + topic + ); + continue; + } + // Create the fused ingest callback using the stored factory - routes.push((topic, link.create_ingest(self))); + let ingest = match &link.match_ingest_factory { + Some(factory) => { + crate::connector::match_as_ingest(factory(self), topic.as_str().into()) + } + None => link.create_ingest(self), + }; + routes.push((topic, ingest)); } } diff --git a/aimdb-core/src/connector.rs b/aimdb-core/src/connector.rs index 67fa738a..ef02e454 100644 --- a/aimdb-core/src/connector.rs +++ b/aimdb-core/src/connector.rs @@ -532,6 +532,22 @@ pub type MatchIngestFn = Arc< /// Available in both `std` and `no_std + alloc` environments. pub type IngestFactoryFn = Arc IngestFn + Send + Sync>; +/// Like [`IngestFactoryFn`], for links set with `with_match_deserializer`. +pub type MatchIngestFactoryFn = Arc MatchIngestFn + Send + Sync>; + +/// Runs a match-aware ingest where only an [`IngestFn`] fits: every message +/// is on `topic`, with no captures and no key. +pub(crate) fn match_as_ingest(ingest: MatchIngestFn, topic: Arc) -> IngestFn { + static NO_SPANS: crate::Spans = [(0, 0); crate::MAX_CAPTURES]; + Arc::new(move |ctx, payload| { + ingest( + ctx, + &crate::TopicMatch::new(&topic, &[], &NO_SPANS, None), + payload, + ) + }) +} + /// Topic resolver function for inbound connections (late-binding) /// /// Called once at connector startup to resolve the subscription topic. @@ -556,6 +572,7 @@ pub type TopicResolverFn = Arc Option + Send + Sync>; /// factory captures the type T at creation time, allowing type-safe /// deserialize+produce later without needing PhantomData or type parameters. #[derive(Clone)] +#[non_exhaustive] pub struct InboundConnectorLink { /// Parsed link address (`scheme://resource`) pub url: LinkAddress, @@ -574,6 +591,13 @@ pub struct InboundConnectorLink { /// Available in both `std` and `no_std + alloc` environments. pub ingest_factory: IngestFactoryFn, + /// Set by `with_match_deserializer`; `ingest_factory` then passes the + /// URL topic as the match. + pub match_ingest_factory: Option, + + /// Set by `.key(..)`: the keyed capture and the key table's capacity. + pub key: Option<(String, core::num::NonZeroU16)>, + /// Optional dynamic topic resolver (late-binding) /// /// Called once at connector startup to determine the subscription topic. @@ -589,6 +613,11 @@ impl Debug for InboundConnectorLink { .field("url", &self.url) .field("config", &self.config) .field("ingest_factory", &"") + .field( + "match_ingest_factory", + &self.match_ingest_factory.as_ref().map(|_| ""), + ) + .field("key", &self.key) .field( "topic_resolver", &self.topic_resolver.as_ref().map(|_| ""), @@ -604,6 +633,8 @@ impl InboundConnectorLink { url, config: Vec::new(), ingest_factory, + match_ingest_factory: None, + key: None, topic_resolver: None, } } diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index eda282d0..37484ffd 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -162,7 +162,7 @@ pub use profiling::{ pub use connector::TopicProvider; pub use connector::TopicResolverFn; pub use connector::{ConnectorLink, ConnectorUrl, LinkAddress, SerializeError}; -pub use connector::{IngestFactoryFn, IngestFn, MatchIngestFn}; +pub use connector::{IngestFactoryFn, IngestFn, MatchIngestFactoryFn, MatchIngestFn}; pub use connector::{ SerializedPayload, SerializedReader, SerializedSource, SerializedValue, SerializedValueInto, SourceFactoryFn, diff --git a/aimdb-core/src/typed_api.rs b/aimdb-core/src/typed_api.rs index 99a40476..f3e8d71e 100644 --- a/aimdb-core/src/typed_api.rs +++ b/aimdb-core/src/typed_api.rs @@ -755,6 +755,8 @@ where url: url.to_string(), config: Vec::new(), context_deserializer: None, + match_deserializer: None, + key: None, topic_resolver: None, } } @@ -903,6 +905,15 @@ where let url_string = url.to_string(); let scheme = url.scheme().to_string(); + if crate::TopicPattern::parse(url.resource_id()).is_ok_and(|p| p.has_captures()) { + self.registrar.rec.push_config_error(ConfigError::new( + record_key, + Some(self.url), + "Outbound links cannot use topic patterns", + )); + return self.registrar; + } + // Adapt the stored serializer to the fused calling convention. Stays // typed: fused with the consumer below, no `Box` per message //. @@ -1021,6 +1032,69 @@ where // InboundConnectorBuilder - Fluent inbound connector configuration // ============================================================================ +/// The producer an inbound ingest factory writes to. Factories run during +/// build() after every record is registered and validated, so a failed lookup +/// is an aimdb bug, not a user mistake. +#[allow( + clippy::panic, + reason = "the factory returns no Result and this lookup was validated at build() time" +)] +fn inbound_producer(db: &AimDb, record_key: &str) -> Producer +where + T: Send + Sync + 'static + Debug + Clone, +{ + let typed_rec = db + .inner() + .get_typed_record_by_key::(record_key) + .unwrap_or_else(|e| { + panic!( + "ingest factory: record '{record_key}' lookup failed ({e:?}) — \ + this is a bug in aimdb-core" + ) + }); + Producer::::new(typed_rec.writer_handle()) +} + +/// Fused ingest factory: resolves the typed producer once at route-collection +/// time; per message the returned closure runs deserialize + produce with no +/// erasure crossing. +fn plain_ingest_factory( + record_key: String, + deser: TypedContextDeserializerFn, +) -> crate::connector::IngestFactoryFn +where + T: Send + Sync + 'static + Debug + Clone, +{ + Arc::new(move |db: &AimDb| { + let producer = inbound_producer::(db, &record_key); + let deser = deser.clone(); + Arc::new(move |ctx: &crate::RuntimeContext, payload: &[u8]| { + producer.produce(deser(ctx.clone(), payload)?); + Ok(()) + }) as crate::connector::IngestFn + }) +} + +/// Like [`plain_ingest_factory`], for `with_match_deserializer`. +fn match_ingest_factory( + record_key: String, + deser: TypedMatchDeserializerFn, +) -> crate::connector::MatchIngestFactoryFn +where + T: Send + Sync + 'static + Debug + Clone, +{ + Arc::new(move |db: &AimDb| { + let producer = inbound_producer::(db, &record_key); + let deser = deser.clone(); + Arc::new( + move |ctx: &crate::RuntimeContext, m: &crate::TopicMatch<'_>, payload: &[u8]| { + producer.produce(deser(ctx, m, payload)?); + Ok(()) + }, + ) as crate::connector::MatchIngestFn + }) +} + /// Type alias for typed context-aware deserializer callbacks /// /// Stays typed until `finish()` fuses it with the producer — no per-message @@ -1028,6 +1102,14 @@ where type TypedContextDeserializerFn = Arc Result + Send + Sync + 'static>; +/// Like [`TypedContextDeserializerFn`], also receiving the topic match. +type TypedMatchDeserializerFn = Arc< + dyn Fn(&crate::RuntimeContext, &crate::TopicMatch<'_>, &[u8]) -> Result + + Send + + Sync + + 'static, +>; + /// Builder for configuring inbound connector links (External → AimDB) /// /// `'r` is the borrow of the registrar taken by `link_from()`; `'a` is the @@ -1037,6 +1119,8 @@ pub struct InboundConnectorBuilder<'r, 'a, T: Send + Sync + 'static + Debug + Cl url: String, config: Vec<(String, String)>, context_deserializer: Option>, + match_deserializer: Option>, + key: Option<(String, u16)>, topic_resolver: Option, } @@ -1064,6 +1148,28 @@ where self } + /// Like [`with_deserializer`](Self::with_deserializer), also receiving the + /// topic the message arrived on, its captures and its key. The context is + /// borrowed, so no reference count changes per message. + pub fn with_match_deserializer(mut self, f: F) -> Self + where + F: Fn(&crate::RuntimeContext, &crate::TopicMatch<'_>, &[u8]) -> Result + + Send + + Sync + + 'static, + { + self.match_deserializer = Some(Arc::new(f)); + self + } + + /// Assigns each value of capture `name` a [`KeyId`](crate::KeyId), up to + /// `capacity` values per record. Messages with a value beyond that are + /// dropped. + pub fn key(mut self, name: &str, capacity: u16) -> Self { + self.key = Some((name.to_string(), capacity)); + self + } + /// Sets the operation timeout in milliseconds (the connector interprets /// it; passed as the `timeout_ms` option — see /// `ConnectorConfig::from_query`) @@ -1108,104 +1214,106 @@ where /// /// The buffer requirement is validated by `build()` (calling `.buffer()` /// after `.link_from()` is fine). - pub fn finish(self) -> &'r mut RecordRegistrar<'a, T> { + pub fn finish(mut self) -> &'r mut RecordRegistrar<'a, T> { + match self.link() { + Ok(link) => self.registrar.rec.add_inbound_connector(link), + Err(message) => { + let key = self.registrar.record_key.clone(); + let error = crate::error::ConfigError::new(key, Some(self.url), message); + self.registrar.rec.push_config_error(error); + } + } + self.registrar + } + + /// The link `finish()` registers, or why it is rejected. + /// + /// The buffer requirement and mutual exclusion with local producers + /// (.source()/.transform()) are validated by `build()`: `.buffer()` may + /// legitimately be called after `.link_from()`. + fn link(&mut self) -> Result { use crate::connector::{InboundConnectorLink, LinkAddress}; - use crate::error::ConfigError; + let url = LinkAddress::parse(&self.url).map_err(|_| "Invalid connector URL")?; let record_key = self.registrar.record_key.clone(); - let Ok(url) = LinkAddress::parse(&self.url) else { - self.registrar.rec.push_config_error(ConfigError::new( - record_key, - Some(self.url), - "Invalid connector URL", - )); - return self.registrar; + let (ingest_factory, match_ingest_factory) = match ( + self.context_deserializer.take(), + self.match_deserializer.take(), + ) { + (Some(deser), None) => (plain_ingest_factory(record_key, deser), None), + (None, Some(deser)) => { + let factory = match_ingest_factory(record_key, deser); + let topic: Arc = url.resource_id().into(); + let inner = factory.clone(); + let plain: crate::connector::IngestFactoryFn = Arc::new(move |db: &AimDb| { + crate::connector::match_as_ingest(inner(db), topic.clone()) + }); + (plain, Some(factory)) + } + (Some(_), Some(_)) => { + return Err( + "Set either .with_deserializer() or .with_match_deserializer(), not both" + .into(), + ) + } + (None, None) => { + return Err("Inbound connector requires a deserializer. Call \ + .with_deserializer() or .with_match_deserializer()" + .into()) + } }; - let scheme = url.scheme().to_string(); - - // NOTE: the buffer requirement is validated by `build()`, not here — - // `.buffer()` may legitimately be called after `.link_from()`. - - // Mutual exclusion with local producers (.source()/.transform()) is - // validated once, in build(), where the record key is known. + // The `{…}` syntax; the connector checks the rest. + let pattern = crate::TopicPattern::parse(url.resource_id()).map_err(|e| e.to_string())?; - // Adapt the stored deserializer to the fused calling convention. Stays - // typed: fused with the producer below, no `Box` per message - //. - type UnifiedDeserializeFn = - Arc Result + Send + Sync>; - let deserialize: UnifiedDeserializeFn = if let Some(deser) = self.context_deserializer { - Arc::new(move |ctx, bytes| deser(ctx.clone(), bytes)) - } else { - self.registrar.rec.push_config_error(ConfigError::new( - record_key, - Some(self.url), - "Inbound connector requires a deserializer. Call .with_deserializer()", - )); - return self.registrar; + let key = match self.key.take() { + None => None, + Some((capture, capacity)) => { + let Some(capacity) = core::num::NonZeroU16::new(capacity) else { + return Err(alloc::format!( + "key '{capture}' needs a capacity of at least 1" + )); + }; + let has_capture = pattern.parts().iter().any( + |p| matches!(p, crate::PatternPart::Capture { name, .. } if *name == capture), + ); + if !has_capture { + return Err(alloc::format!( + "key '{capture}' is not a capture of the topic" + )); + } + let links = self.registrar.rec.inbound_connectors(); + if let Some((_, other)) = links.iter().find_map(|l| l.key.as_ref()) { + if *other != capacity { + return Err(alloc::format!( + "key '{capture}' has capacity {capacity}, another keyed link of \ + this record has {other}" + )); + } + } + Some((capture, capacity)) + } }; - // Validation: Connector builder must be registered - let has_connector = self + let scheme = url.scheme(); + if !self .registrar .connector_builders .iter() - .any(|b| b.scheme() == scheme); - - if !has_connector { - self.registrar.rec.push_config_error(ConfigError::new( - record_key, - Some(self.url), - alloc::format!( - "No connector registered for scheme '{scheme}'. Register via .with_connector()" - ), + .any(|b| b.scheme() == scheme) + { + return Err(alloc::format!( + "No connector registered for scheme '{scheme}'. Register via .with_connector()" )); - return self.registrar; } - // Fused ingest factory that captures type T and record key: resolves - // the typed producer once at route-collection time; per message the - // returned IngestFn runs deserialize + produce with no erasure - // crossing. The factory runs during build() after every record is - // registered and validated, so failures here are aimdb bugs, not - // user mistakes. - #[allow( - clippy::panic, - reason = "the factory returns no Result and these lookups were validated at build() time" - )] - let ingest_factory: crate::connector::IngestFactoryFn = { - let record_key = self.registrar.record_key.clone(); - Arc::new(move |db: &AimDb| { - let typed_rec = db - .inner() - .get_typed_record_by_key::(&record_key) - .unwrap_or_else(|e| { - panic!( - "ingest factory: record '{record_key}' lookup failed ({e:?}) — \ - this is a bug in aimdb-core" - ) - }); - let producer = Producer::::new(typed_rec.writer_handle()); - let deserialize = deserialize.clone(); - Arc::new(move |ctx: &crate::RuntimeContext, payload: &[u8]| { - producer.produce(deserialize(ctx, payload)?); - Ok(()) - }) as crate::connector::IngestFn - }) - }; - - // Create inbound connector link let mut link = InboundConnectorLink::new(url, ingest_factory); - link.config = self.config; - - // Wire through the topic resolver - link.topic_resolver = self.topic_resolver; - - // Add to record - self.registrar.rec.add_inbound_connector(link); - self.registrar + link.config = core::mem::take(&mut self.config); + link.match_ingest_factory = match_ingest_factory; + link.key = key; + link.topic_resolver = self.topic_resolver.take(); + Ok(link) } } @@ -1374,6 +1482,145 @@ mod tests { assert_eq!(errors[0].url.as_deref(), Some("mqtt://broker/topic")); } + // ==================================================================== + // Topic patterns and keys on inbound links + // ==================================================================== + + /// Runs `configure` against a fresh registrar with an `mqtt` connector + /// and returns the record's links and recorded errors. + fn register( + configure: impl FnOnce(&mut RecordRegistrar<'_, TestRecord>), + ) -> ( + Vec, + Vec, + ) { + let mut rec = crate::typed_record::TypedRecord::::new(); + rec.set_buffer(Box::new(MockBuffer)); + let builders: Vec> = + vec![Box::new(MockConnectorBuilder { + scheme: "mqtt".to_string(), + })]; + let extensions = crate::extensions::Extensions::new(); + configure(&mut make_registrar(&mut rec, &builders, &extensions)); + (rec.inbound_connectors().to_vec(), drain_errors(&mut rec)) + } + + fn match_deser( + _ctx: &crate::RuntimeContext, + m: &crate::TopicMatch<'_>, + _bytes: &[u8], + ) -> Result { + Ok(TestRecord { + value: m.topic().len() as i32, + }) + } + + #[test] + fn inbound_finish_registers_match_link_with_key() { + let (links, errors) = register(|reg| { + reg.link_from("mqtt://s/{device}/t") + .key("device", 16) + .with_match_deserializer(match_deser) + .finish(); + }); + assert!(errors.is_empty(), "{errors:?}"); + assert!(links[0].match_ingest_factory.is_some()); + let (capture, capacity) = links[0].key.clone().unwrap(); + assert_eq!((capture.as_str(), capacity.get()), ("device", 16)); + } + + #[test] + fn inbound_finish_rejects_both_deserializers() { + let (links, errors) = register(|reg| { + reg.link_from("mqtt://s/{device}/t") + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) + .with_match_deserializer(match_deser) + .finish(); + }); + assert!(links.is_empty()); + assert!(errors[0].message.contains("not both"), "{:?}", errors[0]); + } + + #[test] + fn inbound_finish_rejects_invalid_pattern_syntax() { + for topic in [ + "s/{d", + "s/{}", + "s/{d-1}", + "{d}/{d}", + "{a}/{b}/{c}/{d}/{e}/{f}/{g}/{h}/{i}", + ] { + let url = alloc::format!("mqtt://{topic}"); + let (links, errors) = register(|reg| { + reg.link_from(&url) + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) + .finish(); + }); + assert!(links.is_empty(), "{topic}"); + assert_eq!(errors.len(), 1, "{topic}"); + assert_eq!(errors[0].record_key, "test::Record"); + assert_eq!(errors[0].url.as_deref(), Some(url.as_str())); + } + } + + #[test] + fn inbound_finish_rejects_bad_keys() { + for (topic, key, capacity, needle) in [ + ("s/{d}/t", "device", 4, "not a capture"), + ("s/{d}/t", "d", 0, "at least 1"), + ] { + let (links, errors) = register(|reg| { + reg.link_from(&alloc::format!("mqtt://{topic}")) + .key(key, capacity) + .with_match_deserializer(match_deser) + .finish(); + }); + assert!(links.is_empty()); + assert!(errors[0].message.contains(needle), "{:?}", errors[0]); + } + } + + #[test] + fn keyed_links_of_one_record_share_a_capacity() { + let (links, errors) = register(|reg| { + reg.link_from("mqtt://temp/{d}") + .key("d", 8) + .with_match_deserializer(match_deser) + .finish(); + reg.link_from("mqtt://hum/{id}") + .key("id", 8) + .with_match_deserializer(match_deser) + .finish(); + reg.link_from("mqtt://co2/{id}") + .key("id", 16) + .with_match_deserializer(match_deser) + .finish(); + }); + assert_eq!(links.len(), 2); + assert_eq!(errors.len(), 1); + assert!(errors[0].message.contains("has 8"), "{:?}", errors[0]); + } + + #[test] + fn outbound_finish_rejects_patterns() { + let mut rec = crate::typed_record::TypedRecord::::new(); + rec.set_buffer(Box::new(MockBuffer)); + let builders: Vec> = + vec![Box::new(MockConnectorBuilder { + scheme: "mqtt".to_string(), + })]; + let extensions = crate::extensions::Extensions::new(); + let mut reg = make_registrar(&mut rec, &builders, &extensions); + + reg.link_to("mqtt://out/{device}") + .with_serializer(|_ctx, r: &TestRecord| Ok(r.value.to_le_bytes().to_vec())) + .finish(); + + assert!(rec.outbound_connectors().is_empty()); + let errors = drain_errors(&mut rec); + assert!(errors[0].message.contains("cannot use topic patterns")); + } + // ==================================================================== // Outbound link registration tests (fused source) // ==================================================================== @@ -1880,6 +2127,39 @@ mod tests { assert_eq!(last.load(Ordering::SeqCst), 7); } + /// The pre-pattern route API skips `{…}` links and gives a literal-topic + /// match link its topic. + #[tokio::test] + async fn collect_inbound_routes_skips_patterns_and_passes_literal_topics() { + let last = Arc::new(AtomicI32::new(-1)); + let count = Arc::new(AtomicUsize::new(0)); + let (buf_last, buf_count) = (last.clone(), count.clone()); + + let mut builder = crate::AimDbBuilder::new() + .runtime(Arc::new(MockRuntime)) + .with_connector(NoopConnectorBuilder); + builder.configure::("rec.in", move |reg| { + reg.buffer_raw(Box::new(RecordingBuffer { + last: buf_last, + count: buf_count, + })); + reg.link_from("mqtt://s/{d}/t") + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) + .finish(); + reg.link_from("mqtt://cmd/in") + .with_match_deserializer(match_deser) + .finish(); + }); + let (db, _runner) = builder.build().await.expect("build must succeed"); + + let routes = db.collect_inbound_routes("mqtt"); + assert_eq!(routes.len(), 1); + let (topic, ingest) = &routes[0]; + assert_eq!(topic, "cmd/in"); + ingest(&db.runtime_ctx(), b"x").expect("ingest must succeed"); + assert_eq!(last.load(Ordering::SeqCst), "cmd/in".len() as i32); + } + // ==================================================================== // Fused outbound reader tests // ==================================================================== diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 8eff3b72..c8194218 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -314,10 +314,10 @@ impl<'a> TopicMatch<'a> { plain `with_deserializer` gets one that ignores the match. A **literal** topic with `with_match_deserializer` becomes an all-literal pattern route so the closure still receives the topic. -- The closure takes `RuntimeContext` by value, like `with_deserializer`. - That clones an `Arc` per message (an atomic increment, no allocation). - Passing `&RuntimeContext` would be cheaper but is a change for both - builders; it belongs in 054's breaking window. +- The closure borrows the context (`&RuntimeContext`), so no reference + count changes per message. `with_deserializer` takes it by value and + clones an `Arc` per message; moving it to a borrow is a breaking change + for a later release. ### 5.6 Keys @@ -436,8 +436,8 @@ for the whole ingest call, exactly as `Router::route` does today. Because unchanged. - `InboundDispatch::new` takes a `&'static dyn TopicGrammar` and replaces `inbound_router` + `pump_source_with` for migrated connectors. -- 054's breaking window is where `&RuntimeContext` can replace the by-value - context in both deserializer builders (§5.5). +- 054 does not change `with_deserializer`; its by-value context stays until + a later breaking release (§5.5). ## 8. Alternatives considered From 2ae708f40a788fa73e4722d08fab77fc55c1077d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 01:27:03 +0000 Subject: [PATCH 07/18] feat(core): implement inbound key tables and enhance router functionality --- aimdb-core/src/builder.rs | 99 ++++++++++++- aimdb-core/src/connector.rs | 5 + aimdb-core/src/inbound_key.rs | 3 + aimdb-core/src/lib.rs | 8 +- aimdb-core/src/remote/metadata.rs | 54 ++++++- aimdb-core/src/remote/mod.rs | 2 +- aimdb-core/src/router.rs | 87 +----------- aimdb-core/src/session/mod.rs | 2 +- aimdb-core/src/session/pump.rs | 16 ++- aimdb-core/src/topic_pattern.rs | 87 ++++++++++++ aimdb-core/src/typed_api.rs | 166 +++++++++++++++++++++- docs/design/055-wildcard-inbound-links.md | 14 +- 12 files changed, 436 insertions(+), 107 deletions(-) diff --git a/aimdb-core/src/builder.rs b/aimdb-core/src/builder.rs index 5c7bc48b..bd9857c1 100644 --- a/aimdb-core/src/builder.rs +++ b/aimdb-core/src/builder.rs @@ -52,6 +52,7 @@ struct RecordEntry { key: StringKey, type_id: TypeId, record: Box, + key_table: Option>, } /// Internal database state @@ -197,7 +198,19 @@ impl AimDbInner { .enumerate() .map(|(i, e)| { let id = RecordId::new(i as u32); - e.record.collect_metadata(e.type_id, e.key, id) + let mut metadata = e.record.collect_metadata(e.type_id, e.key, id); + metadata.inbound_keys = e.key_table.as_ref().map(|table| { + let captures = e.record.inbound_connectors().iter(); + crate::remote::InboundKeysInfo { + captures: captures + .filter_map(|l| l.key.as_ref().map(|(capture, _)| capture.clone())) + .collect(), + capacity: table.capacity(), + assigned: table.assigned(), + dropped: table.dropped(), + } + }); + metadata }) .collect() } @@ -677,10 +690,16 @@ impl AimDbBuilder { } let id = RecordId::new(storages.len() as u32); + // `finish()` checked that keyed links of a record agree on it. + let key_capacity = record + .inbound_connectors() + .iter() + .find_map(|l| l.key.as_ref().map(|(_, capacity)| *capacity)); storages.push(RecordEntry { key, type_id, record, + key_table: key_capacity.map(|c| Arc::new(crate::inbound_key::KeyTable::new(c))), }); by_key.insert(key, id); } @@ -1223,6 +1242,84 @@ impl AimDb { routes } + /// The inbound router for `scheme`: every link compiled against the + /// connector's `grammar`, keyed links sharing their record's key table. + /// + /// A connector subscribes [`Router::subscriptions`](crate::Router::subscriptions) + /// and routes with this same router. Rejects every link the grammar or + /// its key cannot compile, naming the record and the resolved topic. + pub fn inbound_router( + &self, + scheme: &str, + grammar: &'static dyn crate::TopicGrammar, + ) -> DbResult { + let mut routes = Vec::new(); + let mut errors = Vec::new(); + + for entry in &self.inner.storages { + for link in entry.record.inbound_connectors() { + if link.url.scheme() != scheme { + continue; + } + match self.inbound_route(entry, link, grammar) { + Ok(route) => routes.push(route), + Err(message) => errors.push(crate::ConfigError::new( + entry.key.as_str(), + Some(link.url.to_string()), + message, + )), + } + } + } + + if !errors.is_empty() { + return Err(DbError::InvalidConfiguration { errors }); + } + Ok(crate::Router::compiled(grammar, routes)) + } + + fn inbound_route( + &self, + entry: &RecordEntry, + link: &crate::connector::InboundConnectorLink, + grammar: &'static dyn crate::TopicGrammar, + ) -> Result { + let topic = link.resolve_topic(); + let pattern = crate::TopicPattern::parse(&topic).map_err(|e| e.to_string())?; + let filter = grammar.compile(&pattern)?; + let names: Box<[Box]> = pattern.capture_names().map(Box::from).collect(); + + let key = match (&link.key, &entry.key_table) { + (Some((capture, _)), Some(table)) => { + let capture_number = + names.iter().position(|n| **n == **capture).ok_or_else(|| { + alloc::format!("key '{capture}' is not a capture of '{topic}'") + })?; + Some((table.clone(), capture_number)) + } + _ => None, + }; + + let ingest = match &link.match_ingest_factory { + Some(factory) => factory(self), + None => crate::connector::ignore_match(link.create_ingest(self)), + }; + Ok(crate::router::CompiledRoute::pattern( + filter, names, key, ingest, + )) + } + + /// The capture value `key` stands for on record `record_key`. + pub fn inbound_key_name(&self, record_key: &str, key: crate::KeyId) -> Option> { + let id = self.inner.by_key.get(record_key)?; + self.inner + .storages + .get(id.index())? + .key_table + .as_ref()? + .name(key) + } + /// Collects outbound routes for a specific protocol scheme /// /// Mirrors `collect_inbound_routes()` for symmetry. Iterates all records, diff --git a/aimdb-core/src/connector.rs b/aimdb-core/src/connector.rs index ef02e454..323ff9ff 100644 --- a/aimdb-core/src/connector.rs +++ b/aimdb-core/src/connector.rs @@ -535,6 +535,11 @@ pub type IngestFactoryFn = Arc IngestFn + Send + Sync>; /// Like [`IngestFactoryFn`], for links set with `with_match_deserializer`. pub type MatchIngestFactoryFn = Arc MatchIngestFn + Send + Sync>; +/// Runs a plain ingest where a [`MatchIngestFn`] is expected. +pub(crate) fn ignore_match(ingest: IngestFn) -> MatchIngestFn { + Arc::new(move |ctx, _m, payload| ingest(ctx, payload)) +} + /// Runs a match-aware ingest where only an [`IngestFn`] fits: every message /// is on `topic`, with no captures and no key. pub(crate) fn match_as_ingest(ingest: MatchIngestFn, topic: Arc) -> IngestFn { diff --git a/aimdb-core/src/inbound_key.rs b/aimdb-core/src/inbound_key.rs index 4baf5204..8304b528 100644 --- a/aimdb-core/src/inbound_key.rs +++ b/aimdb-core/src/inbound_key.rs @@ -83,15 +83,18 @@ impl KeyTable { lock(&self.keys).names.get(key.index()).cloned() } + #[cfg(any(test, feature = "remote"))] pub(crate) fn capacity(&self) -> u16 { self.capacity.get() } + #[cfg(any(test, feature = "remote"))] pub(crate) fn assigned(&self) -> usize { lock(&self.keys).names.len() } /// Messages turned away because the table was full. + #[cfg(any(test, feature = "remote"))] pub(crate) fn dropped(&self) -> u32 { self.dropped.load(Ordering::Relaxed) } diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 37484ffd..17924cc5 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -85,8 +85,6 @@ mod error; pub mod executor; pub mod extensions; pub mod graph; -// Used by the router once pattern routes land. -#[allow(dead_code)] mod inbound_key; #[cfg(feature = "observability")] pub mod profiling; @@ -142,9 +140,9 @@ pub use remote::topic_leaf; // compatible). See docs/design/remote-access-via-connectors.md. #[cfg(feature = "connector-session")] pub use session::{ - is_wildcard, pattern_contains, pump_sink, pump_source, topic_matches, AuthError, BoxFut, - BoxStream, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, - Outbound, Payload, PeerInfo, RpcError, SessionCtx, SessionLimits, Source, SubUpdate, + is_wildcard, pattern_contains, pump_sink, pump_source, pump_source_with, topic_matches, + AuthError, BoxFut, BoxStream, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, + Listener, Outbound, Payload, PeerInfo, RpcError, SessionCtx, SessionLimits, Source, SubUpdate, TransportError, TransportResult, }; diff --git a/aimdb-core/src/remote/metadata.rs b/aimdb-core/src/remote/metadata.rs index 71fe8cbd..92625815 100644 --- a/aimdb-core/src/remote/metadata.rs +++ b/aimdb-core/src/remote/metadata.rs @@ -8,7 +8,6 @@ use alloc::format; use alloc::string::{String, ToString}; -#[cfg(feature = "observability")] use alloc::vec::Vec; use core::any::TypeId; use serde::{Deserialize, Serialize}; @@ -34,6 +33,7 @@ use crate::record_id::{RecordId, RecordKey}; /// server whose clients need that distinction must be built with /// `observability` on; without it the honest client answer is "unknown". #[derive(Debug, Clone, Serialize, Deserialize)] +#[non_exhaustive] pub struct RecordMetadata { /// Unique record identifier (index in the storage) pub record_id: u32, @@ -85,6 +85,10 @@ pub struct RecordMetadata { #[serde(default, skip_serializing_if = "Option::is_none")] pub entity: Option, + /// The record's key table, when it has keyed inbound links. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inbound_keys: Option, + // ===== Buffer metrics (feature-gated) ===== /// Total items pushed to the buffer (metrics feature only). /// @@ -125,6 +129,19 @@ pub struct RecordMetadata { pub signal_stats: Option>, } +/// A record's key table. Key names are not listed; `inbound_key_name` +/// resolves one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct InboundKeysInfo { + /// The keyed capture of each keyed link. + pub captures: Vec, + pub capacity: u16, + pub assigned: usize, + /// Messages turned away because the table was full. + pub dropped: u32, +} + impl RecordMetadata { /// Creates a new record metadata entry /// @@ -169,6 +186,7 @@ impl RecordMetadata { outbound_connector_count, schema_type: None, entity, + inbound_keys: None, #[cfg(feature = "observability")] produced_count: None, #[cfg(feature = "observability")] @@ -281,5 +299,39 @@ mod tests { assert!(json.contains("\"buffer_type\":\"single_latest\"")); assert!(json.contains("\"writable\":true")); assert!(json.contains("\"outbound_connector_count\":2")); + assert!(!json.contains("inbound_keys")); + } + + #[test] + fn inbound_keys_are_optional_on_the_wire() { + let mut metadata = RecordMetadata::new( + RecordId::new(0), + StringKey::new("sensors.readings"), + TypeId::of::(), + "i32".to_string(), + RecordOrigin::Passive, + "spmc_ring".to_string(), + Some(256), + 0, + 0, + false, + 0, + ); + + // An older server's reply has no `inbound_keys`. + let json = serde_json::to_string(&metadata).unwrap(); + let old: RecordMetadata = serde_json::from_str(&json).unwrap(); + assert_eq!(old.inbound_keys, None); + + let keys = InboundKeysInfo { + captures: alloc::vec!["device".to_string()], + capacity: 1024, + assigned: 3, + dropped: 1, + }; + metadata.inbound_keys = Some(keys.clone()); + let json = serde_json::to_string(&metadata).unwrap(); + let parsed: RecordMetadata = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.inbound_keys, Some(keys)); } } diff --git a/aimdb-core/src/remote/mod.rs b/aimdb-core/src/remote/mod.rs index 93aa03b7..4dec6303 100644 --- a/aimdb-core/src/remote/mod.rs +++ b/aimdb-core/src/remote/mod.rs @@ -51,7 +51,7 @@ mod query; pub use config::{AimxConfig, SecurityPolicy}; pub use error::{RemoteError, RemoteResult}; -pub use metadata::RecordMetadata; +pub use metadata::{InboundKeysInfo, RecordMetadata}; pub use protocol::{ version_compatible, ws_url_with_version, ErrorObject, Event, HelloMessage, Request, Response, WelcomeMessage, PROTOCOL_VERSION, VERSION_PARAM, diff --git a/aimdb-core/src/router.rs b/aimdb-core/src/router.rs index b1351195..9fd80609 100644 --- a/aimdb-core/src/router.rs +++ b/aimdb-core/src/router.rs @@ -92,12 +92,11 @@ impl CompiledRoute { matcher: None, names: Box::new([]), key: None, - ingest: Arc::new(move |ctx, _m, payload| ingest(ctx, payload)), + ingest: crate::connector::ignore_match(ingest), } } /// A route matching through `filter`. - #[allow(dead_code)] pub(crate) fn pattern( filter: Box, names: Box<[Box]>, @@ -143,7 +142,6 @@ impl Router { } } - #[allow(dead_code)] pub(crate) fn compiled(grammar: &'static dyn TopicGrammar, routes: Vec) -> Self { Self { routes, grammar } } @@ -479,82 +477,10 @@ mod tests { // ---- pattern routes --------------------------------------------------- - use crate::topic_pattern::{PatternPart, TopicPattern}; + use crate::topic_pattern::{test_support::Plus, TopicPattern}; use core::num::NonZeroU16; use std::sync::Mutex; - /// `/`-separated levels; `+` and `{name}` match one level. - struct Plus; - - struct PlusFilter { - filter: String, - /// Per level: `None` for a wildcard. - levels: Vec>, - /// Per level: capture number. - captures: Vec>, - } - - impl TopicGrammar for Plus { - fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String> { - let mut filter = String::new(); - let mut captures = Vec::new(); - for part in pattern.parts() { - match part { - PatternPart::Text(t) => filter.push_str(t), - PatternPart::Capture { .. } => { - filter.push('+'); - captures.push(filter.split('/').count() - 1); - } - } - } - let levels: Vec> = filter - .split('/') - .map(|l| (l != "+").then(|| l.to_string())) - .collect(); - let captures = (0..levels.len()) - .map(|i| captures.iter().position(|&c| c == i)) - .collect(); - Ok(Box::new(PlusFilter { - filter, - levels, - captures, - })) - } - - fn covers(&self, a: &str, b: &str) -> bool { - a.split('/').count() == b.split('/').count() - && a.split('/') - .zip(b.split('/')) - .all(|(x, y)| x == "+" || x == y) - } - } - - impl TopicFilter for PlusFilter { - fn filter(&self) -> &str { - &self.filter - } - fn is_literal(&self) -> bool { - self.levels.iter().all(Option::is_some) - } - fn matches(&self, topic: &str, spans: &mut Spans) -> bool { - if topic.split('/').count() != self.levels.len() { - return false; - } - let mut start = 0; - for (i, level) in topic.split('/').enumerate() { - match &self.levels[i] { - Some(lit) if lit != level => return false, - _ => {} - } - if let Some(c) = self.captures[i] { - spans[c] = (start as u16, (start + level.len()) as u16); - } - start += level.len() + 1; - } - true - } - } - type Seen = Arc>, Option)>>>; /// Records the topic, the named captures and the key index. @@ -575,14 +501,7 @@ mod tests { key: Option<(Arc, usize)>, ) -> CompiledRoute { let pattern = TopicPattern::parse(topic).unwrap(); - let names = pattern - .parts() - .iter() - .filter_map(|p| match p { - PatternPart::Capture { name, .. } => Some(Box::from(*name)), - PatternPart::Text(_) => None, - }) - .collect(); + let names = pattern.capture_names().map(Box::from).collect(); CompiledRoute::pattern(Plus.compile(&pattern).unwrap(), names, key, ingest) } diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 491436ad..663d543e 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -54,7 +54,7 @@ pub use io::{ OneShotListener, StreamDialer, StreamListener, }; #[cfg(feature = "connector-session")] -pub use pump::{pump_sink, pump_source}; +pub use pump::{pump_sink, pump_source, pump_source_with}; #[cfg(feature = "connector-session")] pub use server::{run_session, serve, SessionConfig}; diff --git a/aimdb-core/src/session/pump.rs b/aimdb-core/src/session/pump.rs index 54c710f7..0bc3fe6f 100644 --- a/aimdb-core/src/session/pump.rs +++ b/aimdb-core/src/session/pump.rs @@ -21,7 +21,7 @@ use alloc::vec::Vec; use super::Source; use crate::builder::{AimDb, BoxFuture}; -use crate::router::RouterBuilder; +use crate::router::{Router, RouterBuilder}; use crate::transport::{Connector, ConnectorConfig}; /// Outbound pump: one publisher future per outbound route on `scheme`. @@ -138,9 +138,19 @@ pub fn pump_sink(db: &AimDb, scheme: &str, sink: Arc) -> Vec Vec { +pub fn pump_source(db: &AimDb, scheme: &str, src: impl Source + 'static) -> Vec { let routes = db.collect_inbound_routes(scheme); - let router = Arc::new(RouterBuilder::from_routes(routes).build()); + pump_source_with(db, RouterBuilder::from_routes(routes).build(), src) +} + +/// Like [`pump_source`], routing with `router`, typically from +/// [`AimDb::inbound_router`], whose subscriptions the connector made. +pub fn pump_source_with( + db: &AimDb, + router: Router, + mut src: impl Source + 'static, +) -> Vec { + let router = Arc::new(router); let ctx = db.runtime_ctx(); vec![Box::pin(async move { diff --git a/aimdb-core/src/topic_pattern.rs b/aimdb-core/src/topic_pattern.rs index 96cae6c6..7f7610b6 100644 --- a/aimdb-core/src/topic_pattern.rs +++ b/aimdb-core/src/topic_pattern.rs @@ -101,6 +101,14 @@ impl<'a> TopicPattern<'a> { self.topic } + /// Capture names in capture-number order. + pub(crate) fn capture_names(&self) -> impl Iterator + '_ { + self.parts.iter().filter_map(|p| match p { + PatternPart::Capture { name, .. } => Some(*name), + PatternPart::Text(_) => None, + }) + } + /// Text and captures; captures are numbered in this order. pub fn parts(&self) -> &[PatternPart<'a>] { &self.parts @@ -208,6 +216,85 @@ impl TopicFilter for ExactFilter { } } +/// A stub grammar for tests of the router and of `inbound_router`. +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use alloc::string::ToString; + + /// `/`-separated levels; `+` and `{name}` match one level. + pub(crate) struct Plus; + + pub(crate) struct PlusFilter { + filter: String, + /// Per level: `None` for a wildcard. + levels: Vec>, + /// Per level: capture number. + captures: Vec>, + } + + impl TopicGrammar for Plus { + fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String> { + let mut filter = String::new(); + let mut captures = Vec::new(); + for part in pattern.parts() { + match part { + PatternPart::Text(t) => filter.push_str(t), + PatternPart::Capture { .. } => { + filter.push('+'); + captures.push(filter.split('/').count() - 1); + } + } + } + let levels: Vec> = filter + .split('/') + .map(|l| (l != "+").then(|| l.to_string())) + .collect(); + let captures = (0..levels.len()) + .map(|i| captures.iter().position(|&c| c == i)) + .collect(); + Ok(Box::new(PlusFilter { + filter, + levels, + captures, + })) + } + + fn covers(&self, a: &str, b: &str) -> bool { + a.split('/').count() == b.split('/').count() + && a.split('/') + .zip(b.split('/')) + .all(|(x, y)| x == "+" || x == y) + } + } + + impl TopicFilter for PlusFilter { + fn filter(&self) -> &str { + &self.filter + } + fn is_literal(&self) -> bool { + self.levels.iter().all(Option::is_some) + } + fn matches(&self, topic: &str, spans: &mut Spans) -> bool { + if topic.split('/').count() != self.levels.len() { + return false; + } + let mut start = 0; + for (i, level) in topic.split('/').enumerate() { + match &self.levels[i] { + Some(lit) if lit != level => return false, + _ => {} + } + if let Some(c) = self.captures[i] { + spans[c] = (start as u16, (start + level.len()) as u16); + } + start += level.len() + 1; + } + true + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/aimdb-core/src/typed_api.rs b/aimdb-core/src/typed_api.rs index f3e8d71e..c7d64991 100644 --- a/aimdb-core/src/typed_api.rs +++ b/aimdb-core/src/typed_api.rs @@ -1275,10 +1275,7 @@ where "key '{capture}' needs a capacity of at least 1" )); }; - let has_capture = pattern.parts().iter().any( - |p| matches!(p, crate::PatternPart::Capture { name, .. } if *name == capture), - ); - if !has_capture { + if !pattern.capture_names().any(|n| n == capture) { return Err(alloc::format!( "key '{capture}' is not a capture of the topic" )); @@ -2160,6 +2157,167 @@ mod tests { assert_eq!(last.load(Ordering::SeqCst), "cmd/in".len() as i32); } + // ==================================================================== + // inbound_router: patterns, keys and connector-build errors + // ==================================================================== + + use crate::topic_pattern::test_support::Plus; + + /// A record `rec.in` whose buffer stores the last value and a count. + async fn inbound_db( + links: impl FnOnce(&mut RecordRegistrar<'_, TestRecord>) + Send + 'static, + ) -> (crate::AimDb, Arc, Arc) { + let last = Arc::new(AtomicI32::new(-1)); + let count = Arc::new(AtomicUsize::new(0)); + let (buf_last, buf_count) = (last.clone(), count.clone()); + let mut builder = crate::AimDbBuilder::new() + .runtime(Arc::new(MockRuntime)) + .with_connector(NoopConnectorBuilder); + builder.configure::("rec.in", move |reg| { + reg.buffer_raw(Box::new(RecordingBuffer { + last: buf_last, + count: buf_count, + })); + links(reg); + }); + let (db, _runner) = builder.build().await.expect("build must succeed"); + (db, last, count) + } + + /// The key index, or -1 without a key. + fn key_deser( + _ctx: &crate::RuntimeContext, + m: &crate::TopicMatch<'_>, + _bytes: &[u8], + ) -> Result { + Ok(TestRecord { + value: m.key().map_or(-1, |k| k.index() as i32), + }) + } + + #[tokio::test] + async fn inbound_router_routes_patterns_with_shared_keys() { + let keys: Arc>> = Default::default(); + let seen = keys.clone(); + let (db, last, count) = inbound_db(move |reg| { + reg.link_from("mqtt://temp/{d}") + .key("d", 2) + .with_match_deserializer(move |ctx, m, bytes| { + seen.lock().extend(m.key()); + key_deser(ctx, m, bytes) + }) + .finish(); + reg.link_from("mqtt://hum/{id}") + .key("id", 2) + .with_match_deserializer(key_deser) + .finish(); + reg.link_from("mqtt://cmd/in") + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 100 })) + .finish(); + }) + .await; + + let router = db.inbound_router("mqtt", &Plus).expect("routes compile"); + let subscriptions: Vec = router + .subscriptions() + .iter() + .map(|s| s.to_string()) + .collect(); + assert_eq!(subscriptions, ["cmd/in", "hum/+", "temp/+"]); + + let ctx = db.runtime_ctx(); + let route = |topic: &str| { + router.route(topic, b"", &ctx).unwrap(); + last.load(Ordering::SeqCst) + }; + assert_eq!(route("temp/a"), 0); + assert_eq!(route("hum/b"), 1); + assert_eq!(route("hum/a"), 0, "one key table per record"); + assert_eq!(route("cmd/in"), 100); + let produced = count.load(Ordering::SeqCst); + route("temp/c"); + assert_eq!(count.load(Ordering::SeqCst), produced, "full table drops"); + + let a = keys.lock()[0]; + assert_eq!(db.inbound_key_name("rec.in", a).as_deref(), Some("a")); + assert_eq!(db.inbound_key_name("other", a), None); + + #[cfg(feature = "remote")] + { + let records = db.list_records(); + let info = records[0].inbound_keys.as_ref().expect("keyed record"); + assert_eq!(info.captures, ["d", "id"]); + assert_eq!((info.capacity, info.assigned, info.dropped), (2, 2, 1)); + } + } + + fn config_errors(result: crate::DbResult) -> Vec { + match result { + Err(crate::DbError::InvalidConfiguration { errors }) => errors, + Err(e) => panic!("unexpected error {e:?}"), + Ok(_) => panic!("expected configuration errors"), + } + } + + #[tokio::test] + async fn inbound_router_rejects_links_it_cannot_compile() { + let (db, _, _) = inbound_db(|reg| { + reg.link_from("mqtt://s/{d}/t") + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) + .finish(); + reg.link_from("mqtt://r/one") + .with_topic_resolver(|| Some("r/{d".into())) + .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) + .finish(); + reg.link_from("mqtt://k/{d}") + .key("d", 4) + .with_topic_resolver(|| Some("k/{x}".into())) + .with_match_deserializer(key_deser) + .finish(); + }) + .await; + + let errors = config_errors(db.inbound_router("mqtt", &Plus)); + assert_eq!(errors.len(), 2, "{errors:?}"); + assert!(errors.iter().all(|e| e.record_key == "rec.in")); + assert!(errors[0].message.contains("unbalanced '{' in 'r/{d'")); + assert!(errors[1] + .message + .contains("key 'd' is not a capture of 'k/{x}'")); + + let errors = config_errors(db.inbound_router("mqtt", &crate::ExactGrammar)); + assert_eq!(errors.len(), 3, "{errors:?}"); + assert!(errors[0] + .message + .contains("does not support topic patterns")); + } + + #[cfg(feature = "connector-session")] + #[tokio::test] + async fn pump_source_with_routes_through_the_given_router() { + struct Once(Option<(String, crate::Payload)>); + impl crate::Source for Once { + fn next(&mut self) -> crate::BoxFut<'_, Option<(String, crate::Payload)>> { + let next = self.0.take(); + Box::pin(async move { next }) + } + } + + let (db, last, _) = inbound_db(|reg| { + reg.link_from("mqtt://temp/{d}") + .key("d", 2) + .with_match_deserializer(key_deser) + .finish(); + }) + .await; + let router = db.inbound_router("mqtt", &Plus).unwrap(); + let source = Once(Some(("temp/a".into(), Arc::from(&b"x"[..])))); + for pump in crate::pump_source_with(&db, router, source) { + pump.await; + } + assert_eq!(last.load(Ordering::SeqCst), 0); + } + // ==================================================================== // Fused outbound reader tests // ==================================================================== diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index c8194218..a4a79cbc 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -230,9 +230,9 @@ impl Router { pub fn subscriptions(&self) -> Vec>; } -pub fn pump_source_with(db: &AimDb, scheme: &str, src: impl Source + 'static, - grammar: &'static dyn TopicGrammar) - -> DbResult>; +/// Routes `src` with the router the connector subscribed from. +pub fn pump_source_with(db: &AimDb, router: Router, src: impl Source + 'static) + -> Vec; // pump_source(..) keeps its signature and behaviour. ``` @@ -265,8 +265,8 @@ Errors name the record key and URL: capacity of 0, or a capacity that differs from another keyed link on the same record (§5.6). -**At connector build** (`inbound_router` / `pump_source_with`, returning -`DbResult`): everything that needs the grammar: +**At connector build** (`inbound_router`, returning `DbResult`): +everything that needs the grammar: - whatever `TopicGrammar::compile` rejects. For MQTT: a capture sharing a level with text (`sensors/dev-{id}`), a multi-level capture or `#` that @@ -375,8 +375,8 @@ the connector, so it is uncontended. wildcard does not match a `$…` topic, and a wildcard must be a whole level. - Both backends subscribe `subscriptions()` of - `db.inbound_router("mqtt", &MqttGrammar)` and route with - `pump_source_with(.., &MqttGrammar)`. + `db.inbound_router("mqtt", &MqttGrammar)` and pass that router to + `pump_source_with`. - **Covering set.** A filter is left out when another matches every topic it matches (`sensors/kitchen/temp` under `sensors/+/temp`; `a/+/b` under `a/#`). Required for backend parity (§3.1). The router still fans each From e3ae29002e7f62aa8e90ba44d044aab7821f9f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 01:37:00 +0000 Subject: [PATCH 08/18] feat(core): refactor client pump functions to use router for inbound handling --- aimdb-core/src/session/client.rs | 18 +++++++++++++++--- aimdb-core/src/session/connector.rs | 5 +++-- aimdb-core/src/session/mod.rs | 2 +- aimdb-knx-connector/src/connector.rs | 6 +++--- .../src/client/builder.rs | 7 ++++--- .../src/server/builder.rs | 8 +++----- 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index ff16b0d8..b926ef28 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -27,7 +27,7 @@ use super::{ BoxFut, BoxStream, Connection, Dialer, EnvelopeCodec, Inbound, Outbound, Payload, RpcError, SubUpdate, TransportError, }; -use crate::router::RouterBuilder; +use crate::router::{Router, RouterBuilder}; use crate::AimDb; /// Capacity of a subscription's client-side event sink. Bounded (was @@ -857,6 +857,18 @@ where /// Reconnect caveat: inbound pumps subscribe once and are not replayed across a /// reconnect (see [`ClientConfig::reconnect`]); outbound mirroring is unaffected. pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec> { + let router = RouterBuilder::from_routes(db.collect_inbound_routes(scheme)).build(); + pump_client_with(db, scheme, router, handle) +} + +/// Like [`pump_client`], mirroring inbound through `router`, typically from +/// [`AimDb::inbound_router`]. +pub fn pump_client_with( + db: &AimDb, + scheme: &str, + router: Router, + handle: &ClientHandle, +) -> Vec> { // The runtime context for context-aware (de)serializers. let ctx = db.runtime_ctx(); let mut pumps: Vec> = Vec::new(); @@ -897,8 +909,8 @@ pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec local producer (via the Router) --------- // The Router applies each route's deserializer and produces the value; one // subscription per unique remote topic feeds it. - let router = Arc::new(RouterBuilder::from_routes(db.collect_inbound_routes(scheme)).build()); - for id in router.resource_ids() { + let router = Arc::new(router); + for id in router.subscriptions() { pumps.push(Box::pin(inbound_pump( handle.clone(), router.clone(), diff --git a/aimdb-core/src/session/connector.rs b/aimdb-core/src/session/connector.rs index febdd205..1eadc9a3 100644 --- a/aimdb-core/src/session/connector.rs +++ b/aimdb-core/src/session/connector.rs @@ -28,7 +28,7 @@ use core::pin::Pin; use crate::builder::AimDb; use crate::connector::ConnectorBuilder; use crate::session::{ - pump_client, run_client, serve, ClientConfig, Dialer, Dispatch, EnvelopeCodec, Listener, + pump_client_with, run_client, serve, ClientConfig, Dialer, Dispatch, EnvelopeCodec, Listener, OneShot, SessionConfig, }; use crate::{DbError, DbResult}; @@ -103,7 +103,8 @@ where ); // One pump future per route; each holds a `ClientHandle` clone, so the // engine stays alive as long as any mirror runs. `handle` drops here. - let mut futures = pump_client(db, &self.scheme, &handle); + let router = db.inbound_router(&self.scheme, &crate::ExactGrammar)?; + let mut futures = pump_client_with(db, &self.scheme, router, &handle); futures.push(engine_fut); Ok(futures) }) diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 663d543e..2591ae0f 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -42,7 +42,7 @@ mod topic_match; pub use topic_match::{is_wildcard, pattern_contains, topic_matches}; #[cfg(feature = "connector-session")] -pub use client::{pump_client, run_client, ClientConfig, ClientHandle}; +pub use client::{pump_client, pump_client_with, run_client, ClientConfig, ClientHandle}; #[cfg(feature = "connector-session")] pub use connector::{SessionClientConnector, SessionServerConnector}; #[cfg(feature = "connector-session")] diff --git a/aimdb-knx-connector/src/connector.rs b/aimdb-knx-connector/src/connector.rs index df9df0bf..672866f0 100644 --- a/aimdb-knx-connector/src/connector.rs +++ b/aimdb-knx-connector/src/connector.rs @@ -16,7 +16,7 @@ use core::net::SocketAddr; use core::pin::Pin; use aimdb_core::connector::{ConnectorBuilder, ConnectorUrl}; -use aimdb_core::session::{pump_sink, pump_source, Payload}; +use aimdb_core::session::{pump_sink, pump_source_with, Payload}; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; use aimdb_core::{log_info, AimDb, DbError, DbResult, RuntimeOps}; @@ -178,9 +178,9 @@ where )); let mut futures: Vec = vec![task]; - futures.extend(pump_source( + futures.extend(pump_source_with( db, - "knx", + db.inbound_router("knx", &aimdb_core::ExactGrammar)?, KnxSource:: { telegrams: &channels.telegrams, }, diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index c1834221..9f9fe34a 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -19,8 +19,8 @@ use std::pin::Pin; -use aimdb_core::session::{aimx::AimxCodec, pump_client, run_client, ClientConfig}; -use aimdb_core::ConnectorBuilder; +use aimdb_core::session::{aimx::AimxCodec, pump_client_with, run_client, ClientConfig}; +use aimdb_core::{ConnectorBuilder, ExactGrammar}; use crate::transport::WsDialer; @@ -171,7 +171,8 @@ impl ConnectorBuilder for WsClientConnectorBuilder { config, db.runtime_ops(), ); - let mut futures = pump_client(db, "ws-client", &handle); + let router = db.inbound_router("ws-client", &ExactGrammar)?; + let mut futures = pump_client_with(db, "ws-client", router, &handle); futures.push(engine_fut); Ok(futures) }) diff --git a/aimdb-websocket-connector/src/server/builder.rs b/aimdb-websocket-connector/src/server/builder.rs index d8149ee7..59205b5d 100644 --- a/aimdb-websocket-connector/src/server/builder.rs +++ b/aimdb-websocket-connector/src/server/builder.rs @@ -24,7 +24,7 @@ use std::{ use aimdb_data_contracts::Streamable; -use aimdb_core::{pump_sink, router::RouterBuilder, ConnectorBuilder, Dispatch}; +use aimdb_core::{pump_sink, ConnectorBuilder, Dispatch, ExactGrammar}; use axum::Router as AxumRouter; use aimdb_core::topic_matches; @@ -277,16 +277,14 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { { Box::pin(async move { // ── Inbound routes ────────────────────────────────────── - let inbound_routes = db.collect_inbound_routes("ws"); + let router = Arc::new(db.inbound_router("ws", &ExactGrammar)?); #[cfg(feature = "tracing")] tracing::info!( "WS connector: {} inbound routes collected", - inbound_routes.len() + router.route_count() ); - let router = Arc::new(RouterBuilder::from_routes(inbound_routes).build()); - // ── Late-join snapshot cache (only when enabled) ────── let snapshot_map: Option = self.late_join.then(|| Arc::new(Mutex::new(HashMap::new()))); From 46361b5418d0f9bc1fa6076f879f77d2b75c4380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 01:41:12 +0000 Subject: [PATCH 09/18] refactor(core)!: pump_client takes the inbound router --- aimdb-core/src/session/client.rs | 15 ++++----------- aimdb-core/src/session/connector.rs | 4 ++-- aimdb-core/src/session/mod.rs | 2 +- aimdb-websocket-connector/src/client/builder.rs | 4 ++-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index b926ef28..99d7371a 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -27,7 +27,7 @@ use super::{ BoxFut, BoxStream, Connection, Dialer, EnvelopeCodec, Inbound, Outbound, Payload, RpcError, SubUpdate, TransportError, }; -use crate::router::{Router, RouterBuilder}; +use crate::router::Router; use crate::AimDb; /// Capacity of a subscription's client-side event sink. Bounded (was @@ -843,7 +843,7 @@ where /// For the given connector `scheme` (e.g. `"aimx"`): /// - **outbound** routes (`db.collect_outbound_routes`) stream local record /// updates to the remote via [`ClientHandle::write`]; -/// - **inbound** routes (`db.collect_inbound_routes`) subscribe to the remote and +/// - **inbound** routes (`router`, from [`AimDb::inbound_router`]) subscribe to the remote and /// produce each update into the local record through the producer/arbiter path /// — single-writer-per-key stays intact (a mirrored-in record is produced /// through its inbound producer, never a direct co-writer). Mirroring is @@ -856,14 +856,7 @@ where /// /// Reconnect caveat: inbound pumps subscribe once and are not replayed across a /// reconnect (see [`ClientConfig::reconnect`]); outbound mirroring is unaffected. -pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec> { - let router = RouterBuilder::from_routes(db.collect_inbound_routes(scheme)).build(); - pump_client_with(db, scheme, router, handle) -} - -/// Like [`pump_client`], mirroring inbound through `router`, typically from -/// [`AimDb::inbound_router`]. -pub fn pump_client_with( +pub fn pump_client( db: &AimDb, scheme: &str, router: Router, @@ -1571,7 +1564,7 @@ mod tests { Ok(()) }); let routes = alloc::vec![(String::from("tele"), ingest)]; - let router = Arc::new(RouterBuilder::from_routes(routes).build()); + let router = Arc::new(crate::router::RouterBuilder::from_routes(routes).build()); let ctx = crate::RuntimeContext::new(Arc::new(crate::executor::test_support::NoopRuntimeOps)); let (handle, cmd_rx, _prune_rx) = test_handle(); diff --git a/aimdb-core/src/session/connector.rs b/aimdb-core/src/session/connector.rs index 1eadc9a3..6e883022 100644 --- a/aimdb-core/src/session/connector.rs +++ b/aimdb-core/src/session/connector.rs @@ -28,7 +28,7 @@ use core::pin::Pin; use crate::builder::AimDb; use crate::connector::ConnectorBuilder; use crate::session::{ - pump_client_with, run_client, serve, ClientConfig, Dialer, Dispatch, EnvelopeCodec, Listener, + pump_client, run_client, serve, ClientConfig, Dialer, Dispatch, EnvelopeCodec, Listener, OneShot, SessionConfig, }; use crate::{DbError, DbResult}; @@ -104,7 +104,7 @@ where // One pump future per route; each holds a `ClientHandle` clone, so the // engine stays alive as long as any mirror runs. `handle` drops here. let router = db.inbound_router(&self.scheme, &crate::ExactGrammar)?; - let mut futures = pump_client_with(db, &self.scheme, router, &handle); + let mut futures = pump_client(db, &self.scheme, router, &handle); futures.push(engine_fut); Ok(futures) }) diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 2591ae0f..663d543e 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -42,7 +42,7 @@ mod topic_match; pub use topic_match::{is_wildcard, pattern_contains, topic_matches}; #[cfg(feature = "connector-session")] -pub use client::{pump_client, pump_client_with, run_client, ClientConfig, ClientHandle}; +pub use client::{pump_client, run_client, ClientConfig, ClientHandle}; #[cfg(feature = "connector-session")] pub use connector::{SessionClientConnector, SessionServerConnector}; #[cfg(feature = "connector-session")] diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index 9f9fe34a..ae0db84c 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -19,7 +19,7 @@ use std::pin::Pin; -use aimdb_core::session::{aimx::AimxCodec, pump_client_with, run_client, ClientConfig}; +use aimdb_core::session::{aimx::AimxCodec, pump_client, run_client, ClientConfig}; use aimdb_core::{ConnectorBuilder, ExactGrammar}; use crate::transport::WsDialer; @@ -172,7 +172,7 @@ impl ConnectorBuilder for WsClientConnectorBuilder { db.runtime_ops(), ); let router = db.inbound_router("ws-client", &ExactGrammar)?; - let mut futures = pump_client_with(db, "ws-client", router, &handle); + let mut futures = pump_client(db, "ws-client", router, &handle); futures.push(engine_fut); Ok(futures) }) From 856ce0f20ffa0a9f5445c7920818470ddd5fddc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 02:00:33 +0000 Subject: [PATCH 10/18] refactor(core)!: remove the grammar-less inbound route API (055) --- aimdb-bench/benches/b0_alloc_connector.rs | 8 +- aimdb-core/src/builder.rs | 76 +--- aimdb-core/src/connector.rs | 52 +-- aimdb-core/src/lib.rs | 10 +- aimdb-core/src/router.rs | 408 +++++------------- aimdb-core/src/session/client.rs | 13 +- aimdb-core/src/session/mod.rs | 2 +- aimdb-core/src/session/pump.rs | 27 +- aimdb-core/src/typed_api.rs | 172 ++------ aimdb-data-contracts/src/link_codec.rs | 25 +- aimdb-knx-connector/src/connector.rs | 4 +- aimdb-mqtt-connector/src/embedded/mod.rs | 22 +- aimdb-mqtt-connector/src/native.rs | 49 +-- aimdb-mqtt-connector/tests/link_ext_tests.rs | 11 +- .../tests/topic_provider_tests.rs | 2 +- 15 files changed, 223 insertions(+), 658 deletions(-) diff --git a/aimdb-bench/benches/b0_alloc_connector.rs b/aimdb-bench/benches/b0_alloc_connector.rs index 9c8ab489..b64bf02b 100644 --- a/aimdb-bench/benches/b0_alloc_connector.rs +++ b/aimdb-bench/benches/b0_alloc_connector.rs @@ -33,10 +33,9 @@ use aimdb_core::connector::{ ConnectorBuilder, SerializeError, SerializedPayload, SerializedReader, SerializedValueInto, TopicProvider, }; -use aimdb_core::router::RouterBuilder; use aimdb_core::session::{pump_source, Payload, Source}; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; -use aimdb_core::{AimDb, AimDbBuilder, BoxFut, DbResult, RuntimeContext, StringKey}; +use aimdb_core::{AimDb, AimDbBuilder, BoxFut, DbResult, ExactGrammar, RuntimeContext, StringKey}; use aimdb_tokio_adapter::{TokioAdapter, TokioRecordRegistrarExt}; #[global_allocator] @@ -165,7 +164,7 @@ async fn inbound_db() -> AimDb { async fn measure_route() -> (u64, u64) { let db = inbound_db().await; let ctx = db.runtime_ctx(); - let router = RouterBuilder::from_routes(db.collect_inbound_routes(SCHEME)).build(); + let router = db.inbound_router(SCHEME, &ExactGrammar).unwrap(); let payload = [1u8; 8]; for _ in 0..WARMUP_ITERS { router.route("in/target", &payload, &ctx).unwrap(); @@ -207,7 +206,8 @@ async fn pump_run(db: &AimDb, messages: usize) -> (u64, u64) { remaining: messages, }; reset(); - for fut in pump_source(db, SCHEME, source) { + let router = db.inbound_router(SCHEME, &ExactGrammar).unwrap(); + for fut in pump_source(db, router, source) { fut.await; } snapshot() diff --git a/aimdb-core/src/builder.rs b/aimdb-core/src/builder.rs index bd9857c1..127fc89d 100644 --- a/aimdb-core/src/builder.rs +++ b/aimdb-core/src/builder.rs @@ -1179,69 +1179,6 @@ impl AimDb { self.inner.set_record_from_json(record_name, json_value) } - /// Collects inbound connector routes for automatic router construction (std only) - /// - /// Iterates all records, filters their inbound_connectors by scheme, - /// and returns routes with fused ingest callbacks (deserialize + produce - /// in one typed closure — no `Box` per message). - /// - /// # Arguments - /// * `scheme` - URL scheme to filter by (e.g., "mqtt", "kafka") - /// - /// # Returns - /// Vector of tuples: (topic, ingest) - /// - /// The topic is resolved dynamically if a `TopicResolverFn` is configured, - /// otherwise the static topic from the URL is used. Links whose topic has - /// `{…}` captures are skipped with a warning. - pub fn collect_inbound_routes( - &self, - scheme: &str, - ) -> Vec<(String, crate::connector::IngestFn)> { - let mut routes = Vec::new(); - - for entry in &self.inner.storages { - let inbound_links = entry.record.inbound_connectors(); - - for link in inbound_links { - // Filter by scheme - if link.url.scheme() != scheme { - continue; - } - - // Resolve topic: dynamic (from resolver) or static (from URL) - let topic = link.resolve_topic(); - - if crate::TopicPattern::parse(&topic).is_ok_and(|p| p.has_captures()) { - log_warn!( - "Skipping inbound link '{}': this connector does not support topic patterns", - topic - ); - continue; - } - - // Create the fused ingest callback using the stored factory - let ingest = match &link.match_ingest_factory { - Some(factory) => { - crate::connector::match_as_ingest(factory(self), topic.as_str().into()) - } - None => link.create_ingest(self), - }; - routes.push((topic, ingest)); - } - } - - if !routes.is_empty() { - log_debug!( - "Collected {} inbound routes for scheme '{}'", - routes.len(), - scheme - ); - } - - routes - } - /// The inbound router for `scheme`: every link compiled against the /// connector's `grammar`, keyed links sharing their record's key table. /// @@ -1275,7 +1212,7 @@ impl AimDb { if !errors.is_empty() { return Err(DbError::InvalidConfiguration { errors }); } - Ok(crate::Router::compiled(grammar, routes)) + Ok(crate::Router::new(grammar, routes)) } fn inbound_route( @@ -1300,12 +1237,11 @@ impl AimDb { _ => None, }; - let ingest = match &link.match_ingest_factory { - Some(factory) => factory(self), - None => crate::connector::ignore_match(link.create_ingest(self)), - }; Ok(crate::router::CompiledRoute::pattern( - filter, names, key, ingest, + filter, + names, + key, + link.create_ingest(self), )) } @@ -1322,7 +1258,7 @@ impl AimDb { /// Collects outbound routes for a specific protocol scheme /// - /// Mirrors `collect_inbound_routes()` for symmetry. Iterates all records, + /// Mirrors [`inbound_router`](Self::inbound_router). Iterates all records, /// filters their outbound_connectors by scheme, and returns /// [`OutboundRoute`]s carrying fused serialized sources (subscribe → /// recv → resolve topic → serialize, all typed inside — no diff --git a/aimdb-core/src/connector.rs b/aimdb-core/src/connector.rs index 323ff9ff..1364cd3f 100644 --- a/aimdb-core/src/connector.rs +++ b/aimdb-core/src/connector.rs @@ -511,12 +511,10 @@ impl ConnectorLink { ///, so the only failure is the user /// deserializer's — reported as the same `String` the deserializer API uses. /// -/// The [`RuntimeContext`](crate::RuntimeContext) is threaded per call (not -/// captured) for context-aware deserializers. -pub type IngestFn = Arc Result<(), String> + Send + Sync>; - -/// Fused ingest callback of a pattern route: also receives the match. -pub type MatchIngestFn = Arc< +/// The [`RuntimeContext`](crate::RuntimeContext) and the +/// [`TopicMatch`](crate::TopicMatch) the message arrived on are threaded per +/// call (not captured). +pub type IngestFn = Arc< dyn Fn(&crate::RuntimeContext, &crate::TopicMatch<'_>, &[u8]) -> Result<(), String> + Send + Sync, @@ -532,27 +530,6 @@ pub type MatchIngestFn = Arc< /// Available in both `std` and `no_std + alloc` environments. pub type IngestFactoryFn = Arc IngestFn + Send + Sync>; -/// Like [`IngestFactoryFn`], for links set with `with_match_deserializer`. -pub type MatchIngestFactoryFn = Arc MatchIngestFn + Send + Sync>; - -/// Runs a plain ingest where a [`MatchIngestFn`] is expected. -pub(crate) fn ignore_match(ingest: IngestFn) -> MatchIngestFn { - Arc::new(move |ctx, _m, payload| ingest(ctx, payload)) -} - -/// Runs a match-aware ingest where only an [`IngestFn`] fits: every message -/// is on `topic`, with no captures and no key. -pub(crate) fn match_as_ingest(ingest: MatchIngestFn, topic: Arc) -> IngestFn { - static NO_SPANS: crate::Spans = [(0, 0); crate::MAX_CAPTURES]; - Arc::new(move |ctx, payload| { - ingest( - ctx, - &crate::TopicMatch::new(&topic, &[], &NO_SPANS, None), - payload, - ) - }) -} - /// Topic resolver function for inbound connections (late-binding) /// /// Called once at connector startup to resolve the subscription topic. @@ -596,10 +573,6 @@ pub struct InboundConnectorLink { /// Available in both `std` and `no_std + alloc` environments. pub ingest_factory: IngestFactoryFn, - /// Set by `with_match_deserializer`; `ingest_factory` then passes the - /// URL topic as the match. - pub match_ingest_factory: Option, - /// Set by `.key(..)`: the keyed capture and the key table's capacity. pub key: Option<(String, core::num::NonZeroU16)>, @@ -618,10 +591,6 @@ impl Debug for InboundConnectorLink { .field("url", &self.url) .field("config", &self.config) .field("ingest_factory", &"") - .field( - "match_ingest_factory", - &self.match_ingest_factory.as_ref().map(|_| ""), - ) .field("key", &self.key) .field( "topic_resolver", @@ -638,7 +607,6 @@ impl InboundConnectorLink { url, config: Vec::new(), ingest_factory, - match_ingest_factory: None, key: None, topic_resolver: None, } @@ -765,7 +733,9 @@ fn parse_connector_url(url: &str) -> DbResult { /// # Example /// /// Illustrative sketch of a connector author's `build()` (not compiled: the -/// client types are fictional — see `aimdb-mqtt-connector` for a real one): +/// client types and `MqttGrammar`, the connector's +/// [`TopicGrammar`](crate::TopicGrammar), are the connector's own — see +/// `aimdb-mqtt-connector` for a real one): /// /// ```rust,ignore /// pub struct MqttConnectorBuilder { @@ -778,8 +748,8 @@ fn parse_connector_url(url: &str) -> DbResult { /// db: &'a AimDb, /// ) -> Pin>> + Send + 'a>> { /// Box::pin(async move { -/// let routes = db.collect_inbound_routes(self.scheme()); -/// let router = RouterBuilder::from_routes(routes).build(); +/// // Wildcard rules are the connector's; `&ExactGrammar` if it has none. +/// let router = db.inbound_router(self.scheme(), &MqttGrammar)?; /// let connector = MqttConnector::new(&self.broker_url, router).await?; /// Ok(connector.futures()) /// }) @@ -826,7 +796,7 @@ pub trait ConnectorBuilder: Send + Sync { /// Whether registering a second connector under this scheme is an error. /// /// Say `true` when [`build`](Self::build) claims every route for its - /// scheme — [`collect_inbound_routes`](crate::AimDb::collect_inbound_routes), + /// scheme — [`inbound_router`](crate::AimDb::inbound_router), /// [`collect_outbound_routes`](crate::AimDb::collect_outbound_routes), /// and `crate::session`'s `pump_source`, `pump_sink` and `pump_client` /// (left unlinked: that module is behind `connector-session`, and this @@ -1064,7 +1034,7 @@ mod tests { /// Dummy ingest factory for link-construction tests (never invoked). fn dummy_ingest_factory() -> super::IngestFactoryFn { - Arc::new(|_db| Arc::new(|_ctx: &crate::RuntimeContext, _bytes: &[u8]| Ok(()))) + Arc::new(|_db| Arc::new(|_ctx, _m, _bytes| Ok(()))) } #[test] diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 17924cc5..5dcac47d 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -140,9 +140,9 @@ pub use remote::topic_leaf; // compatible). See docs/design/remote-access-via-connectors.md. #[cfg(feature = "connector-session")] pub use session::{ - is_wildcard, pattern_contains, pump_sink, pump_source, pump_source_with, topic_matches, - AuthError, BoxFut, BoxStream, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, - Listener, Outbound, Payload, PeerInfo, RpcError, SessionCtx, SessionLimits, Source, SubUpdate, + is_wildcard, pattern_contains, pump_sink, pump_source, topic_matches, AuthError, BoxFut, + BoxStream, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, + Outbound, Payload, PeerInfo, RpcError, SessionCtx, SessionLimits, Source, SubUpdate, TransportError, TransportResult, }; @@ -160,14 +160,14 @@ pub use profiling::{ pub use connector::TopicProvider; pub use connector::TopicResolverFn; pub use connector::{ConnectorLink, ConnectorUrl, LinkAddress, SerializeError}; -pub use connector::{IngestFactoryFn, IngestFn, MatchIngestFactoryFn, MatchIngestFn}; +pub use connector::{IngestFactoryFn, IngestFn}; pub use connector::{ SerializedPayload, SerializedReader, SerializedSource, SerializedValue, SerializedValueInto, SourceFactoryFn, }; // Router exports for connector implementations -pub use router::{Route, Router, RouterBuilder}; +pub use router::Router; // Topic grammar for connectors with wildcard subscriptions pub use topic_pattern::{ diff --git a/aimdb-core/src/router.rs b/aimdb-core/src/router.rs index 9fd80609..403586fc 100644 --- a/aimdb-core/src/router.rs +++ b/aimdb-core/src/router.rs @@ -13,42 +13,15 @@ use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; -use crate::connector::{IngestFn, MatchIngestFn}; +use crate::connector::IngestFn; use crate::inbound_key::{KeyId, KeyTable}; -use crate::topic_pattern::{ - ExactGrammar, Spans, TopicFilter, TopicGrammar, TopicMatch, MAX_CAPTURES, -}; - -/// A single routing entry -/// -/// Maps one (resource_id, type) pair to a fused ingest callback. -/// Multiple routes can exist for the same resource_id (different types). -/// -/// # Resource ID Examples -/// -/// - MQTT: "sensors/temperature" (topic) -/// - Kafka: "events:0" (topic:partition) -/// - HTTP: "/api/v1/sensors" (path) -/// - DDS: "TelemetryData" (topic name) -/// - Shmem: "temperature_buffer" (segment name) -pub struct Route { - /// Resource identifier to match (reference-counted for proper memory management) - /// - /// Examples: MQTT topic, Kafka topic, HTTP path, DDS topic, shmem segment - /// - /// Uses `Arc` instead of `&'static str` to avoid memory leaks from `Box::leak()`. - /// This adds ~8 bytes overhead per route (Arc control block) but enables proper cleanup. - pub resource_id: Arc, - - /// Fused ingest callback: deserialize + produce in one typed closure - /// built at registration time (no `Box` per message). - pub ingest: IngestFn, -} +use crate::topic_pattern::{Spans, TopicFilter, TopicGrammar, TopicMatch, MAX_CAPTURES}; /// Generic message router for connector dispatch /// -/// Routes incoming messages to the matching records' ingest callbacks based on -/// resource_id. Uses linear search which is efficient for <100 routes. +/// Built by [`AimDb::inbound_router`](crate::AimDb::inbound_router). Routes +/// incoming messages to the matching records' ingest callbacks. Uses linear +/// search which is efficient for <100 routes. /// /// # Performance /// @@ -81,19 +54,17 @@ pub(crate) struct CompiledRoute { names: Box<[Box]>, /// Key table and the capture number whose value is keyed. key: Option<(Arc, usize)>, - ingest: MatchIngestFn, + ingest: IngestFn, } impl CompiledRoute { - /// A route comparing `resource_id` as a string. - pub(crate) fn exact(resource_id: Arc, ingest: IngestFn) -> Self { - Self { - filter: resource_id, - matcher: None, - names: Box::new([]), - key: None, - ingest: crate::connector::ignore_match(ingest), - } + /// A route comparing `topic` as a string. + #[cfg(test)] + pub(crate) fn exact(topic: &str, ingest: IngestFn) -> Self { + use crate::topic_pattern::{ExactGrammar, TopicPattern}; + let pattern = TopicPattern::parse(topic).expect("an exact topic"); + let filter = ExactGrammar.compile(&pattern).expect("an exact topic"); + Self::pattern(filter, Box::new([]), None, ingest) } /// A route matching through `filter`. @@ -101,7 +72,7 @@ impl CompiledRoute { filter: Box, names: Box<[Box]>, key: Option<(Arc, usize)>, - ingest: MatchIngestFn, + ingest: IngestFn, ) -> Self { Self { filter: filter.filter().into(), @@ -131,18 +102,7 @@ impl CompiledRoute { } impl Router { - /// Create a new router with the given routes - pub fn new(routes: Vec) -> Self { - Self { - routes: routes - .into_iter() - .map(|r| CompiledRoute::exact(r.resource_id, r.ingest)) - .collect(), - grammar: &ExactGrammar, - } - } - - pub(crate) fn compiled(grammar: &'static dyn TopicGrammar, routes: Vec) -> Self { + pub(crate) fn new(grammar: &'static dyn TopicGrammar, routes: Vec) -> Self { Self { routes, grammar } } @@ -262,80 +222,13 @@ impl Router { } } -/// Builder for constructing routers -/// -/// Provides a fluent API for adding routes before creating the router. -pub struct RouterBuilder { - routes: Vec, -} - -impl RouterBuilder { - /// Create a new router builder - pub fn new() -> Self { - Self { routes: Vec::new() } - } - - /// Create a router builder from a collection of routes - /// - /// This is a convenience method for automatic router construction from - /// `AimDb::collect_inbound_routes()`. The resource_ids are converted to - /// `Arc` for proper memory management. - /// - /// # Arguments - /// * `routes` - Vector of (resource_id, ingest) tuples - pub fn from_routes(routes: Vec<(String, IngestFn)>) -> Self { - let mut builder = Self::new(); - for (resource_id, ingest) in routes { - // Convert String to Arc - no leaking needed! - let resource_id_arc: Arc = Arc::from(resource_id.as_str()); - builder = builder.add_route(resource_id_arc, ingest); - } - builder - } - - /// Add a route to the router - /// - /// # Arguments - /// * `resource_id` - Resource identifier to match (as `Arc`) - /// * `ingest` - Fused ingest callback (deserialize + produce) - /// - /// # Resource ID Memory Management - /// The resource_id is stored as `Arc` for proper reference counting and cleanup. - /// You can create an `Arc` from: - /// - String literal: `Arc::from("sensors/temperature")` - /// - Owned String: `Arc::from(string.as_str())` - pub fn add_route(mut self, resource_id: Arc, ingest: IngestFn) -> Self { - self.routes.push(Route { - resource_id, - ingest, - }); - self - } - - /// Build the router - /// - /// Consumes the builder and returns a configured Router. - pub fn build(self) -> Router { - Router::new(self.routes) - } - - /// Get the number of routes that will be created - pub fn route_count(&self) -> usize { - self.routes.len() - } -} - -impl Default for RouterBuilder { - fn default() -> Self { - Self::new() - } -} - #[cfg(all(test, feature = "std"))] mod tests { use super::*; + use crate::topic_pattern::{test_support::Plus, ExactGrammar, TopicPattern}; + use core::num::NonZeroU16; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; /// A `RuntimeContext` backed by the shared no-op RuntimeOps. fn test_ctx() -> crate::RuntimeContext { @@ -344,147 +237,20 @@ mod tests { /// Ingest callback that counts successful invocations. fn counting_ingest(call_count: Arc) -> IngestFn { - Arc::new(move |_ctx, _payload| { + Arc::new(move |_ctx, _m, _payload| { call_count.fetch_add(1, Ordering::SeqCst); Ok(()) }) } - #[test] - fn test_single_route() { - let call_count = Arc::new(AtomicUsize::new(0)); - - let routes = vec![Route { - resource_id: Arc::from("test/resource"), - ingest: counting_ingest(call_count.clone()), - }]; - - let router = Router::new(routes); - - router - .route("test/resource", b"dummy", &test_ctx()) - .unwrap(); - - assert_eq!(call_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_multiple_routes_same_resource() { - let call_count1 = Arc::new(AtomicUsize::new(0)); - let call_count2 = Arc::new(AtomicUsize::new(0)); - - let routes = vec![ - Route { - resource_id: Arc::from("shared/resource"), - ingest: counting_ingest(call_count1.clone()), - }, - Route { - resource_id: Arc::from("shared/resource"), - ingest: counting_ingest(call_count2.clone()), - }, - ]; - - let router = Router::new(routes); - - router - .route("shared/resource", b"dummy", &test_ctx()) - .unwrap(); - - // Both ingest callbacks should be called - assert_eq!(call_count1.load(Ordering::SeqCst), 1); - assert_eq!(call_count2.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_unknown_resource() { - let call_count = Arc::new(AtomicUsize::new(0)); - - let routes = vec![Route { - resource_id: Arc::from("test/resource"), - ingest: counting_ingest(call_count.clone()), - }]; - - let router = Router::new(routes); - - // Should not panic on unknown resource - router - .route("unknown/resource", b"dummy", &test_ctx()) - .unwrap(); - - assert_eq!(call_count.load(Ordering::SeqCst), 0); - } - - #[test] - fn test_resource_ids_deduplication() { - let routes = vec![ - Route { - resource_id: Arc::from("resource1"), - ingest: counting_ingest(Arc::new(AtomicUsize::new(0))), - }, - Route { - resource_id: Arc::from("resource1"), // Duplicate - ingest: counting_ingest(Arc::new(AtomicUsize::new(0))), - }, - Route { - resource_id: Arc::from("resource2"), - ingest: counting_ingest(Arc::new(AtomicUsize::new(0))), - }, - ]; - - let router = Router::new(routes); - let ids = router.resource_ids(); - - assert_eq!(ids.len(), 2); - assert!(ids.iter().any(|id| id.as_ref() == "resource1")); - assert!(ids.iter().any(|id| id.as_ref() == "resource2")); - } - - #[test] - fn test_ingest_receives_payload_and_ctx() { - let seen_len = Arc::new(AtomicUsize::new(0)); - let seen_len_clone = seen_len.clone(); - - let ingest: IngestFn = Arc::new(move |_ctx, payload| { - seen_len_clone.store(payload.len(), Ordering::SeqCst); - Ok(()) - }); - - let routes = vec![Route { - resource_id: Arc::from("ctx/resource"), - ingest, - }]; - - let router = Router::new(routes); - router.route("ctx/resource", b"dummy", &test_ctx()).unwrap(); - - assert_eq!(seen_len.load(Ordering::SeqCst), 5); - } - - #[test] - fn test_ingest_error_does_not_propagate() { - let ingest: IngestFn = Arc::new(|_ctx, _payload| Err("deserialize failed".into())); - - let routes = vec![Route { - resource_id: Arc::from("err/resource"), - ingest, - }]; - - let router = Router::new(routes); - - // Ingest failures are logged, not propagated. - router.route("err/resource", b"dummy", &test_ctx()).unwrap(); + fn noop() -> IngestFn { + counting_ingest(Arc::new(AtomicUsize::new(0))) } - // ---- pattern routes --------------------------------------------------- - - use crate::topic_pattern::{test_support::Plus, TopicPattern}; - use core::num::NonZeroU16; - use std::sync::Mutex; - type Seen = Arc>, Option)>>>; /// Records the topic, the named captures and the key index. - fn recording(seen: &Seen, names: &'static [&'static str]) -> MatchIngestFn { + fn recording(seen: &Seen, names: &'static [&'static str]) -> IngestFn { let seen = seen.clone(); Arc::new(move |_ctx, m, _payload| { let caps = names.iter().map(|n| m.get(n).map(String::from)).collect(); @@ -495,9 +261,9 @@ mod tests { }) } - fn pattern_route( + fn keyed_route( topic: &str, - ingest: MatchIngestFn, + ingest: IngestFn, key: Option<(Arc, usize)>, ) -> CompiledRoute { let pattern = TopicPattern::parse(topic).unwrap(); @@ -505,19 +271,75 @@ mod tests { CompiledRoute::pattern(Plus.compile(&pattern).unwrap(), names, key, ingest) } - fn exact_route(topic: &str, ingest: IngestFn) -> CompiledRoute { - CompiledRoute::exact(Arc::from(topic), ingest) + fn route(topic: &str, ingest: IngestFn) -> CompiledRoute { + keyed_route(topic, ingest, None) + } + + #[test] + fn every_matching_route_receives_the_message() { + let (a, b) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let router = Router::new( + &Plus, + vec![ + route("shared/resource", counting_ingest(a.clone())), + route("shared/resource", counting_ingest(b.clone())), + ], + ); + let ctx = test_ctx(); + router.route("shared/resource", b"", &ctx).unwrap(); + router.route("unknown/resource", b"", &ctx).unwrap(); + + assert_eq!(a.load(Ordering::SeqCst), 1); + assert_eq!(b.load(Ordering::SeqCst), 1); + } + + #[test] + fn ingest_receives_payload_and_its_errors_do_not_propagate() { + let seen_len = Arc::new(AtomicUsize::new(0)); + let len = seen_len.clone(); + let router = Router::new( + &Plus, + vec![ + route( + "r", + Arc::new(move |_ctx, _m, payload| { + len.store(payload.len(), Ordering::SeqCst); + Ok(()) + }), + ), + route("r", Arc::new(|_ctx, _m, _payload| Err("bad".into()))), + ], + ); + router.route("r", b"dummy", &test_ctx()).unwrap(); + assert_eq!(seen_len.load(Ordering::SeqCst), 5); + } + + #[test] + fn resource_ids_are_deduplicated() { + let router = Router::new( + &Plus, + vec![ + route("r1", noop()), + route("r1", noop()), + route("r2", noop()), + ], + ); + let ids: Vec = router + .resource_ids() + .iter() + .map(|s| s.to_string()) + .collect(); + assert_eq!(ids, ["r1", "r2"]); } #[test] fn pattern_route_receives_topic_and_captures() { let seen: Seen = Default::default(); - let router = Router::compiled( + let router = Router::new( &Plus, - vec![pattern_route( + vec![route( "{site}/+/{dev}", recording(&seen, &["site", "dev", "nope"]), - None, )], ); let ctx = test_ctx(); @@ -535,23 +357,20 @@ mod tests { } #[test] - fn exact_and_pattern_routes_both_receive_a_message() { - let exact = Arc::new(AtomicUsize::new(0)); + fn literal_and_pattern_routes_both_receive_a_message() { let seen: Seen = Default::default(); - let router = Router::compiled( + let router = Router::new( &Plus, vec![ - exact_route("s/kitchen/t", counting_ingest(exact.clone())), - pattern_route("s/{d}/t", recording(&seen, &["d"]), None), - pattern_route("s/kitchen/t", recording(&seen, &[]), None), + route("s/{d}/t", recording(&seen, &["d"])), + route("s/kitchen/t", recording(&seen, &[])), ], ); router.route("s/kitchen/t", b"", &test_ctx()).unwrap(); - assert_eq!(exact.load(Ordering::SeqCst), 1); let seen = seen.lock().unwrap(); assert_eq!(seen.len(), 2); - // A literal pattern route still receives the topic. + // A literal route still receives the topic. assert_eq!(seen[1].0, "s/kitchen/t"); } @@ -559,9 +378,9 @@ mod tests { fn keyed_route_assigns_keys_and_drops_when_full() { let seen: Seen = Default::default(); let table = Arc::new(KeyTable::new(NonZeroU16::new(2).unwrap())); - let router = Router::compiled( + let router = Router::new( &Plus, - vec![pattern_route( + vec![keyed_route( "s/{d}/t", recording(&seen, &["d"]), Some((table.clone(), 0)), @@ -580,10 +399,7 @@ mod tests { #[test] fn overlong_topics_skip_pattern_routes() { let seen: Seen = Default::default(); - let router = Router::compiled( - &Plus, - vec![pattern_route("{x}", recording(&seen, &[]), None)], - ); + let router = Router::new(&Plus, vec![route("{x}", recording(&seen, &[]))]); let topic = "x".repeat(usize::from(u16::MAX) + 1); router.route(&topic, b"", &test_ctx()).unwrap(); assert!(seen.lock().unwrap().is_empty()); @@ -599,37 +415,27 @@ mod tests { #[test] fn subscriptions_drop_covered_filters() { - let noop = || counting_ingest(Arc::new(AtomicUsize::new(0))); - let seen: Seen = Default::default(); - let router = Router::compiled( + let router = Router::new( &Plus, vec![ - exact_route("s/kitchen/t", noop()), - exact_route("other/x", noop()), - exact_route("s/+/t", noop()), - pattern_route("s/{d}/t", recording(&seen, &[]), None), + route("s/kitchen/t", noop()), + route("other/x", noop()), + route("s/+/t", noop()), + route("s/{d}/t", noop()), ], ); assert_eq!(subscriptions(&router), ["other/x", "s/+/t"]); } #[test] - fn plain_router_subscribes_each_id_once() { - let noop = || counting_ingest(Arc::new(AtomicUsize::new(0))); - let router = Router::new(vec![ - Route { - resource_id: Arc::from("a"), - ingest: noop(), - }, - Route { - resource_id: Arc::from("a"), - ingest: noop(), - }, - Route { - resource_id: Arc::from("b"), - ingest: noop(), - }, - ]); - assert_eq!(subscriptions(&router), ["a", "b"]); + fn exact_grammar_routes_compare_strings() { + let hits = Arc::new(AtomicUsize::new(0)); + let exact = CompiledRoute::exact("a/+", counting_ingest(hits.clone())); + let router = Router::new(&ExactGrammar, vec![exact]); + let ctx = test_ctx(); + router.route("a/b", b"", &ctx).unwrap(); + router.route("a/+", b"", &ctx).unwrap(); + assert_eq!(hits.load(Ordering::SeqCst), 1); + assert_eq!(subscriptions(&router), ["a/+"]); } } diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index 99d7371a..d3a966cb 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -1558,13 +1558,12 @@ mod tests { async fn a_mirror_gap_still_routes_and_keeps_mirroring() { let seen: Arc>>> = Arc::new(spin::Mutex::new(Vec::new())); let recorder = seen.clone(); - let ingest: crate::connector::IngestFn = - Arc::new(move |_ctx: &crate::RuntimeContext, bytes: &[u8]| { - recorder.lock().push(bytes.to_vec()); - Ok(()) - }); - let routes = alloc::vec![(String::from("tele"), ingest)]; - let router = Arc::new(crate::router::RouterBuilder::from_routes(routes).build()); + let ingest: crate::connector::IngestFn = Arc::new(move |_ctx, _m, bytes: &[u8]| { + recorder.lock().push(bytes.to_vec()); + Ok(()) + }); + let route = crate::router::CompiledRoute::exact("tele", ingest); + let router = Arc::new(Router::new(&crate::ExactGrammar, alloc::vec![route])); let ctx = crate::RuntimeContext::new(Arc::new(crate::executor::test_support::NoopRuntimeOps)); let (handle, cmd_rx, _prune_rx) = test_handle(); diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 663d543e..491436ad 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -54,7 +54,7 @@ pub use io::{ OneShotListener, StreamDialer, StreamListener, }; #[cfg(feature = "connector-session")] -pub use pump::{pump_sink, pump_source, pump_source_with}; +pub use pump::{pump_sink, pump_source}; #[cfg(feature = "connector-session")] pub use server::{run_session, serve, SessionConfig}; diff --git a/aimdb-core/src/session/pump.rs b/aimdb-core/src/session/pump.rs index 0bc3fe6f..7030acbf 100644 --- a/aimdb-core/src/session/pump.rs +++ b/aimdb-core/src/session/pump.rs @@ -8,7 +8,8 @@ //! //! ```rust,ignore //! let mut f = pump_sink(db, "redis", self.sink().await?); // outbound -//! f.extend(pump_source(db, "redis", self.subscription().await?)); // inbound +//! let router = db.inbound_router("redis", &ExactGrammar)?; +//! f.extend(pump_source(db, router, self.subscription().await?)); // inbound //! Ok(f) //! ``` //! @@ -21,7 +22,7 @@ use alloc::vec::Vec; use super::Source; use crate::builder::{AimDb, BoxFuture}; -use crate::router::{Router, RouterBuilder}; +use crate::router::Router; use crate::transport::{Connector, ConnectorConfig}; /// Outbound pump: one publisher future per outbound route on `scheme`. @@ -126,30 +127,16 @@ pub fn pump_sink(db: &AimDb, scheme: &str, sink: Arc) -> Vec Vec { - let routes = db.collect_inbound_routes(scheme); - pump_source_with(db, RouterBuilder::from_routes(routes).build(), src) -} - -/// Like [`pump_source`], routing with `router`, typically from -/// [`AimDb::inbound_router`], whose subscriptions the connector made. -pub fn pump_source_with( - db: &AimDb, - router: Router, - mut src: impl Source + 'static, -) -> Vec { +pub fn pump_source(db: &AimDb, router: Router, mut src: impl Source + 'static) -> Vec { let router = Arc::new(router); let ctx = db.runtime_ctx(); diff --git a/aimdb-core/src/typed_api.rs b/aimdb-core/src/typed_api.rs index c7d64991..fbb2a51b 100644 --- a/aimdb-core/src/typed_api.rs +++ b/aimdb-core/src/typed_api.rs @@ -1058,28 +1058,10 @@ where /// Fused ingest factory: resolves the typed producer once at route-collection /// time; per message the returned closure runs deserialize + produce with no /// erasure crossing. -fn plain_ingest_factory( - record_key: String, - deser: TypedContextDeserializerFn, -) -> crate::connector::IngestFactoryFn -where - T: Send + Sync + 'static + Debug + Clone, -{ - Arc::new(move |db: &AimDb| { - let producer = inbound_producer::(db, &record_key); - let deser = deser.clone(); - Arc::new(move |ctx: &crate::RuntimeContext, payload: &[u8]| { - producer.produce(deser(ctx.clone(), payload)?); - Ok(()) - }) as crate::connector::IngestFn - }) -} - -/// Like [`plain_ingest_factory`], for `with_match_deserializer`. -fn match_ingest_factory( +fn ingest_factory( record_key: String, deser: TypedMatchDeserializerFn, -) -> crate::connector::MatchIngestFactoryFn +) -> crate::connector::IngestFactoryFn where T: Send + Sync + 'static + Debug + Clone, { @@ -1091,7 +1073,7 @@ where producer.produce(deser(ctx, m, payload)?); Ok(()) }, - ) as crate::connector::MatchIngestFn + ) as crate::connector::IngestFn }) } @@ -1237,20 +1219,12 @@ where let url = LinkAddress::parse(&self.url).map_err(|_| "Invalid connector URL")?; let record_key = self.registrar.record_key.clone(); - let (ingest_factory, match_ingest_factory) = match ( + let deser: TypedMatchDeserializerFn = match ( self.context_deserializer.take(), self.match_deserializer.take(), ) { - (Some(deser), None) => (plain_ingest_factory(record_key, deser), None), - (None, Some(deser)) => { - let factory = match_ingest_factory(record_key, deser); - let topic: Arc = url.resource_id().into(); - let inner = factory.clone(); - let plain: crate::connector::IngestFactoryFn = Arc::new(move |db: &AimDb| { - crate::connector::match_as_ingest(inner(db), topic.clone()) - }); - (plain, Some(factory)) - } + (Some(deser), None) => Arc::new(move |ctx, _m, bytes| deser(ctx.clone(), bytes)), + (None, Some(deser)) => deser, (Some(_), Some(_)) => { return Err( "Set either .with_deserializer() or .with_match_deserializer(), not both" @@ -1305,9 +1279,8 @@ where )); } - let mut link = InboundConnectorLink::new(url, ingest_factory); + let mut link = InboundConnectorLink::new(url, ingest_factory(record_key, deser)); link.config = core::mem::take(&mut self.config); - link.match_ingest_factory = match_ingest_factory; link.key = key; link.topic_resolver = self.topic_resolver.take(); Ok(link) @@ -1521,7 +1494,6 @@ mod tests { .finish(); }); assert!(errors.is_empty(), "{errors:?}"); - assert!(links[0].match_ingest_factory.is_some()); let (capture, capacity) = links[0].key.clone().unwrap(); assert_eq!((capture.as_str(), capacity.get()), ("device", 16)); } @@ -2018,22 +1990,17 @@ mod tests { } } + /// Routes `payload` on `topic` through the `mqtt` inbound router. + fn route(db: &crate::AimDb, topic: &str, payload: &[u8]) { + let router = db.inbound_router("mqtt", &Plus).expect("routes compile"); + router.route(topic, payload, &db.runtime_ctx()).unwrap(); + } + /// End-to-end inbound path: bytes → fused ingest → typed buffer push, /// with no `Box` in between. #[tokio::test] async fn ingest_roundtrip_produces_value() { - let last = Arc::new(AtomicI32::new(-1)); - let count = Arc::new(AtomicUsize::new(0)); - let (buf_last, buf_count) = (last.clone(), count.clone()); - - let mut builder = crate::AimDbBuilder::new() - .runtime(Arc::new(MockRuntime)) - .with_connector(NoopConnectorBuilder); - builder.configure::("rec.in", move |reg| { - reg.buffer_raw(Box::new(RecordingBuffer { - last: buf_last, - count: buf_count, - })); + let (db, last, count) = inbound_db(|reg| { reg.link_from("mqtt://cmd/in") .with_deserializer(|_ctx, bytes: &[u8]| { if bytes.is_empty() { @@ -2044,119 +2011,34 @@ mod tests { }) }) .finish(); - }); - let (db, _runner) = builder.build().await.expect("build must succeed"); - - let routes = db.collect_inbound_routes("mqtt"); - assert_eq!(routes.len(), 1); - let (topic, ingest) = &routes[0]; - assert_eq!(topic, "cmd/in"); + }) + .await; - let ctx = db.runtime_ctx(); - ingest(&ctx, &[1, 2, 3]).expect("ingest must succeed"); + route(&db, "cmd/in", &[1, 2, 3]); assert_eq!(count.load(Ordering::SeqCst), 1); assert_eq!(last.load(Ordering::SeqCst), 3); - // Bad bytes: the deserializer error propagates, nothing is produced. - let err = ingest(&ctx, &[]).expect_err("empty payload must fail"); - assert_eq!(err, "empty payload"); + // Bad bytes: the deserializer fails, nothing is produced. + route(&db, "cmd/in", &[]); assert_eq!(count.load(Ordering::SeqCst), 1); } - /// Raw/context mutual exclusion is behavior now (the kind enum is gone): - /// the variant set last wins inside the fused ingest closure. + /// The deserializer set last wins, whichever way its context is typed. #[tokio::test] - async fn context_deserializer_set_last_wins() { - let last = Arc::new(AtomicI32::new(-1)); - let count = Arc::new(AtomicUsize::new(0)); - let (buf_last, buf_count) = (last.clone(), count.clone()); - - let mut builder = crate::AimDbBuilder::new() - .runtime(Arc::new(MockRuntime)) - .with_connector(NoopConnectorBuilder); - builder.configure::("rec.in", move |reg| { - reg.buffer_raw(Box::new(RecordingBuffer { - last: buf_last, - count: buf_count, - })); + async fn deserializer_set_last_wins() { + let (db, last, _) = inbound_db(|reg| { reg.link_from("mqtt://cmd/in") .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) .with_deserializer(|_ctx: crate::RuntimeContext, _bytes: &[u8]| { Ok(TestRecord { value: 99 }) }) .finish(); - }); - let (db, _runner) = builder.build().await.expect("build must succeed"); - - let routes = db.collect_inbound_routes("mqtt"); - let (_, ingest) = &routes[0]; - ingest(&db.runtime_ctx(), b"x").expect("ingest must succeed"); + }) + .await; + route(&db, "cmd/in", b"x"); assert_eq!(last.load(Ordering::SeqCst), 99); } - /// And the reverse: raw set last wins over a prior context deserializer. - #[tokio::test] - async fn raw_deserializer_set_last_wins() { - let last = Arc::new(AtomicI32::new(-1)); - let count = Arc::new(AtomicUsize::new(0)); - let (buf_last, buf_count) = (last.clone(), count.clone()); - - let mut builder = crate::AimDbBuilder::new() - .runtime(Arc::new(MockRuntime)) - .with_connector(NoopConnectorBuilder); - builder.configure::("rec.in", move |reg| { - reg.buffer_raw(Box::new(RecordingBuffer { - last: buf_last, - count: buf_count, - })); - reg.link_from("mqtt://cmd/in") - .with_deserializer(|_ctx: crate::RuntimeContext, _bytes: &[u8]| { - Ok(TestRecord { value: 0 }) - }) - .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 7 })) - .finish(); - }); - let (db, _runner) = builder.build().await.expect("build must succeed"); - - let routes = db.collect_inbound_routes("mqtt"); - let (_, ingest) = &routes[0]; - ingest(&db.runtime_ctx(), b"x").expect("ingest must succeed"); - assert_eq!(last.load(Ordering::SeqCst), 7); - } - - /// The pre-pattern route API skips `{…}` links and gives a literal-topic - /// match link its topic. - #[tokio::test] - async fn collect_inbound_routes_skips_patterns_and_passes_literal_topics() { - let last = Arc::new(AtomicI32::new(-1)); - let count = Arc::new(AtomicUsize::new(0)); - let (buf_last, buf_count) = (last.clone(), count.clone()); - - let mut builder = crate::AimDbBuilder::new() - .runtime(Arc::new(MockRuntime)) - .with_connector(NoopConnectorBuilder); - builder.configure::("rec.in", move |reg| { - reg.buffer_raw(Box::new(RecordingBuffer { - last: buf_last, - count: buf_count, - })); - reg.link_from("mqtt://s/{d}/t") - .with_deserializer(|_ctx, _bytes: &[u8]| Ok(TestRecord { value: 0 })) - .finish(); - reg.link_from("mqtt://cmd/in") - .with_match_deserializer(match_deser) - .finish(); - }); - let (db, _runner) = builder.build().await.expect("build must succeed"); - - let routes = db.collect_inbound_routes("mqtt"); - assert_eq!(routes.len(), 1); - let (topic, ingest) = &routes[0]; - assert_eq!(topic, "cmd/in"); - ingest(&db.runtime_ctx(), b"x").expect("ingest must succeed"); - assert_eq!(last.load(Ordering::SeqCst), "cmd/in".len() as i32); - } - // ==================================================================== // inbound_router: patterns, keys and connector-build errors // ==================================================================== @@ -2294,7 +2176,7 @@ mod tests { #[cfg(feature = "connector-session")] #[tokio::test] - async fn pump_source_with_routes_through_the_given_router() { + async fn pump_source_routes_through_the_given_router() { struct Once(Option<(String, crate::Payload)>); impl crate::Source for Once { fn next(&mut self) -> crate::BoxFut<'_, Option<(String, crate::Payload)>> { @@ -2312,7 +2194,7 @@ mod tests { .await; let router = db.inbound_router("mqtt", &Plus).unwrap(); let source = Once(Some(("temp/a".into(), Arc::from(&b"x"[..])))); - for pump in crate::pump_source_with(&db, router, source) { + for pump in crate::pump_source(&db, router, source) { pump.await; } assert_eq!(last.load(Ordering::SeqCst), 0); diff --git a/aimdb-data-contracts/src/link_codec.rs b/aimdb-data-contracts/src/link_codec.rs index 6abd90a4..aa5a6b52 100644 --- a/aimdb-data-contracts/src/link_codec.rs +++ b/aimdb-data-contracts/src/link_codec.rs @@ -607,25 +607,22 @@ mod tests { }; assert_eq!(replacement_bytes, json_bytes); - let inbound = db.collect_inbound_routes("test"); - assert_eq!(inbound.len(), 2); - let json_ingest = inbound - .iter() - .find(|(topic, _)| topic == "json-in") - .map(|(_, ingest)| ingest) - .expect("JSON ingest route"); - json_ingest(&db.runtime_ctx(), &json_bytes).expect("JSON ingest"); + let inbound = db + .inbound_router("test", &aimdb_core::ExactGrammar) + .expect("inbound routes"); + assert_eq!(inbound.route_count(), 2); + let ctx = db.runtime_ctx(); + inbound + .route("json-in", &json_bytes, &ctx) + .expect("JSON ingest"); assert_eq!( json_in.lock().expect("JSON capture lock").as_ref(), Some(&reading) ); - let postcard_ingest = inbound - .iter() - .find(|(topic, _)| topic == "postcard-in") - .map(|(_, ingest)| ingest) - .expect("Postcard ingest route"); - postcard_ingest(&db.runtime_ctx(), &scratch[..len]).expect("Postcard ingest"); + inbound + .route("postcard-in", &scratch[..len], &ctx) + .expect("Postcard ingest"); assert_eq!( postcard_in.lock().expect("Postcard capture lock").as_ref(), Some(&reading) diff --git a/aimdb-knx-connector/src/connector.rs b/aimdb-knx-connector/src/connector.rs index 672866f0..f11bf947 100644 --- a/aimdb-knx-connector/src/connector.rs +++ b/aimdb-knx-connector/src/connector.rs @@ -16,7 +16,7 @@ use core::net::SocketAddr; use core::pin::Pin; use aimdb_core::connector::{ConnectorBuilder, ConnectorUrl}; -use aimdb_core::session::{pump_sink, pump_source_with, Payload}; +use aimdb_core::session::{pump_sink, pump_source, Payload}; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; use aimdb_core::{log_info, AimDb, DbError, DbResult, RuntimeOps}; @@ -178,7 +178,7 @@ where )); let mut futures: Vec = vec![task]; - futures.extend(pump_source_with( + futures.extend(pump_source( db, db.inbound_router("knx", &aimdb_core::ExactGrammar)?, KnxSource:: { diff --git a/aimdb-mqtt-connector/src/embedded/mod.rs b/aimdb-mqtt-connector/src/embedded/mod.rs index d8fcef49..ccb5848e 100644 --- a/aimdb-mqtt-connector/src/embedded/mod.rs +++ b/aimdb-mqtt-connector/src/embedded/mod.rs @@ -23,7 +23,6 @@ pub mod tls; extern crate alloc; use aimdb_core::connector::ConnectorUrl; -use aimdb_core::router::RouterBuilder; use aimdb_core::session::{pump_sink, pump_source, Payload}; use aimdb_core::transport::{ConnectorConfig, PublishError}; use alloc::boxed::Box; @@ -205,7 +204,8 @@ where + 'static, { Box::pin(async move { - let topics = inbound_topics(db); + let router = db.inbound_router("mqtt", &aimdb_core::ExactGrammar)?; + let topics = inbound_topics(&router); warn_unsupported_qos(db); let broker = parse_broker_url(broker_url)?; if broker.tls { @@ -222,7 +222,7 @@ where Settings::from_keep_alive_secs(keep_alive_secs), db.runtime_ops(), )?; - Ok(collect_pumps(db, actions, events, manager_tasks)) + Ok(collect_pumps(db, router, actions, events, manager_tasks)) }) } @@ -245,7 +245,8 @@ where + 'static, { Box::pin(async move { - let topics = inbound_topics(db); + let router = db.inbound_router("mqtt", &aimdb_core::ExactGrammar)?; + let topics = inbound_topics(&router); warn_unsupported_qos(db); let broker = parse_broker_url(broker_url)?; if !broker.tls { @@ -267,16 +268,14 @@ where Settings::from_keep_alive_secs(keep_alive_secs), db.runtime_ops(), )?; - Ok(collect_pumps(db, actions, events, manager_tasks)) + Ok(collect_pumps(db, router, actions, events, manager_tasks)) }) } /// The inbound topics the session must subscribe on every connection. -fn inbound_topics(db: &aimdb_core::builder::AimDb) -> Vec { - let inbound_routes = db.collect_inbound_routes("mqtt"); - let topics: Vec = RouterBuilder::from_routes(inbound_routes) - .build() - .resource_ids() +fn inbound_topics(router: &aimdb_core::Router) -> Vec { + let topics: Vec = router + .subscriptions() .iter() .map(|t| t.to_string()) .collect(); @@ -291,12 +290,13 @@ fn inbound_topics(db: &aimdb_core::builder::AimDb) -> Vec { /// join them. fn collect_pumps( db: &aimdb_core::builder::AimDb, + router: aimdb_core::Router, actions: Arc, events: Arc, manager_tasks: Vec, ) -> Vec { let mut futures = pump_sink(db, "mqtt", Arc::new(MqttSink { actions })); - futures.extend(pump_source(db, "mqtt", MqttSource { events })); + futures.extend(pump_source(db, router, MqttSource { events })); futures.extend(manager_tasks); futures } diff --git a/aimdb-mqtt-connector/src/native.rs b/aimdb-mqtt-connector/src/native.rs index 10725781..b617e42a 100644 --- a/aimdb-mqtt-connector/src/native.rs +++ b/aimdb-mqtt-connector/src/native.rs @@ -5,10 +5,9 @@ //! adapters that core's pumps drive. use aimdb_core::connector::ConnectorUrl; -use aimdb_core::router::{Router, RouterBuilder}; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; use aimdb_core::{log_debug, log_error, log_info}; -use aimdb_core::{pump_sink, pump_source, BoxFut, Payload, Source}; +use aimdb_core::{pump_sink, pump_source, BoxFut, ExactGrammar, Payload, Source}; use rumqttc::{AsyncClient, Event, EventLoop, MqttOptions, Packet}; use std::future::Future; use std::pin::Pin; @@ -27,14 +26,11 @@ pub(crate) fn build<'a>( keep_alive_secs: u16, ) -> Pin>> + Send + 'a>> { Box::pin(async move { - // Build a router from the inbound routes purely to drive the MQTT - // subscriptions + channel-capacity sizing in `build_internal`. The - // routing `Router` that fans incoming frames out to producers is - // (re)built by `pump_source` from the same `collect_inbound_routes`. - let inbound_routes = db.collect_inbound_routes("mqtt"); - let router = RouterBuilder::from_routes(inbound_routes).build(); + // One router both subscribes (here) and routes (`pump_source`). + let router = db.inbound_router("mqtt", &ExactGrammar)?; + let topics = router.subscriptions(); - log_info!("MQTT subscribing to {} topics", router.resource_ids().len()); + log_info!("MQTT subscribing to {} topics", topics.len()); // Connect, subscribe, and hand back the raw event loop. let (client, event_loop) = MqttConnectorImpl::build_internal( @@ -42,7 +38,7 @@ pub(crate) fn build<'a>( client_id, credentials, keep_alive_secs, - router, + &topics, ) .await .map_err(|e| { @@ -54,7 +50,7 @@ pub(crate) fn build<'a>( // Inbound: one multiplexed reader future fanning publishes out to producers. futures.extend(pump_source( db, - "mqtt", + router, MqttEventLoopSource { event_loop, broker_key: broker_url.to_string(), @@ -73,8 +69,8 @@ pub(crate) fn build<'a>( pub struct MqttConnectorImpl; impl MqttConnectorImpl { - /// Connect to the broker and subscribe to every topic in `router`, sizing - /// the send channel from the route count. + /// Connect to the broker and subscribe to `topics`, sizing the send + /// channel from their count. /// /// Returns the shared client (for the outbound `pump_sink`) plus the raw /// event loop (for [`MqttEventLoopSource`] and the inbound `pump_source`). @@ -84,7 +80,7 @@ impl MqttConnectorImpl { client_id: Option<&str>, credentials: Option<&(String, String)>, keep_alive_secs: u16, - router: Router, + topics: &[Arc], ) -> Result<(Arc, EventLoop), String> { // Parse the broker URL - we accept it with or without a topic let mut url = broker_url.to_string(); @@ -151,9 +147,7 @@ impl MqttConnectorImpl { return Err(no_tls_backend()); } - // Wrap router early so we can count topics for capacity calculation - let router_arc = Arc::new(router); - let topic_count = router_arc.resource_ids().len(); + let topic_count = topics.len(); // Dynamic channel capacity: scales with topic count. // @@ -177,11 +171,9 @@ impl MqttConnectorImpl { let (client, event_loop) = AsyncClient::new(mqtt_opts, channel_capacity); let client_arc = Arc::new(client); - let topics = router_arc.resource_ids(); - log_info!("Subscribing to {} MQTT topics...", topics.len()); - for topic in &topics { + for topic in topics { log_debug!("Subscribing to MQTT topic: {}", topic); client_arc @@ -346,31 +338,26 @@ fn no_tls_backend() -> String { #[cfg(test)] mod tests { use super::*; - use aimdb_core::router::RouterBuilder; #[tokio::test] async fn test_connector_creation_with_router() { - let router = RouterBuilder::new().build(); let connector = - MqttConnectorImpl::build_internal("mqtt://localhost:1883", None, None, 60, router) - .await; + MqttConnectorImpl::build_internal("mqtt://localhost:1883", None, None, 60, &[]).await; assert!(connector.is_ok()); } #[tokio::test] async fn test_connector_with_port() { - let router = RouterBuilder::new().build(); let connector = - MqttConnectorImpl::build_internal("mqtt://broker.local:9999", None, None, 60, router) + MqttConnectorImpl::build_internal("mqtt://broker.local:9999", None, None, 60, &[]) .await; assert!(connector.is_ok()); } #[tokio::test] async fn test_invalid_url() { - let router = RouterBuilder::new().build(); let connector = - MqttConnectorImpl::build_internal("not-a-valid-url", None, None, 60, router).await; + MqttConnectorImpl::build_internal("not-a-valid-url", None, None, 60, &[]).await; assert!(connector.is_err()); } @@ -378,13 +365,12 @@ mod tests { async fn test_connector_mqtts_url_with_credentials() { // mqtts:// with URL-embedded credentials must parse and build; the TLS // handshake itself only happens once the event loop is polled. - let router = RouterBuilder::new().build(); let connector = MqttConnectorImpl::build_internal( "mqtts://hub-sub:secret@broker.example.com:8883", None, None, 60, - router, + &[], ) .await; @@ -410,13 +396,12 @@ mod tests { /// The plain scheme is unaffected by which backend, if any, is selected. #[tokio::test] async fn test_connector_mqtt_url_needs_no_tls_backend() { - let router = RouterBuilder::new().build(); let connector = MqttConnectorImpl::build_internal( "mqtt://broker.example.com:1883", None, None, 60, - router, + &[], ) .await; assert!(connector.is_ok()); diff --git a/aimdb-mqtt-connector/tests/link_ext_tests.rs b/aimdb-mqtt-connector/tests/link_ext_tests.rs index 686efd4a..1e4076f1 100644 --- a/aimdb-mqtt-connector/tests/link_ext_tests.rs +++ b/aimdb-mqtt-connector/tests/link_ext_tests.rs @@ -119,11 +119,14 @@ async fn per_link_codec_preserves_mqtt_extensions_and_wiring() { let inbound_config = &record.inbound_connectors()[0].config; assert!(inbound_config.contains(&("qos".to_string(), "0".to_string()))); - let inbound = db.collect_inbound_routes("mqtt"); - assert_eq!(inbound.len(), 1); - assert_eq!(inbound[0].0, "commands/codec"); + let inbound = db + .inbound_router("mqtt", &aimdb_core::ExactGrammar) + .expect("inbound routes"); + assert_eq!(inbound.subscriptions(), [Arc::from("commands/codec")]); let encoded = link_codecs::Postcard::<64> .encode(&Reading { value: 17.5 }) .expect("Postcard encode must succeed"); - inbound[0].1(&db.runtime_ctx(), &encoded).expect("Postcard ingest must succeed"); + inbound + .route("commands/codec", &encoded, &db.runtime_ctx()) + .expect("Postcard ingest must succeed"); } diff --git a/aimdb-mqtt-connector/tests/topic_provider_tests.rs b/aimdb-mqtt-connector/tests/topic_provider_tests.rs index 14b04680..03602d41 100644 --- a/aimdb-mqtt-connector/tests/topic_provider_tests.rs +++ b/aimdb-mqtt-connector/tests/topic_provider_tests.rs @@ -465,7 +465,7 @@ fn test_connector_topic_resolution_with_threshold_provider() { /// Test that verifies the inbound topic resolver is called correctly #[test] fn test_inbound_topic_resolver_simulation() { - // Simulate what collect_inbound_routes does for TopicResolver + // Simulate what inbound_router does for TopicResolver fn resolve_inbound_topic( default_topic: &str, resolver: Option<&dyn Fn() -> Option>, From 8fcec93fc6b0f48c86d9c1017748536350db7315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 02:01:00 +0000 Subject: [PATCH 11/18] refactor(core)!: remove the grammar-less inbound route API (055) --- aimdb-core/src/router.rs | 2 +- docs/design/055-wildcard-inbound-links.md | 85 +++++++++++------------ 2 files changed, 40 insertions(+), 47 deletions(-) diff --git a/aimdb-core/src/router.rs b/aimdb-core/src/router.rs index 403586fc..8e08685c 100644 --- a/aimdb-core/src/router.rs +++ b/aimdb-core/src/router.rs @@ -59,7 +59,7 @@ pub(crate) struct CompiledRoute { impl CompiledRoute { /// A route comparing `topic` as a string. - #[cfg(test)] + #[cfg(all(test, feature = "connector-session"))] pub(crate) fn exact(topic: &str, ingest: IngestFn) -> Self { use crate::topic_pattern::{ExactGrammar, TopicPattern}; let pattern = TopicPattern::parse(topic).expect("an exact topic"); diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index a4a79cbc..3b65bb14 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -7,11 +7,11 @@ moved into connectors (§3.3), 2026-09-28 `aimdb-core`'s router, the matched topic and captures passed to a match-aware deserializer, optional per-record key interning surfaced in record metadata, a grammar trait through which each connector matches its -own topics, and the MQTT grammar in `aimdb-mqtt-connector`. `RuntimeContext` and `IngestFn` are -untouched; §5.8 lists the two public structs that gain fields. +own topics, and the MQTT grammar in `aimdb-mqtt-connector`. The connector +interface changes (§5.8); the user-facing link API does not. -**Independent of** [054](./054-zero-alloc-connector-boundary.md). This -design ships on today's interfaces; §7 describes what changes when 054 lands. +**Independent of** [054](./054-zero-alloc-connector-boundary.md). §7 +describes what changes when 054 lands. --- @@ -47,8 +47,7 @@ Two facts make a small change sufficient: `&str`, together with the full topic. - G3. Optionally, a capture becomes a **key**: a small integer assigned the first time a value is seen, from a bounded table per record. -- G4. No change to existing links, routers or connectors that do not use - patterns. +- G4. No behaviour change for existing links that do not use patterns. - G5. No per-message allocation on pattern routes, and none for a known key. - G6. Protocols with different wildcard rules (MQTT, Zenoh) plug in without changing core. @@ -103,16 +102,17 @@ tests. Everything below was measured on that spike, not estimated. 3. **Fewer checks run at `build()`.** "Capture shares a level with text" needs the separator, and "multi-level capture must be last" depends on the grammar. Both run when the connector builds (§5.3). -4. **Hand-written `+`/`#` stay broken on the old path.** Whether `+` is a - wildcard depends on the grammar, so `collect_inbound_routes` cannot spot - such links. Every in-tree connector moves to `inbound_router` (§5.2). +4. **One inbound path.** Whether `+` is a wildcard depends on the grammar, + so a route list built without one cannot tell. Every connector builds + its router with `inbound_router`, and the grammar-less path is removed + (§5.2). 5. **Keys are per record.** With per-link tables, two keyed links on one record handed out overlapping `KeyId`s. One table per record fixes that and matches the "array indexed by `KeyId`" guidance (§5.6). 6. **Key tables grow lazily.** Reserving full capacity cost 67 KB for 1,024 keys before any device appeared; lazy growth costs nothing measurable on the per-message path (§5.6). -7. **Two public structs gain fields** (§5.8), so "additive" needs a caveat. +7. **Not additive** (§5.8): the connector interface changes. 8. **Rules the first draft left implicit:** setting both deserializers is an error; a literal topic with a match-aware deserializer takes the pattern path; the dropped counter is `AtomicU32` (thumbv7em has no 64-bit @@ -219,8 +219,8 @@ pub struct MqttGrammar; // '/', "+", "#" last only, `$` hidden at level 0 ```rust impl AimDb { - /// Exact links as `collect_inbound_routes`, plus pattern links compiled - /// against `grammar`, with key tables attached. + /// Every link on `scheme`, compiled against `grammar`, with key tables + /// attached. pub fn inbound_router(&self, scheme: &str, grammar: &'static dyn TopicGrammar) -> DbResult; } @@ -231,27 +231,21 @@ impl Router { } /// Routes `src` with the router the connector subscribed from. -pub fn pump_source_with(db: &AimDb, router: Router, src: impl Source + 'static) +pub fn pump_source(db: &AimDb, router: Router, src: impl Source + 'static) -> Vec; -// pump_source(..) keeps its signature and behaviour. ``` - A connector subscribes and routes with the **same** router, so the two - cannot disagree. The spike replaced the separate - `RouterBuilder::from_routes(..)` calls in `native.rs` and - `embedded/mod.rs::inbound_topics`. -- The router keeps its grammar, which covering needs (§5.7). `Route` and - `collect_inbound_routes` are unchanged. + cannot disagree. +- The router keeps its grammar, which covering needs (§5.7). - **Every in-tree connector moves to `inbound_router`** in the same change: MQTT with `&MqttGrammar`; KNX, the WebSocket server and client, and core's AimX session client (TCP, UDS, serial) with `&ExactGrammar`. Only then is a `{…}` link on those connectors an error instead of a warning, and a hand-written `+` on MQTT starts working. -- `collect_inbound_routes` keeps its signature for out-of-tree connectors. - It returns exact links only and logs a warning for each `{…}` link it - skips. A hand-written `+`/`#` link without braces still goes through it - as an exact route: it subscribes and never matches, which is today's - behaviour. +- The grammar-less path is removed: `collect_inbound_routes`, + `RouterBuilder`, `Route` and `Router::new`. A `Router` comes only from + `inbound_router`, and `pump_source` and `pump_client` take it. ### 5.3 Validation @@ -292,10 +286,10 @@ over an exact one (§3.1). An index can come later if a benchmark asks for it. ### 5.5 The match reaches the deserializer as a borrow -Exact routes keep `IngestFn`. Pattern routes use a second ingest type: +Every route's ingest receives the match: ```rust -pub type MatchIngestFn = +pub type IngestFn = Arc, &[u8]) -> Result<(), String> + Send + Sync>; pub struct TopicMatch<'a> { /* topic: &'a str, spans, names, key */ } @@ -310,10 +304,9 @@ impl<'a> TopicMatch<'a> { - `Router::route(&self, topic: &str, ..)` already borrows the topic for the whole call, so the router builds `TopicMatch` on its stack. No allocation, no change to `RuntimeContext`, and the match cannot outlive its message. -- `with_match_deserializer` builds a `MatchIngestFn`. A `{…}` link with a - plain `with_deserializer` gets one that ignores the match. A **literal** - topic with `with_match_deserializer` becomes an all-literal pattern route - so the closure still receives the topic. +- `with_match_deserializer` passes the match to the closure; a plain + `with_deserializer` ignores it. An exact topic is a route whose filter + matches only itself, and the router compares it as a string. - The closure borrows the context (`&RuntimeContext`), so no reference count changes per message. `with_deserializer` takes it by value and clones an `Arc` per message; moving it to a borrow is a breaking change @@ -376,7 +369,7 @@ the connector, so it is uncontended. level. - Both backends subscribe `subscriptions()` of `db.inbound_router("mqtt", &MqttGrammar)` and pass that router to - `pump_source_with`. + `pump_source`. - **Covering set.** A filter is left out when another matches every topic it matches (`sensors/kitchen/temp` under `sensors/+/temp`; `a/+/b` under `a/#`). Required for backend parity (§3.1). The router still fans each @@ -395,19 +388,17 @@ the connector, so it is uncontended. ### 5.8 Compatibility -No existing function or type signature changes. Two public structs with -public fields gain fields, which breaks code that builds them as struct -literals: +A breaking change to the connector interface. Every connector lives in this +repository and moves with it; the user-facing link API (`link_from`, +`with_deserializer`) does not change. -- `InboundConnectorLink` gains `match_ingest_factory` and `key`. It has - `InboundConnectorLink::new`, and no in-tree code uses a literal. -- `RecordMetadata` gains `inbound_keys`. It has `RecordMetadata::new`; the - serde form is backward compatible. - -Mark both `#[non_exhaustive]` in the same change, so later fields are not -breaking. The attribute itself also breaks struct literals and exhaustive -destructuring outside the crate, so it ships in the same breaking change as -the fields. The new `InboundKeysInfo` is `#[non_exhaustive]` from the start. +- Removed: `AimDb::collect_inbound_routes`, `RouterBuilder`, `Route`, + `Router::new`. Connectors call `inbound_router`. +- `IngestFn` takes the `TopicMatch`. `pump_source` and `pump_client` take + the `Router`. +- `InboundConnectorLink` gains `key`, `RecordMetadata` gains + `inbound_keys` (the serde form stays backward compatible). Both become + `#[non_exhaustive]`; the new `InboundKeysInfo` is from the start. ## 6. Guidance for pattern records @@ -435,7 +426,7 @@ for the whole ingest call, exactly as `Router::route` does today. Because - `with_match_deserializer`'s closure signature and `TopicMatch<'a>` are unchanged. - `InboundDispatch::new` takes a `&'static dyn TopicGrammar` and replaces - `inbound_router` + `pump_source_with` for migrated connectors. + `inbound_router` + `pump_source` for migrated connectors. - 054 does not change `with_deserializer`; its by-value context stays until a later breaking release (§5.5). @@ -453,8 +444,10 @@ for the whole ingest call, exactly as `Router::route` does today. Because §3.3). Core would carry every protocol's matching rules, Zenoh's backtracking included, and a Zenoh connector could not reuse the `zenoh` crate's own key-expression logic. -5. **Change `IngestFn` to take the topic.** Clean, but breaking; 054 is the - breaking window that removes the need. +5. **Keeping the grammar-less route API** (`collect_inbound_routes`, + `RouterBuilder`) beside `inbound_router`, for connectors outside this + repository. There are none, and keeping it meant two ingest types and + two paths through the link builder. 6. **Match carried in `RuntimeContext` (`ctx.inbound_match()`).** Reaches plain `with_deserializer` users, but costs one allocation per message, lets the match outlive the message through a cloned context, and breaks From d5640519173482444f4336d95fb65bf8a676657c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 02:06:20 +0000 Subject: [PATCH 12/18] feat(mqtt): MqttGrammar (055) --- aimdb-mqtt-connector/src/grammar.rs | 336 ++++++++++++++++++++++++++++ aimdb-mqtt-connector/src/lib.rs | 4 + 2 files changed, 340 insertions(+) create mode 100644 aimdb-mqtt-connector/src/grammar.rs diff --git a/aimdb-mqtt-connector/src/grammar.rs b/aimdb-mqtt-connector/src/grammar.rs new file mode 100644 index 00000000..f3ea98e1 --- /dev/null +++ b/aimdb-mqtt-connector/src/grammar.rs @@ -0,0 +1,336 @@ +//! MQTT topic filters as a [`TopicGrammar`] (MQTT 3.1.1 §4.7). + +use aimdb_core::{PatternPart, Spans, TopicFilter, TopicGrammar, TopicPattern}; +use alloc::{boxed::Box, format, string::String, vec::Vec}; + +/// MQTT 3.1.1 §4.7: `/` separates levels, `+` matches one level, `#` the rest +/// including the parent level (`a/#` matches `a`) and only as the last level. +/// A wildcard is a whole level, and a leading wildcard does not match a `$…` +/// topic. +#[derive(Debug, Clone, Copy, Default)] +pub struct MqttGrammar; + +enum Level { + Literal(String), + /// `+` or `{name}`, with the capture number. + Single(Option), + /// `#` or `{name..}`, with the capture number. + Multi(Option), +} + +struct MqttFilter { + filter: String, + levels: Vec, +} + +/// A level as written: its text and, for `{…}`, the capture number and +/// whether it is `{name..}`. +#[derive(Default)] +struct RawLevel { + text: String, + capture: Option<(usize, bool)>, +} + +impl TopicGrammar for MqttGrammar { + fn compile(&self, pattern: &TopicPattern<'_>) -> Result, String> { + let topic = pattern.as_str(); + + let mut raw = Vec::from([RawLevel::default()]); + let mut captures = 0; + for part in pattern.parts() { + match part { + PatternPart::Text(text) => { + for (i, segment) in text.split('/').enumerate() { + if i > 0 { + raw.push(RawLevel::default()); + } + if let Some(level) = raw.last_mut() { + level.text.push_str(segment); + } + } + } + PatternPart::Capture { multi, .. } => { + if let Some(level) = raw.last_mut() { + level.capture = Some((captures, *multi)); + } + captures += 1; + } + } + } + + let last = raw.len() - 1; + let mut levels = Vec::with_capacity(raw.len()); + for (i, RawLevel { text, capture }) in raw.into_iter().enumerate() { + let level = match (capture, text.as_str()) { + (Some(_), t) if !t.is_empty() => { + return Err(format!( + "'{topic}': a capture must be a whole level, not part of one" + )) + } + (Some((n, false)), _) => Level::Single(Some(n)), + (Some((n, true)), _) => Level::Multi(Some(n)), + (None, "+") => Level::Single(None), + (None, "#") => Level::Multi(None), + (None, t) if t.contains(['+', '#']) => { + return Err(format!( + "'{topic}': a wildcard must be a whole level, not part of '{t}'" + )) + } + (None, _) => Level::Literal(text), + }; + if matches!(level, Level::Multi(_)) && i != last { + return Err(format!( + "'{topic}': a multi-level wildcard must be the last level" + )); + } + levels.push(level); + } + + let filter = levels + .iter() + .map(|level| match level { + Level::Literal(text) => text.as_str(), + Level::Single(_) => "+", + Level::Multi(_) => "#", + }) + .collect::>() + .join("/"); + Ok(Box::new(MqttFilter { filter, levels })) + } + + fn covers(&self, a: &str, b: &str) -> bool { + let (a, b): (Vec<&str>, Vec<&str>) = (a.split('/').collect(), b.split('/').collect()); + for i in 0.. { + match (a.get(i).copied(), b.get(i).copied()) { + // The rest of `b`, possibly nothing; at level 0 not a `$…` topic. + (Some("#"), first) => { + return i > 0 || !first.is_some_and(|l| l.starts_with('$')); + } + (Some("+"), Some(level)) if level != "#" => { + if i == 0 && level.starts_with('$') { + return false; + } + } + (Some(x), Some(y)) if x == y && x != "+" => {} + (None, None) => return true, + _ => return false, + } + } + false + } +} + +impl TopicFilter for MqttFilter { + fn filter(&self) -> &str { + &self.filter + } + + fn is_literal(&self) -> bool { + self.levels.iter().all(|l| matches!(l, Level::Literal(_))) + } + + fn matches(&self, topic: &str, spans: &mut Spans) -> bool { + // The router never passes more than `u16::MAX` bytes. + let span = |start: usize, end: usize| (start as u16, end as u16); + let mut rest = Some(topic); + let mut pos = 0; + + for (i, level) in self.levels.iter().enumerate() { + if let Level::Multi(capture) = level { + if i == 0 && topic.starts_with('$') { + return false; + } + if let Some(n) = capture { + let start = if rest.is_some() { pos } else { topic.len() }; + spans[*n] = span(start, topic.len()); + } + return true; + } + + let Some(r) = rest else { return false }; + let (text, next) = match r.split_once('/') { + Some((text, next)) => (text, Some(next)), + None => (r, None), + }; + match level { + Level::Literal(lit) if lit != text => return false, + Level::Single(capture) => { + if i == 0 && text.starts_with('$') { + return false; + } + if let Some(n) = capture { + spans[*n] = span(pos, pos + text.len()); + } + } + _ => {} + } + pos += text.len() + 1; + rest = next; + } + rest.is_none() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aimdb_core::MAX_CAPTURES; + use alloc::string::ToString; + + fn compile(topic: &str) -> Result, String> { + MqttGrammar.compile(&TopicPattern::parse(topic).map_err(|e| e.to_string())?) + } + + fn is_match(pattern: &str, topic: &str) -> bool { + compile(pattern) + .unwrap() + .matches(topic, &mut [(0, 0); MAX_CAPTURES]) + } + + /// Named captures of `topic`, or `None` when it does not match. + fn captures(pattern: &str, topic: &str) -> Option> { + let parsed = TopicPattern::parse(pattern).unwrap(); + let names = parsed.parts().iter().filter_map(|p| match p { + PatternPart::Capture { name, .. } => Some(name.to_string()), + PatternPart::Text(_) => None, + }); + let mut spans = [(0, 0); MAX_CAPTURES]; + let filter = MqttGrammar.compile(&parsed).unwrap(); + filter.matches(topic, &mut spans).then(|| { + names + .zip(spans) + .map(|(n, (s, e))| (n, topic[usize::from(s)..usize::from(e)].to_string())) + .collect() + }) + } + + fn caps(pairs: &[(&str, &str)]) -> Option> { + Some( + pairs + .iter() + .map(|(n, v)| (n.to_string(), v.to_string())) + .collect(), + ) + } + + #[test] + fn filter_renders_captures_as_wildcards() { + assert_eq!( + compile("sensors/{d}/temp").unwrap().filter(), + "sensors/+/temp" + ); + assert_eq!(compile("a/{rest..}").unwrap().filter(), "a/#"); + assert_eq!(compile("a/+/{x}/#").unwrap().filter(), "a/+/+/#"); + assert!(compile("a/b").unwrap().is_literal()); + assert!(!compile("a/+").unwrap().is_literal()); + } + + #[test] + fn compile_errors() { + for (topic, needle) in [ + ("sensors/dev-{id}", "capture must be a whole level"), + ("a/{x}y", "capture must be a whole level"), + ("a/{rest..}/b", "must be the last level"), + ("a/#/b", "must be the last level"), + ("sport/tennis#", "wildcard must be a whole level"), + ("sport+", "wildcard must be a whole level"), + ] { + let err = compile(topic).err().unwrap(); + assert!(err.contains(needle), "'{topic}': {err}"); + } + } + + // §4.7.1.2 + #[test] + fn multi_level_wildcard() { + let p = "sport/tennis/player1/#"; + assert!(is_match(p, "sport/tennis/player1")); + assert!(is_match(p, "sport/tennis/player1/ranking")); + assert!(is_match(p, "sport/tennis/player1/score/wimbledon")); + assert!(!is_match(p, "sport/tennis/player2")); + assert!(is_match("sport/#", "sport")); + assert!(is_match("#", "anything/at/all")); + } + + // §4.7.1.3 + #[test] + fn single_level_wildcard() { + assert!(is_match("sport/tennis/+", "sport/tennis/player1")); + assert!(!is_match("sport/tennis/+", "sport/tennis/player1/ranking")); + assert!(!is_match("sport/+", "sport")); + assert!(is_match("sport/+", "sport/")); + assert!(is_match("+/+", "/finance")); + assert!(is_match("/+", "/finance")); + assert!(!is_match("+", "/finance")); + assert!(is_match("+/tennis/#", "sport/tennis/x")); + } + + // §4.7.2 + #[test] + fn dollar_topics_are_hidden_from_leading_wildcards() { + assert!(!is_match("#", "$SYS/monitor/Clients")); + assert!(!is_match("+/monitor/Clients", "$SYS/monitor/Clients")); + assert!(is_match("$SYS/#", "$SYS/monitor/Clients")); + assert!(is_match("$SYS/monitor/+", "$SYS/monitor/Clients")); + assert!(is_match("a/+", "a/$x")); + } + + #[test] + fn captures_at_first_middle_and_last_level() { + assert_eq!( + captures("{d}/temp", "kitchen/temp"), + caps(&[("d", "kitchen")]) + ); + assert_eq!( + captures("sensors/{d}/temp", "sensors/kitchen/temp"), + caps(&[("d", "kitchen")]) + ); + assert_eq!( + captures("sensors/{d}", "sensors/kitchen"), + caps(&[("d", "kitchen")]) + ); + assert_eq!( + captures("{site}/+/{d}", "vienna/x/k1"), + caps(&[("site", "vienna"), ("d", "k1")]) + ); + assert_eq!(captures("sensors/{d}/temp", "sensors/kitchen/hum"), None); + } + + #[test] + fn rest_capture_takes_the_remainder_or_nothing() { + assert_eq!( + captures("logs/{rest..}", "logs/a/b/c"), + caps(&[("rest", "a/b/c")]) + ); + assert_eq!(captures("logs/{rest..}", "logs"), caps(&[("rest", "")])); + assert_eq!(captures("logs/{rest..}", "logs/"), caps(&[("rest", "")])); + assert_eq!(captures("logs/{rest..}", "other"), None); + assert_eq!(captures("a/{x}/b", "a//b"), caps(&[("x", "")])); + } + + #[test] + fn covering() { + let g = MqttGrammar; + assert!(g.covers("sensors/+/temp", "sensors/kitchen/temp")); + assert!(!g.covers("sensors/kitchen/temp", "sensors/+/temp")); + assert!(g.covers("a/#", "a")); + assert!(g.covers("a/#", "a/+/b")); + assert!(g.covers("a/#", "a/#")); + assert!(g.covers("a/+", "a/+")); + assert!(!g.covers("a/+", "a/#")); + assert!(!g.covers("a/+", "b/+")); + assert!(!g.covers("a/+", "a/+/c")); + assert!(!g.covers("a/+/c", "a/#")); + assert!(!g.covers("a/+", "a")); + } + + #[test] + fn covering_respects_dollar_topics() { + let g = MqttGrammar; + assert!(!g.covers("#", "$SYS/x")); + assert!(!g.covers("+/x", "$SYS/x")); + assert!(g.covers("$SYS/#", "$SYS/x")); + assert!(g.covers("#", "+/x")); + assert!(g.covers("sensors/+", "sensors/$x")); + } +} diff --git a/aimdb-mqtt-connector/src/lib.rs b/aimdb-mqtt-connector/src/lib.rs index c0984ad9..b5d1389f 100644 --- a/aimdb-mqtt-connector/src/lib.rs +++ b/aimdb-mqtt-connector/src/lib.rs @@ -111,6 +111,10 @@ extern crate alloc; // One `MqttConnector` over the `Native` and `Embedded` protocol backends. pub mod connector; +// MQTT topic filters for inbound links (works on every feature leg). +pub mod grammar; +pub use grammar::MqttGrammar; + // MQTT knobs over core's generic link builders (works on every feature leg). pub mod link_ext; pub use link_ext::{MqttLinkExt, MqttOutboundLinkExt}; From 32aafefef1710a9a9cf2a26e276602b94dcd08a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 02:13:11 +0000 Subject: [PATCH 13/18] feat(mqtt): route inbound links through MqttGrammar (055) --- aimdb-mqtt-connector/src/embedded/mod.rs | 4 +- aimdb-mqtt-connector/src/native.rs | 4 +- aimdb-mqtt-connector/tests/backend_parity.rs | 118 +++++++++++++++++++ 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/aimdb-mqtt-connector/src/embedded/mod.rs b/aimdb-mqtt-connector/src/embedded/mod.rs index ccb5848e..1201b07d 100644 --- a/aimdb-mqtt-connector/src/embedded/mod.rs +++ b/aimdb-mqtt-connector/src/embedded/mod.rs @@ -204,7 +204,7 @@ where + 'static, { Box::pin(async move { - let router = db.inbound_router("mqtt", &aimdb_core::ExactGrammar)?; + let router = db.inbound_router("mqtt", &crate::MqttGrammar)?; let topics = inbound_topics(&router); warn_unsupported_qos(db); let broker = parse_broker_url(broker_url)?; @@ -245,7 +245,7 @@ where + 'static, { Box::pin(async move { - let router = db.inbound_router("mqtt", &aimdb_core::ExactGrammar)?; + let router = db.inbound_router("mqtt", &crate::MqttGrammar)?; let topics = inbound_topics(&router); warn_unsupported_qos(db); let broker = parse_broker_url(broker_url)?; diff --git a/aimdb-mqtt-connector/src/native.rs b/aimdb-mqtt-connector/src/native.rs index b617e42a..af67fe04 100644 --- a/aimdb-mqtt-connector/src/native.rs +++ b/aimdb-mqtt-connector/src/native.rs @@ -7,7 +7,7 @@ use aimdb_core::connector::ConnectorUrl; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; use aimdb_core::{log_debug, log_error, log_info}; -use aimdb_core::{pump_sink, pump_source, BoxFut, ExactGrammar, Payload, Source}; +use aimdb_core::{pump_sink, pump_source, BoxFut, Payload, Source}; use rumqttc::{AsyncClient, Event, EventLoop, MqttOptions, Packet}; use std::future::Future; use std::pin::Pin; @@ -27,7 +27,7 @@ pub(crate) fn build<'a>( ) -> Pin>> + Send + 'a>> { Box::pin(async move { // One router both subscribes (here) and routes (`pump_source`). - let router = db.inbound_router("mqtt", &ExactGrammar)?; + let router = db.inbound_router("mqtt", &crate::MqttGrammar)?; let topics = router.subscriptions(); log_info!("MQTT subscribing to {} topics", topics.len()); diff --git a/aimdb-mqtt-connector/tests/backend_parity.rs b/aimdb-mqtt-connector/tests/backend_parity.rs index 6e7061c5..7e97e86d 100644 --- a/aimdb-mqtt-connector/tests/backend_parity.rs +++ b/aimdb-mqtt-connector/tests/backend_parity.rs @@ -6,6 +6,7 @@ //! payloads on the wire. #![cfg(feature = "_test-backend-parity")] +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -183,6 +184,123 @@ async fn both_backends_round_trip_against_one_broker() { ); } +fn parse(data: &[u8]) -> Result { + core::str::from_utf8(data) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .ok_or_else(|| String::from("bad payload")) +} + +/// Deserializer calls per record. +#[derive(Default)] +struct Calls { + pattern: AtomicUsize, + exact: AtomicUsize, +} + +/// A keyed pattern record beside an exact record whose topic it covers. +fn build_pattern_db( + connector: impl aimdb_core::ConnectorBuilder + 'static, + calls: Arc, +) -> impl std::future::Future { + let mut builder = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .with_connector(connector); + + let pattern_calls = calls.clone(); + builder.configure::("readings", move |reg| { + reg.buffer(BufferCfg::SingleLatest) + .link_from("mqtt://parity/{device}/in") + .key("device", 4) + .with_match_deserializer(move |_ctx, m, data: &[u8]| { + pattern_calls.pattern.fetch_add(1, Ordering::SeqCst); + match (m.get("device"), m.key().map(|k| k.index())) { + (Some("kitchen"), Some(0)) => parse(data), + other => Err(format!("unexpected match {other:?}")), + } + }) + .finish(); + }); + + builder.configure::("kitchen", move |reg| { + reg.buffer(BufferCfg::SingleLatest) + .link_from("mqtt://parity/kitchen/in") + .with_deserializer(move |_ctx, data: &[u8]| { + calls.exact.fetch_add(1, Ordering::SeqCst); + parse(data) + }) + .finish(); + }); + + async move { builder.build().await.expect("build db") } +} + +/// A pattern link beside an exact link it covers: each backend subscribes only +/// the covering filter, each record receives the broker's one PUBLISH once, +/// and the capture and key reach the deserializer. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pattern_link_covers_exact_link_on_both_backends() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let url = format!("mqtt://127.0.0.1:{port}"); + let seen = Arc::new(Mutex::new(Seen::default())); + + let native = MqttConnector::new(url.clone()).with_client_id("pattern-native"); + let embedded = MqttConnector::new(url) + .transport(TokioNet::tcp()) + .with_client_id("pattern-embedded"); + let (native_calls, embedded_calls) = (Arc::new(Calls::default()), Arc::new(Calls::default())); + let (native_db, native_runner) = build_pattern_db(native, native_calls.clone()).await; + let (embedded_db, embedded_runner) = build_pattern_db(embedded, embedded_calls.clone()).await; + + let mut inbound = Vec::new(); + for db in [&native_db, &embedded_db] { + for record in ["readings", "kitchen"] { + inbound.push(db.consumer::(record).expect("consumer").subscribe()); + } + } + + let broker = fake_broker_concurrent(listener, seen.clone(), Some(("parity/kitchen/in", b"7"))); + let values = tokio::select! { + _ = native_runner.run() => panic!("the native runner returned"), + _ = embedded_runner.run() => panic!("the embedded runner returned"), + _ = broker => panic!("the broker returned"), + values = async { + let mut values = Vec::new(); + for reader in &mut inbound { + values.push(reader.recv().await.expect("inbound")); + } + // Room for a second delivery, which must not come. + tokio::time::sleep(Duration::from_millis(200)).await; + values + } => values, + _ = tokio::time::sleep(Duration::from_secs(30)) => { + let seen = seen.lock().unwrap(); + panic!("watchdog: {:?} subscribed", seen.subscribed_topics()); + } + }; + + assert_eq!(values, [7, 7, 7, 7], "every record on both backends gets 7"); + for (backend, calls) in [("native", &native_calls), ("embedded", &embedded_calls)] { + assert_eq!( + calls.pattern.load(Ordering::SeqCst), + 1, + "{backend} pattern link" + ); + assert_eq!( + calls.exact.load(Ordering::SeqCst), + 1, + "{backend} exact link" + ); + } + let seen = seen.lock().unwrap(); + assert_eq!( + seen.subscribed_topics(), + ["parity/+/in", "parity/+/in"], + "each backend subscribes only the covering filter" + ); +} + /// `with_credentials` reaches the wire on both backends. /// /// A setter that was accepted and then dropped would look exactly like success, From 9bd25465cf959e5d63f4529c24d069641104c913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 28 Sep 2026 02:24:30 +0000 Subject: [PATCH 14/18] chore: wrap up 055 wildcard inbound links --- CHANGELOG.md | 24 ++++ Cargo.lock | 1 + aimdb-bench/Cargo.toml | 2 + aimdb-bench/benches/b0_alloc_connector.rs | 118 ++++++++++++++++-- .../data/baselines/b0_alloc_connector.json | 27 ++++ aimdb-core/CHANGELOG.md | 31 +++++ aimdb-mqtt-connector/CHANGELOG.md | 16 +++ docs/design/055-wildcard-inbound-links.md | 4 +- 8 files changed, 209 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 520c5436..620b1899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added — Design 055: wildcard inbound links + +- **One inbound link can feed many topics into one record.** + `link_from("mqtt://sensors/{device}/temp")` matches every device; + `with_match_deserializer(|ctx, m, bytes| …)` sees the topic and its captures + (`m.get("device")`), and `.key("device", 1024)` turns a capture into a small + `KeyId` from a bounded table per record, reported in record metadata + (`inbound_keys`). Routing a pattern, and a known key, allocates nothing. + ([aimdb-core](aimdb-core/CHANGELOG.md)) +- **MQTT understands topic filters.** `MqttGrammar` implements MQTT 3.1.1 + §4.7; both backends subscribe only filters no other filter covers, so the + MQTT 3.1.1 and MQTT 5 backends receive an overlapping topic once each, and a + hand-written `+`/`#` topic now matches. + ([aimdb-mqtt-connector](aimdb-mqtt-connector/CHANGELOG.md)) + +### Changed (breaking) — one inbound path + +Every connector builds its router with `AimDb::inbound_router(scheme, +grammar)`; `collect_inbound_routes`, `RouterBuilder`, `Route` and the public +`Router::new` are gone, `IngestFn` receives the `TopicMatch`, and `pump_source` +and `pump_client` take the router. KNX, WebSocket, TCP, UDS and serial use +`ExactGrammar`: a `{…}` link on them fails the build. The user-facing link API +is unchanged. ([aimdb-core](aimdb-core/CHANGELOG.md)) + ## [2.0.0] - 2026-09-18 ### Added diff --git a/Cargo.lock b/Cargo.lock index acbe12ba..265f74df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,7 @@ dependencies = [ "aimdb-core", "aimdb-data-contracts", "aimdb-embassy-adapter", + "aimdb-mqtt-connector", "aimdb-tokio-adapter", "criterion", "critical-section", diff --git a/aimdb-bench/Cargo.toml b/aimdb-bench/Cargo.toml index fcb2645b..06136f8d 100644 --- a/aimdb-bench/Cargo.toml +++ b/aimdb-bench/Cargo.toml @@ -154,3 +154,5 @@ critical-section = { version = "1.1", features = ["std"] } defmt = { workspace = true } embassy-time-driver = "0.2.2" postcard = { version = "1.0", default-features = false, features = ["alloc"] } +# `MqttGrammar` for the pattern rows of b0_alloc_connector (no backend). +aimdb-mqtt-connector = { path = "../aimdb-mqtt-connector", default-features = false } diff --git a/aimdb-bench/benches/b0_alloc_connector.rs b/aimdb-bench/benches/b0_alloc_connector.rs index b64bf02b..45778fcd 100644 --- a/aimdb-bench/benches/b0_alloc_connector.rs +++ b/aimdb-bench/benches/b0_alloc_connector.rs @@ -3,7 +3,9 @@ //! Baseline for design 054. Measures what AimDB's connector interfaces cost //! per message, independent of any real transport: //! -//! - **Inbound:** `Router::route` alone, and the real `pump_source` driven by +//! - **Inbound:** `Router::route` alone — for an exact topic, a pattern +//! (`{device}`, MQTT grammar), and a keyed pattern with a known and a new +//! key — and the real `pump_source` driven by //! the smallest possible `Source` (it clones a pre-built topic `String` and //! payload `Arc` — the least any `Source` can do, since the trait returns //! owned values). @@ -36,6 +38,7 @@ use aimdb_core::connector::{ use aimdb_core::session::{pump_source, Payload, Source}; use aimdb_core::transport::{Connector, ConnectorConfig, PublishError}; use aimdb_core::{AimDb, AimDbBuilder, BoxFut, DbResult, ExactGrammar, RuntimeContext, StringKey}; +use aimdb_mqtt_connector::MqttGrammar; use aimdb_tokio_adapter::{TokioAdapter, TokioRecordRegistrarExt}; #[global_allocator] @@ -46,11 +49,16 @@ const WARMUP_ITERS: usize = 100; const MEASURE_ITERS: usize = 2_000; const DECOY_ROUTES: usize = 63; const SCRATCH_CAPACITY: usize = 64; +/// Room for a new key on every warm-up and measured message. +const KEY_CAPACITY: u16 = 4096; /// Allocations per message on `main` when this bench was added. Update /// together with `data/baselines/b0_alloc_connector.json`. const EXPECTED: &[(&str, u64)] = &[ ("inbound_route", 0), + ("inbound_route_pattern", 0), + ("inbound_route_keyed_known", 0), + ("inbound_route_keyed_new", 1), ("inbound_pump_source_minimal", 2), ("outbound_scratch_static_topic", 2), ("outbound_scratch_dynamic_topic", 3), @@ -138,8 +146,20 @@ async fn build_db(configure: impl FnOnce(&mut AimDbBuilder)) -> AimDb { // --- Inbound ---------------------------------------------------------------- -/// One linked record plus `DECOY_ROUTES` others, so routing scans a realistic -/// table. +/// `DECOY_ROUTES` exact links, so routing scans a realistic table. +fn add_decoys(b: &mut AimDbBuilder) { + for i in 0..DECOY_ROUTES { + let topic = format!("bench://in/decoy/{i}"); + b.configure::(StringKey::intern(format!("in.decoy{i}")), |reg| { + reg.buffer(BufferCfg::SingleLatest) + .link_from(&topic) + .with_deserializer(|_ctx, _bytes| Ok(reading(0))) + .finish(); + }); + } +} + +/// One linked record plus the decoys. async fn inbound_db() -> AimDb { build_db(|b| { b.configure::("in.target", |reg| { @@ -148,19 +168,63 @@ async fn inbound_db() -> AimDb { .with_deserializer(|_ctx, bytes| Ok(reading(bytes[0] as usize))) .finish(); }); - for i in 0..DECOY_ROUTES { - let topic = format!("bench://in/decoy/{i}"); - b.configure::(StringKey::intern(format!("in.decoy{i}")), |reg| { - reg.buffer(BufferCfg::SingleLatest) - .link_from(&topic) - .with_deserializer(|_ctx, _bytes| Ok(reading(0))) - .finish(); - }); - } + add_decoys(b); }) .await } +/// One record on `in/{device}/target`, keyed or not, plus the decoys. +async fn pattern_db(keyed: bool) -> AimDb { + build_db(|b| { + b.configure::("in.pattern", move |reg| { + let link = reg + .buffer(BufferCfg::SpmcRing { capacity: 64 }) + .link_from("bench://in/{device}/target"); + let link = if keyed { + link.key("device", KEY_CAPACITY) + } else { + link + }; + link.with_match_deserializer(|_ctx, m, bytes| { + Ok(reading( + bytes[0] as usize + m.key().map_or(0, |k| k.index()), + )) + }) + .finish(); + }); + add_decoys(b); + }) + .await +} + +/// Routes `warmup` then `measured`, counting only the second. +async fn measure_pattern_route(keyed: bool, warmup: &[String], measured: &[String]) -> (u64, u64) { + let db = pattern_db(keyed).await; + let ctx = db.runtime_ctx(); + let router = db.inbound_router(SCHEME, &MqttGrammar).unwrap(); + let payload = [1u8; 8]; + for topic in warmup { + router.route(topic, &payload, &ctx).unwrap(); + } + reset(); + for topic in measured { + router + .route(black_box(topic), black_box(&payload), &ctx) + .unwrap(); + } + snapshot() +} + +/// `n` copies of one topic, or `n` topics each naming a new device. +fn pattern_topics(n: usize, first_device: usize, distinct: bool) -> Vec { + (0..n) + .map(|i| { + let device = if distinct { first_device + i } else { 0 }; + format!("in/dev{device}/target") + }) + .collect() +} + async fn measure_route() -> (u64, u64) { let db = inbound_db().await; let ctx = db.runtime_ctx(); @@ -318,6 +382,36 @@ fn main() { let measured: Vec<(&str, &str, (u64, u64))> = runtime.block_on(async { vec![ ("inbound_route", "SpmcRing", measure_route().await), + ( + "inbound_route_pattern", + "SpmcRing", + measure_pattern_route( + false, + &pattern_topics(WARMUP_ITERS, 0, false), + &pattern_topics(MEASURE_ITERS, 0, false), + ) + .await, + ), + ( + "inbound_route_keyed_known", + "SpmcRing", + measure_pattern_route( + true, + &pattern_topics(WARMUP_ITERS, 0, false), + &pattern_topics(MEASURE_ITERS, 0, false), + ) + .await, + ), + ( + "inbound_route_keyed_new", + "SpmcRing", + measure_pattern_route( + true, + &pattern_topics(WARMUP_ITERS, 0, true), + &pattern_topics(MEASURE_ITERS, WARMUP_ITERS, true), + ) + .await, + ), ( "inbound_pump_source_minimal", "SpmcRing", diff --git a/aimdb-bench/data/baselines/b0_alloc_connector.json b/aimdb-bench/data/baselines/b0_alloc_connector.json index 3afe97e6..fbd67fcf 100644 --- a/aimdb-bench/data/baselines/b0_alloc_connector.json +++ b/aimdb-bench/data/baselines/b0_alloc_connector.json @@ -8,6 +8,33 @@ "allocs_per_msg": 0.0, "bytes_per_msg": 0.0 }, + { + "profile": "inbound_route_pattern", + "buffer_type": "SpmcRing", + "total_allocs": 0, + "total_bytes": 0, + "batch_size": 2000, + "allocs_per_msg": 0.0, + "bytes_per_msg": 0.0 + }, + { + "profile": "inbound_route_keyed_known", + "buffer_type": "SpmcRing", + "total_allocs": 0, + "total_bytes": 0, + "batch_size": 2000, + "allocs_per_msg": 0.0, + "bytes_per_msg": 0.0 + }, + { + "profile": "inbound_route_keyed_new", + "buffer_type": "SpmcRing", + "total_allocs": 2010, + "total_bytes": 373456, + "batch_size": 2000, + "allocs_per_msg": 1.005, + "bytes_per_msg": 186.728 + }, { "profile": "inbound_pump_source_minimal", "buffer_type": "SpmcRing", diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 981610de..41808ff9 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -7,8 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Topic patterns on inbound links (design 055).** `{name}` captures one + level, `{name..}` the rest; the syntax is checked at `build()`, and the + connector's `TopicGrammar` compiles each pattern into a `TopicFilter` when it + builds. `ExactGrammar` is the grammar for connectors without wildcards. +- **`InboundConnectorBuilder::with_match_deserializer`** passes a `TopicMatch` + (`topic()`, `get(name)`, `key()`) borrowed from the router's stack, and a + borrowed `&RuntimeContext`, so no reference count changes per message. +- **`.key(name, capacity)`** interns a capture into a `KeyId` from one table + per record, shared by all its keyed links. The table grows as values arrive; + when full, the message is dropped and counted. `AimDb::inbound_key_name` + resolves a key; `RecordMetadata::inbound_keys` (`InboundKeysInfo`) reports + captures, capacity, assigned and dropped. +- **`AimDb::inbound_router(scheme, grammar)`** compiles a scheme's links, + including patterns a `TopicResolverFn` returns, and reports every link it + cannot compile at once. `Router::subscriptions()` lists the filters to + subscribe, without those another filter covers. + ### Changed (breaking, API) +- **One inbound path.** Removed `AimDb::collect_inbound_routes`, + `RouterBuilder`, `Route` and the public `Router::new`; a `Router` comes from + `inbound_router`. `IngestFn` takes the `TopicMatch`, and + `InboundConnectorLink` has one `ingest_factory` of that type. + `pump_source(db, router, src)` and `pump_client(db, scheme, router, handle)` + take the router, so a connector subscribes and routes with the same one. +- **`InboundConnectorLink` gains `key`, `RecordMetadata` gains + `inbound_keys`; both are now `#[non_exhaustive]`.** The serde form of + `RecordMetadata` stays backward compatible. +- **Outbound links reject `{…}` topics** at `build()`: a filter cannot be + published to. + - **`ConnectorConfig` gains `record_index: Option`**, the id of the record an outbound publish comes from — its registration index, the same `record_id` `AimDb::list_records` reports. A topic alone cannot identify the diff --git a/aimdb-mqtt-connector/CHANGELOG.md b/aimdb-mqtt-connector/CHANGELOG.md index adc757cf..4f0f62e0 100644 --- a/aimdb-mqtt-connector/CHANGELOG.md +++ b/aimdb-mqtt-connector/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`MqttGrammar`**: MQTT 3.1.1 §4.7 topic filters for inbound links. `+` and + `{name}` match one level, `#` and `{name..}` the rest (last only, including + the parent level); a leading wildcard does not match a `$…` topic. + +### Changed + +- **Both backends route through `inbound_router("mqtt", &MqttGrammar)`** and + subscribe `subscriptions()`: a filter another one covers is not subscribed, + so the MQTT 3.1.1 (`Native`) and MQTT 5 (`Embedded`) backends each receive an + overlapping topic once. A hand-written `+`/`#` topic now matches; it used to + subscribe and never deliver. +- **The `with_qos` doc no longer claims an inbound subscribe QoS.** Inbound + subscriptions stay at QoS 1, as before. + ## [0.7.0] - 2026-09-18 ### Changed (breaking) diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 3b65bb14..13aacd18 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -1,6 +1,6 @@ # 055 — Wildcard inbound links -**Status:** 📝 Proposed — validated by a spike (§3), 2026-09-27; matching +**Status:** ✅ Implemented — validated by a spike (§3), 2026-09-27; matching moved into connectors (§3.3), 2026-09-28 **Scope:** inbound links whose topic is a pattern: matching in @@ -505,7 +505,7 @@ None. `inbound_route_keyed_known` (0) and `inbound_route_keyed_new` (1). Existing rows unchanged. 7. `weather-station-gamma` and the embedded MQTT demo build for - `thumbv7em-none-eabihf` with no behaviour change. + `thumbv8m.main-none-eabihf` with no behaviour change. ## 11. References From 12df4cedbe54b51ffaf8a69520a68067da7051d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 29 Sep 2026 01:33:41 +0000 Subject: [PATCH 15/18] fix(mqtt): ensure captures are whole levels in MqttGrammar --- aimdb-core/src/router.rs | 3 ++- aimdb-mqtt-connector/src/grammar.rs | 7 +++++++ docs/design/055-wildcard-inbound-links.md | 5 ++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/aimdb-core/src/router.rs b/aimdb-core/src/router.rs index 8e08685c..88eee71b 100644 --- a/aimdb-core/src/router.rs +++ b/aimdb-core/src/router.rs @@ -136,8 +136,9 @@ impl Router { // Linear search through all routes // Note: Multiple routes may match the same resource_id (different types) - let mut spans: Spans = [(0, 0); MAX_CAPTURES]; for route in &self.routes { + // Fresh per route: a failed match may leave partial spans. + let mut spans: Spans = [(0, 0); MAX_CAPTURES]; if route.matches(resource_id, &mut spans) { matched = true; let Some(key) = route.key(resource_id, &spans) else { diff --git a/aimdb-mqtt-connector/src/grammar.rs b/aimdb-mqtt-connector/src/grammar.rs index f3ea98e1..1687ef34 100644 --- a/aimdb-mqtt-connector/src/grammar.rs +++ b/aimdb-mqtt-connector/src/grammar.rs @@ -51,6 +51,11 @@ impl TopicGrammar for MqttGrammar { } PatternPart::Capture { multi, .. } => { if let Some(level) = raw.last_mut() { + if level.capture.is_some() { + return Err(format!( + "'{topic}': a capture must be a whole level, not part of one" + )); + } level.capture = Some((captures, *multi)); } captures += 1; @@ -230,6 +235,8 @@ mod tests { for (topic, needle) in [ ("sensors/dev-{id}", "capture must be a whole level"), ("a/{x}y", "capture must be a whole level"), + ("a/{x}{y}", "capture must be a whole level"), + ("a/{x}{y..}", "capture must be a whole level"), ("a/{rest..}/b", "must be the last level"), ("a/#/b", "must be the last level"), ("sport/tennis#", "wildcard must be a whole level"), diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index 13aacd18..ceea585c 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -373,7 +373,10 @@ the connector, so it is uncontended. - **Covering set.** A filter is left out when another matches every topic it matches (`sensors/kitchen/temp` under `sensors/+/temp`; `a/+/b` under `a/#`). Required for backend parity (§3.1). The router still fans each - message out to every route. + message out to every route. Filters that overlap only partly (`r/+/c` + and `r/b/+`) both stay, so the embedded backend (MQTT 5) receives such a + message once per filter and the native one (MQTT 3.1.1) once: a known + difference. - `MqttGrammar::covers` compares level by level. A wildcard covers a literal level only where it may match it: `#` and `+/x` do not cover `$SYS/x`, because the broker never delivers `$…` topics to a leading From 703f421469c9a5cc3643312871af8c69d7d2839d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 29 Sep 2026 01:39:06 +0000 Subject: [PATCH 16/18] docs: clarify dropped message behavior in metadata and design documents --- aimdb-core/src/inbound_key.rs | 3 ++- aimdb-core/src/remote/metadata.rs | 3 ++- docs/design/055-wildcard-inbound-links.md | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/aimdb-core/src/inbound_key.rs b/aimdb-core/src/inbound_key.rs index 8304b528..48da3f53 100644 --- a/aimdb-core/src/inbound_key.rs +++ b/aimdb-core/src/inbound_key.rs @@ -93,7 +93,8 @@ impl KeyTable { lock(&self.keys).names.len() } - /// Messages turned away because the table was full. + /// Messages turned away because the table was full, once per matching + /// keyed link. #[cfg(any(test, feature = "remote"))] pub(crate) fn dropped(&self) -> u32 { self.dropped.load(Ordering::Relaxed) diff --git a/aimdb-core/src/remote/metadata.rs b/aimdb-core/src/remote/metadata.rs index 92625815..ee9f1637 100644 --- a/aimdb-core/src/remote/metadata.rs +++ b/aimdb-core/src/remote/metadata.rs @@ -138,7 +138,8 @@ pub struct InboundKeysInfo { pub captures: Vec, pub capacity: u16, pub assigned: usize, - /// Messages turned away because the table was full. + /// Messages turned away because the table was full, once per matching + /// keyed link. pub dropped: u32, } diff --git a/docs/design/055-wildcard-inbound-links.md b/docs/design/055-wildcard-inbound-links.md index ceea585c..0b03d239 100644 --- a/docs/design/055-wildcard-inbound-links.md +++ b/docs/design/055-wildcard-inbound-links.md @@ -332,7 +332,8 @@ pub struct KeyId(NonZeroU16); // Option is 2 bytes; .index() is 0-based - A new value costs one allocation (its name); a known value costs none. - A `{name..}` capture can be a key; its value is the whole remainder. - **When the table is full**, the message is dropped and the table's - `dropped` counter (`AtomicU32`) increases. A message that reaches the + `dropped` counter (`AtomicU32`) increases, once per matching keyed + link. A message that reaches the deserializer of a keyed link always has `m.key() == Some(_)`. - Keys are never reused while the process runs. - `db.inbound_key_name("sensors.readings", key) -> Option>` From 7586a2fe5431b6580adaca0fcb15f1989d79a4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 29 Sep 2026 01:40:11 +0000 Subject: [PATCH 17/18] fix(docs): correct example in builder documentation for inbound router --- aimdb-websocket-connector/src/client/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index ae0db84c..5d898b8a 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -9,7 +9,7 @@ //! ```text //! AimDbBuilder::build() //! └─ WsClientConnectorBuilder::build(&db) -//! ├─ db.collect_inbound_routes("ws-client") → Router +//! ├─ db.inbound_router("ws-client", &ExactGrammar) → Router //! ├─ db.collect_outbound_routes("ws-client") → outbound futures //! ├─ connect to remote WebSocket server //! ├─ build connector_future (read + write + keepalive + reconnect) From 0ad07f77cf1dd8f6e9a623a5bcb44288c9ed83bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 29 Sep 2026 01:47:38 +0000 Subject: [PATCH 18/18] fix(mqtt): update error messages for outbound topic pattern restrictions --- aimdb-core/CHANGELOG.md | 2 ++ aimdb-core/src/inbound_key.rs | 3 ++- aimdb-core/src/typed_api.rs | 10 ++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 41808ff9..df7f5dd1 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `RecordMetadata` stays backward compatible. - **Outbound links reject `{…}` topics** at `build()`: a filter cannot be published to. +- **`{` and `}` in topics are pattern syntax** on every connector, with no + escape: a topic containing a literal brace can no longer be linked. - **`ConnectorConfig` gains `record_index: Option`**, the id of the record an outbound publish comes from — its registration index, the same diff --git a/aimdb-core/src/inbound_key.rs b/aimdb-core/src/inbound_key.rs index 48da3f53..25abac78 100644 --- a/aimdb-core/src/inbound_key.rs +++ b/aimdb-core/src/inbound_key.rs @@ -21,7 +21,8 @@ fn lock(m: &Mutex) -> spin::MutexGuard<'_, T> { m.lock() } -/// A capture value's key, assigned the first time the value is seen. +/// A capture value's key, assigned the first time the value is seen. Only +/// meaningful for the record whose link produced it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct KeyId(NonZeroU16); diff --git a/aimdb-core/src/typed_api.rs b/aimdb-core/src/typed_api.rs index fbb2a51b..e677ca3a 100644 --- a/aimdb-core/src/typed_api.rs +++ b/aimdb-core/src/typed_api.rs @@ -905,11 +905,11 @@ where let url_string = url.to_string(); let scheme = url.scheme().to_string(); - if crate::TopicPattern::parse(url.resource_id()).is_ok_and(|p| p.has_captures()) { + if url.resource_id().contains(['{', '}']) { self.registrar.rec.push_config_error(ConfigError::new( record_key, Some(self.url), - "Outbound links cannot use topic patterns", + "Outbound topics cannot contain '{' or '}'", )); return self.registrar; } @@ -1146,7 +1146,9 @@ where /// Assigns each value of capture `name` a [`KeyId`](crate::KeyId), up to /// `capacity` values per record. Messages with a value beyond that are - /// dropped. + /// dropped. A value keeps its key even if its payload fails to + /// deserialize, and keys are never freed, so anyone who can publish under + /// the pattern can fill the table. pub fn key(mut self, name: &str, capacity: u16) -> Self { self.key = Some((name.to_string(), capacity)); self @@ -1587,7 +1589,7 @@ mod tests { assert!(rec.outbound_connectors().is_empty()); let errors = drain_errors(&mut rec); - assert!(errors[0].message.contains("cannot use topic patterns")); + assert!(errors[0].message.contains("cannot contain '{' or '}'")); } // ====================================================================