From 79eb631a120fdd001da9da2c2f18cf726d591907 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:01:05 +0000 Subject: [PATCH 01/25] refactor: split backend deployment plans and bind stored outputs --- Cargo.lock | 22 +- Cargo.toml | 7 +- README.md | 4 +- .../examples/calibration_candidates.rs | 6 +- .../examples/workload_cost_manifest.rs | 4 +- control_plane/src/clickhouse.rs | 34 +- control_plane/src/physical/backend_stage.rs | 2 +- control_plane/src/physical/compiler.rs | 298 +++++++------ control_plane/src/physical/erp.rs | 7 +- .../src/physical/executable_binding.rs | 2 +- control_plane/src/physical/realization.rs | 6 +- control_plane/src/physical/workload_cost.rs | 18 +- control_plane/src/query_plan.rs | 14 + control_plane/src/query_plan/residual.rs | 75 ++-- crates/asap_sketch_codec/Cargo.toml | 8 + crates/asap_sketch_codec/src/lib.rs | 84 ++++ crates/asap_types/src/derived_input.rs | 12 +- crates/asap_types/src/executable_plan.rs | 126 +++++- crates/asap_types/src/plan_publication.rs | 135 ++++-- crates/asap_types/src/precompute_plan.rs | 12 +- .../asap_types/src/precompute_plan/catalog.rs | 9 +- crates/asap_types/src/producer_plan.rs | 2 +- crates/asap_types/src/query_plan.rs | 63 ++- crates/asap_types/src/sds.rs | 146 ++++++- crates/asap_types/src/summary_catalog.rs | 147 ++----- data_plane/Cargo.toml | 4 +- .../examples/audit_clickhouse_fallback.rs | 1 + data_plane/src/drivers/ingest/otel.rs | 86 +--- .../drivers/ingest/prometheus_remote_write.rs | 3 + data_plane/src/drivers/query/servers/http.rs | 118 ++++-- data_plane/src/main.rs | 2 +- .../precompute_engine/maintenance_runtime.rs | 43 +- .../operators/datasketches_kll_accumulator.rs | 60 +-- .../operators/dd_sketch_accumulator.rs | 45 +- .../operators/edge_runtime_adapter.rs | 391 ------------------ .../src/precompute_engine/operators/mod.rs | 1 - .../src/precompute_engine/subdag_scheduler.rs | 59 ++- .../accelerator.rs | 4 +- .../asap_query_engine/catalog_resolver.rs | 10 +- .../asap_query_engine/exact_subqueries.rs | 4 + .../asap_query_engine/live_serve.rs | 4 + .../asap_query_engine/post_asap_readout.rs | 23 +- .../asap_query_engine/summary_executor.rs | 21 +- .../asap_query_engine/test_plan.rs | 5 + .../storage_engines/sketch_db/index/mod.rs | 197 +++++++-- .../sketch_db/query/delta_apply.rs | 5 +- .../src/storage_engines/sketch_db/sds.rs | 16 +- .../types/hot_reload_config.rs | 1 + .../src/tests/test_utilities/planning.rs | 6 +- .../tests/all_sketches_process_oracle_e2e.rs | 11 +- .../asapquery_compatibility_process_e2e.rs | 122 +++--- data_plane/tests/backend_process_e2e.rs | 79 ++-- data_plane/tests/component_process_e2e.rs | 9 +- ...e2e_controller_plans_and_backend_serves.rs | 37 +- .../edge_runtime_consumes_precompute_rs.rs | 195 --------- data_plane/tests/edge_sketch_codec.rs | 75 ++++ .../tests/promql_differential_process_e2e.rs | 9 +- .../tests/support/current_series_process.rs | 4 +- .../tests/support/issue_701_702_process.rs | 6 +- data_plane/tests/support/physical_fixture.rs | 5 + .../architecture-naming-review.zh.md | 6 +- .../control-plane/physical-compiler.md | 12 +- .../control-plane/planning-terminology.md | 2 +- scripts/e2e.sh | 2 +- tools/shared-workload/ACCURACY_E2E.md | 4 +- tools/test_shared_panes.py | 16 +- 66 files changed, 1552 insertions(+), 1394 deletions(-) create mode 100644 crates/asap_sketch_codec/Cargo.toml create mode 100644 crates/asap_sketch_codec/src/lib.rs delete mode 100644 data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs delete mode 100644 data_plane/tests/edge_runtime_consumes_precompute_rs.rs create mode 100644 data_plane/tests/edge_sketch_codec.rs diff --git a/Cargo.lock b/Cargo.lock index 4ba3a108d..d4dfd33fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,18 +393,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "asap-precompute-rs" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPCollector?branch=main#1d8efd07e40fc151cbd4678a5c6aa9774b1aed34" -dependencies = [ - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", - "prost", - "serde", - "serde_json", - "thiserror 1.0.69", -] - [[package]] name = "asap-sql-function-catalog" version = "0.1.0" @@ -431,6 +419,14 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "asap_sketch_codec" +version = "0.1.0" +dependencies = [ + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", + "prost", +] + [[package]] name = "asap_sketchlib" version = "0.3.0" @@ -1159,9 +1155,9 @@ dependencies = [ "arrow", "asap-aware-mapping", "asap-frontend-promql", - "asap-precompute-rs", "asap-types", "asap_otel_proto", + "asap_sketch_codec", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", "asap_types", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index eaca35231..5bce59217 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "crates/asap_otel_proto", "crates/asap_types", + "crates/asap_sketch_codec", "data_plane", "control_plane", ] @@ -11,12 +12,6 @@ members = [ edition = "2021" version = "0.1.0" -# ASAPCollector's `asap-precompute-rs` currently declares Sketchlib as a -# relative path. When Collector is consumed from Git, resolve that dependency -# to the same Git-sourced Sketchlib package as the backend. -[patch."https://github.com/ProjectASAP/ASAPCollector"] -asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch = "main" } - [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. diff --git a/README.md b/README.md index 3293c7f8a..f9391cbc1 100644 --- a/README.md +++ b/README.md @@ -241,8 +241,8 @@ git -C ../ASAPCollector checkout main [MVP CI](.github/workflows/mvp-ci.yml) is the source for compatible dependency checkouts; currently Collector uses its default branch. For reproducible runs, -record all three exact revisions. ASAPPlanner is fetched at the revision pinned -in Cargo manifests; do not substitute an unrelated local planner checkout. +record the exact revisions. ASAPPlanner is fetched at the revision pinned in +the Cargo manifests; do not substitute an unrelated local planner checkout. ### 2. Check prerequisites and build diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index f2894c4fb..95f7f543b 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -1,6 +1,6 @@ //! Export every bindable candidate for isolated measurement, without selecting a winner. use control_plane::physical::{ - compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}, + compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}, workload_cost, }; use planner_types::post_asap::{SummaryExpr, SummaryNode}; @@ -138,9 +138,9 @@ fn main() -> Result<(), Box> { let enabled_materialization_keys = candidate.enabled_materialization_keys.clone(); let planner_selected_queries = planner_forest(&queries); let compiled = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate, environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate, environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) }; let plan = match compiled { Ok(plan) => plan, diff --git a/control_plane/examples/workload_cost_manifest.rs b/control_plane/examples/workload_cost_manifest.rs index 142f4a07c..f01fd0574 100644 --- a/control_plane/examples/workload_cost_manifest.rs +++ b/control_plane/examples/workload_cost_manifest.rs @@ -1,6 +1,6 @@ //! Emit pricing requirements; never fabricate quotes or publish a plan. use control_plane::physical::{ - compiler::BackendLocalPlanningInput, compiler::PhysicalPlanCompiler, workload_cost, + compiler::BackendLocalPlanningInput, compiler::DeploymentPlanCompiler, workload_cost, }; fn main() -> Result<(), Box> { @@ -14,7 +14,7 @@ fn main() -> Result<(), Box> { .into_iter() .filter_map(|candidate| { let queries = candidate.queries.clone(); - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(candidate, environment.clone()) .and_then(|plan| workload_cost::manifest(&plan, &queries)) .ok() diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f1e1b1df0..f22bd867b 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -247,6 +247,7 @@ pub async fn compile_automatic_clickhouse_workload( let mut entries = std::collections::BTreeMap::new(); let mut window_templates = std::collections::BTreeMap::>::new(); let mut installed_dags = std::collections::BTreeMap::new(); + let mut selected_dags = std::collections::BTreeMap::new(); let mut materializations = std::collections::BTreeMap::new(); let mut selection_traces = std::collections::BTreeMap::new(); for query in &request.queries { @@ -267,6 +268,9 @@ pub async fn compile_automatic_clickhouse_workload( ) .then_some(config.slide_interval.saturating_mul(1_000)), materialization: config.policy_fingerprint().into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::Reduce(config.grouping_labels.names()), window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, @@ -286,7 +290,13 @@ pub async fn compile_automatic_clickhouse_workload( "duplicate canonical SQL query identity".into(), )); } - installed_dags.insert(query.sql.clone(), installed); + selected_dags.insert(query.sql.clone(), installed.document.clone()); + installed_dags.insert( + query.sql.clone(), + installed + .maintenance_projection() + .map_err(ClickHousePlanningError::Lower)?, + ); } let configs: Vec<_> = materializations.into_values().collect(); let sds = SummaryCatalog::from_materializations( @@ -322,6 +332,7 @@ pub async fn compile_automatic_clickhouse_workload( tables: request.tables.clone(), accuracy: request.accuracy.clone(), }), + selected_dags, entries, }, }; @@ -423,6 +434,7 @@ pub async fn compile_clickhouse_workload( let mut entries = std::collections::BTreeMap::new(); let mut window_templates = std::collections::BTreeMap::>::new(); let mut installed_dags = std::collections::BTreeMap::new(); + let mut selected_dags = std::collections::BTreeMap::new(); for query in &request.queries { let planned = plan_clickhouse_sql(&query.sql, &catalog, request.accuracy.clone()).await?; let template = planned.canonical_sql.clone(); @@ -430,7 +442,13 @@ pub async fn compile_clickhouse_workload( bind_selected_node(node, family, query, request) })?; index_sql_template(&mut window_templates, template, &executable); - installed_dags.insert(query.sql.clone(), installed); + selected_dags.insert(query.sql.clone(), installed.document.clone()); + installed_dags.insert( + query.sql.clone(), + installed + .maintenance_projection() + .map_err(ClickHousePlanningError::Lower)?, + ); let identity = QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &executable.canonical_query); if entries.insert(identity.clone(), executable).is_some() { @@ -454,6 +472,7 @@ pub async fn compile_clickhouse_workload( tables: request.tables.clone(), accuracy: request.accuracy.clone(), }), + selected_dags, entries, }, }; @@ -612,6 +631,9 @@ fn bind_selected_node( ) .then_some(selected.slide_interval.saturating_mul(1_000)), materialization: selected.policy_fingerprint().into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + selected.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::Reduce(selected.grouping_labels.names()), window_ms: selected.stored_window_ms(), pane_origin_ms: selected.pane_origin_ms, @@ -1641,10 +1663,16 @@ mod tests { installed.binding.nodes.len(), installed.document.nodes.len() ); - assert!(installed.binding.nodes.values().any(|binding| matches!( + assert!(!installed.binding.nodes.values().any(|binding| matches!( binding, crate::physical::executable_binding::BackendNodeBinding::Query { .. } ))); + assert!( + publication.query_plan.selected_dags[&request.queries[0].sql] + .nodes + .len() + > installed.document.nodes.len() + ); let entry = publication.query_plan.entries.values().next().unwrap(); // External SQL retains its literal time range until it can be bound. assert!(!entry.canonical_query.starts_with("moving-window-v1:")); diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs index 073ea1d6f..165c6d111 100644 --- a/control_plane/src/physical/backend_stage.rs +++ b/control_plane/src/physical/backend_stage.rs @@ -1,7 +1,7 @@ //! Backend-facing projection of one planning cycle. //! //! These types are the input to [`crate::backend_plan::from_stage_config`] and -//! to `emit::backend_wire`'s backend JSON builders. `PhysicalPlanCompiler` builds +//! to `emit::backend_wire`'s backend JSON builders. `DeploymentPlanCompiler` builds //! them directly from the summaries ASAPPlanner selected. //! //! They are deliberately not `Serialize`/`Deserialize`: `SummaryFamilyType` diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 69254a730..29ea07b08 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -527,7 +527,7 @@ impl AccuracyEvidenceProvider for QueryEvidence<'_> { } #[derive(Debug, Default)] -pub struct PhysicalPlanCompiler; +pub struct DeploymentPlanCompiler; impl BackendLocalPlanningInput { /// Invoke the pinned Planner from canonical startup workloads and compile @@ -940,7 +940,7 @@ fn preserve_metricsql_counter_only_roots( Ok(()) } -impl PhysicalPlanCompiler { +impl DeploymentPlanCompiler { pub fn compile_promql( &self, request: PhysicalCompilationRequest, @@ -1713,6 +1713,7 @@ impl PhysicalPlanCompiler { .then_some(materialization.slide_interval.saturating_mul(1_000)), readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint.into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()), output_grouping: PhysicalGrouping::Reduce( materialization.grouping_labels.names(), ), @@ -1850,10 +1851,11 @@ impl PhysicalPlanCompiler { }); } } - let query_plan = QueryPlan { + let mut query_plan = QueryPlan { plan_id, plan_version: envelope.plan_version, clickhouse_context: None, + selected_dags: BTreeMap::new(), entries: query_entries, }; let mut installed_dags = BTreeMap::new(); @@ -1882,6 +1884,9 @@ impl PhysicalPlanCompiler { query_id: query_id.clone(), reason, })?; + query_plan + .selected_dags + .insert(query_id.clone(), installed.document.clone()); installed_dags.insert(query_id, installed); } for materialization in &mut materializations { @@ -1933,6 +1938,18 @@ impl PhysicalPlanCompiler { } })?; } + // QueryPlan already owns executable readout nodes. Persist only + // maintenance ancestors under PrecomputePlan. + installed_dags = installed_dags + .into_iter() + .filter(|(_, installed)| !installed.binding.precompute_sinks.is_empty()) + .map(|(query_id, installed)| { + installed + .maintenance_projection() + .map(|projected| (query_id.clone(), projected)) + .map_err(|reason| CompileError::Query { query_id, reason }) + }) + .collect::>()?; let mut precompute_plan = match environment.target { PhysicalDeploymentTarget::DistributedCollectors => { PrecomputePlan::build(envelope.clone(), materializations, &producer_ids).and_then( @@ -2356,7 +2373,7 @@ fn requires_exact_erp_fallback( #[cfg(test)] /// Planner-adapter selection step used before physical compilation. Keeping /// this separate makes the ownership boundary explicit: callers supply the -/// selected post-ASAP DAG to [`PhysicalPlanCompiler::compile`]. +/// selected post-ASAP DAG to [`DeploymentPlanCompiler::compile`]. pub fn select_post_asap( expr: &QueryExpr, accuracy: AccuracyTarget, @@ -3798,7 +3815,7 @@ impl QueryFrontend { request: PhysicalCompilationRequest, environment: PhysicalDeploymentContext, ) -> Result { - PhysicalPlanCompiler.compile_for_frontend(request, environment, self) + DeploymentPlanCompiler.compile_for_frontend(request, environment, self) } } #[cfg(test)] @@ -3839,7 +3856,7 @@ pub(crate) mod tests { .unwrap() .into_iter() .filter_map(|r| { - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(r, environment.clone()) .ok() }) @@ -3872,8 +3889,8 @@ pub(crate) mod tests { assert_eq!(populations.len(), 1); let installed = serde_json::to_string(&plan.precompute_plan.executable_dags).unwrap(); assert!( - installed.contains("MaintainPopulation"), - "shared state must originate in the installed Planner DAG" + !installed.contains("MaintainPopulation"), + "query-only population readout must not be executable maintenance" ); } @@ -3955,7 +3972,7 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); let request = request("average", "avg_over_time(a[1m])"); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!( @@ -3985,7 +4002,7 @@ pub(crate) mod tests { env.target_collector_ids.clear(); let mut input = request("minimum", "min_over_time(data[1m])"); input.allow_mixed_summary_and_exact_execution = true; - let plan = PhysicalPlanCompiler.compile_promql(input, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(input, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].aggregation_type, @@ -4089,9 +4106,9 @@ pub(crate) mod tests { .enumerate() .filter_map(|(index, candidate)| { let plan = if frontend == QueryFrontend::MetricsQl { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -4139,7 +4156,7 @@ pub(crate) mod tests { let mut reasons = vec![]; assert!( candidates.into_iter().any(|candidate| { - match PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) { + match DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { Ok(plan) => { !plan.precompute_plan.materializations.is_empty() && plan @@ -4170,7 +4187,7 @@ pub(crate) mod tests { snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query(text.into()); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{text}"); @@ -4229,7 +4246,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let mut plan = PhysicalPlanCompiler + let mut plan = DeploymentPlanCompiler .compile_promql(request("scope", "sum_over_time(m[1m])"), env) .unwrap(); let installed = plan @@ -4260,6 +4277,8 @@ pub(crate) mod tests { &dag, ) .unwrap(); + installed.document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; assert!(plan .precompute_plan .validate() @@ -4279,7 +4298,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("per-entity", query), environment) .unwrap(); assert!( @@ -4298,7 +4317,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("reduced", query), environment) .unwrap(); assert!( @@ -4314,7 +4333,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -4338,7 +4357,7 @@ pub(crate) mod tests { #[test] fn raw_counter_artifact_is_valid_for_backend_precompute() { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("counter", "rate(m[1m])"), environment(10_000)) .unwrap(); plan.precompute_plan.validate().unwrap(); @@ -4365,7 +4384,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); @@ -4421,7 +4440,7 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk", query, Some(evidence)).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); @@ -4443,7 +4462,7 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .unwrap(); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -4537,7 +4556,7 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); @@ -4575,11 +4594,29 @@ pub(crate) mod tests { .precompute_plan .executable_dags .get(&entry.query_id) - .expect("compiled query retains its Planner DAG and backend placement"); - installed.validate().expect("typed DAG document"); - crate::physical::executable_binding::validate_query_plan(installed, entry) - .expect("query node bindings"); + .expect("compiled query retains its maintenance projection"); + installed + .validate() + .expect("typed maintenance DAG document"); + assert_eq!( + installed.document.schema_version, + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION + ); + assert!(installed + .document + .nodes + .iter() + .all(|node| node.output_state.timing + == planner_types::post_asap::ExecutionTiming::MaintenanceTime)); assert_eq!(installed.binding.query_plan_sink, entry.root); + let mut mismatched = plan.to_publication_artifact().unwrap(); + let projected = mismatched + .precompute_plan + .executable_dags + .get_mut(&entry.query_id) + .unwrap(); + projected.binding.query_plan_sink = asap_types::executable_plan::QueryNodeId(u64::MAX); + assert!(mismatched.validate().is_err()); assert!(installed.binding.nodes.values().any(|placement| matches!( placement, crate::physical::executable_binding::BackendNodeBinding::Materialization { .. } @@ -4602,7 +4639,7 @@ pub(crate) mod tests { observed_at_unix_ms: 9_500, source: "unit-fixture".into(), }; - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql( request_with_evidence( "topk-rate", @@ -4652,7 +4689,7 @@ pub(crate) mod tests { let mut request = request("bounded", "sum(sum_over_time(m[1m]))"); request.retained_summary_memory_budget_bytes = Some(1); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .unwrap_err(); assert!(error.to_string().contains("retained summary footprint")); @@ -4810,7 +4847,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); @@ -4856,7 +4893,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 3); @@ -4931,7 +4968,7 @@ pub(crate) mod tests { } }) .expect("unknown HLL candidate stays inspectable"); - let result = PhysicalPlanCompiler.compile_metricsql(workload, environment(10_000)); + let result = DeploymentPlanCompiler.compile_metricsql(workload, environment(10_000)); assert!( matches!(result, Err(CompileError::Query { reason, .. }) if reason.contains("no certified accuracy guarantee")) ); @@ -4959,7 +4996,7 @@ pub(crate) mod tests { None, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -4985,7 +5022,7 @@ pub(crate) mod tests { deployment.target_collector_ids.clear(); let mut workload = request("confidence", "distinct_over_time(m[1m])"); workload.allow_mixed_summary_and_exact_execution = true; - let result = PhysicalPlanCompiler.compile_metricsql(workload, deployment); + let result = DeploymentPlanCompiler.compile_metricsql(workload, deployment); assert!(result.unwrap().precompute_plan.materializations.is_empty()); } @@ -5016,7 +5053,7 @@ pub(crate) mod tests { second.query_string = "max_over_time(b[1m])".into(); workload.queries.push(second); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap_err(); assert!(matches!( @@ -5037,7 +5074,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -5054,7 +5091,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); @@ -5092,7 +5129,7 @@ pub(crate) mod tests { workload.queries[0].query_string = query.into(); workload.queries[0].selected_plan_root = crate::planner_selection::keep_pre_asap(&canonical).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, environment(10_000)) .unwrap(); let identity = canonical_promql(query).unwrap(); @@ -5225,7 +5262,7 @@ pub(crate) mod tests { workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -5323,11 +5360,11 @@ pub(crate) mod tests { let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; backend.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(with_evidence, backend) .unwrap(); assert!( - !plan.summary_catalog.materializations.is_empty(), + !plan.summary_catalog.definitions.is_empty(), "measured exact-composition evidence must expose the rate child as a SummaryStore binding" ); } @@ -5354,7 +5391,7 @@ pub(crate) mod tests { let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; backend.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(unavailable, backend) .unwrap(); // Planner 54f can realize this particular shape directly as an exact @@ -5362,7 +5399,7 @@ pub(crate) mod tests { // ExactComposition candidate. The absence of evidence must therefore // leave that direct legal path intact rather than inventing a composed // cost or forcing an exact fallback. - assert!(!plan.summary_catalog.materializations.is_empty()); + assert!(!plan.summary_catalog.definitions.is_empty()); let entry = plan .query_plan .entries @@ -5469,7 +5506,7 @@ pub(crate) mod tests { &workload.exact_composition_costs, ) .unwrap(); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -5501,7 +5538,7 @@ pub(crate) mod tests { // Adding another readout adds recurring reads, not another update stream. #[test] fn joint_lifecycle_charges_shared_updates_once() { - let baseline = PhysicalPlanCompiler + let baseline = DeploymentPlanCompiler .compile_promql( request("q90", "quantile_over_time(0.9, m[1m])"), environment(10000), @@ -5513,7 +5550,7 @@ pub(crate) mod tests { .remove(0); second.summary_lifecycle_inputs.evaluation_interval_ms = 20000; workload.queries.push(second); - let shared = PhysicalPlanCompiler + let shared = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(shared.lifecycle_estimates.len(), 1); @@ -5542,7 +5579,7 @@ pub(crate) mod tests { second.summary_lifecycle_inputs.ingestion_rate_per_second = 200.0; workload.queries.push(second); assert!(matches!( - PhysicalPlanCompiler.compile_promql(workload, environment(10000)), + DeploymentPlanCompiler.compile_promql(workload, environment(10000)), Err(CompileError::Lifecycle { .. }) )); } @@ -5568,11 +5605,11 @@ pub(crate) mod tests { if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { env.target_collector_ids.clear(); } - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.definitions.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); let bindings = bundle @@ -5592,14 +5629,14 @@ pub(crate) mod tests { #[test] fn adding_shared_consumer_changes_plan_identity() { let workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let one = PhysicalPlanCompiler + let one = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); workload .queries .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let two = PhysicalPlanCompiler + let two = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); assert_ne!(one.envelope.plan_id, two.envelope.plan_id); @@ -5612,10 +5649,10 @@ pub(crate) mod tests { let mut other = request("qn", "quantile_over_time(0.90, n[1m])"); other.queries[0].legacy_query_source = Source::TimeSeries { metric: "n".into() }; workload.queries.extend(other.queries); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); - assert_eq!(bundle.summary_catalog.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.definitions.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { assert_eq!(collector.materializations.len(), 2); @@ -5628,7 +5665,7 @@ pub(crate) mod tests { workload .queries .extend(request("increase", "increase(m[1m])").queries); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -5646,7 +5683,7 @@ pub(crate) mod tests { other.queries[0].window_realization_candidates[0].realization_id = "another-implementation".into(); workload.queries.extend(other.queries); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .expect_err("conflicting shared state must fail before publication"); assert!(error @@ -5671,7 +5708,7 @@ pub(crate) mod tests { ); entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { @@ -5773,7 +5810,7 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap_err(); @@ -5800,7 +5837,7 @@ pub(crate) mod tests { entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); request.allow_mixed_summary_and_exact_execution = false; - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); @@ -5826,13 +5863,13 @@ pub(crate) mod tests { let candidates = super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap(); - match PhysicalPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { + match DeploymentPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), Err(error) => assert!(error .to_string() .contains("semantically identical original subtree witness")), } - let native = PhysicalPlanCompiler + let native = DeploymentPlanCompiler .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert!(native.precompute_plan.materializations.is_empty()); @@ -5849,7 +5886,7 @@ pub(crate) mod tests { entry.query = Query("sum_over_time(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].partitioning, @@ -5876,7 +5913,7 @@ pub(crate) mod tests { // The derivation now covers both range selectors, so this no longer // needs a hand-supplied 5m candidate to keep `b` from falling back. let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let bindings = plan .query_plan .entries @@ -5887,7 +5924,7 @@ pub(crate) mod tests { let actual = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.materializations[&binding.materialization]; + let identity = &plan.summary_catalog.definitions[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -5920,7 +5957,7 @@ pub(crate) mod tests { value["data_workload"]["ingestion_rate"]["value"] = json!(rate); let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -5997,7 +6034,7 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::Pane { .. } ) }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -6051,7 +6088,7 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(0), }; let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), expected_states); let entry = plan.query_plan.entries.values().next().unwrap(); let bindings = entry.materialization_bindings(); @@ -6089,7 +6126,7 @@ pub(crate) mod tests { }; entries.push(second); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if phase == 0 { 1 } else { 2 } @@ -6143,7 +6180,7 @@ pub(crate) mod tests { .window_realization_candidates .iter() .any(|c| !c.derived)); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); } @@ -6505,7 +6542,7 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::FullWindow ) == full }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; assert_eq!(config.slide_interval, u64::from(evaluation)); assert_eq!(config.window_size, 60); @@ -6561,7 +6598,7 @@ pub(crate) mod tests { ) }); } - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if read_cost == 0.0 { 1 } else { 2 } @@ -6614,7 +6651,7 @@ pub(crate) mod tests { ) }); } - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); assert!(plan .lifecycle_estimates @@ -6635,7 +6672,7 @@ pub(crate) mod tests { }; let (request, env) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.queries[0].window_realization_candidates.is_empty()); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); } @@ -6712,7 +6749,7 @@ pub(crate) mod tests { fn retained_state_count_follows_the_derived_pane_width() { let snapshot = planning_snapshot(); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!( @@ -6737,7 +6774,7 @@ pub(crate) mod tests { }; } let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); let materialization = &plan.precompute_plan.materializations[0]; @@ -6774,7 +6811,7 @@ pub(crate) mod tests { Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let query = plan.query_plan.entries.values().next().unwrap(); let bindings = query.materialization_bindings(); // Both operands now hold a summary. The filtered denominator is no @@ -6786,7 +6823,7 @@ pub(crate) mod tests { let bound = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.materializations[&binding.materialization]; + let identity = &plan.summary_catalog.definitions[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -6859,7 +6896,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); @@ -6962,11 +6999,37 @@ pub(crate) mod tests { .collect::>(); assert_eq!(bindings.len(), 1); query_plan.validate(&bindings).unwrap(); + let stored_outputs = query_plan + .entries + .values() + .flat_map(|entry| entry.materialization_bindings()) + .map(|binding| binding.stored_output_reference) + .collect::>(); + assert_eq!(stored_outputs.len(), 2); + assert_eq!(stored_outputs[0], stored_outputs[1]); + assert_eq!( + stored_outputs[0], + bundle.precompute_plan.schemas[0].stored_output_reference + ); + asap_types::plan_publication::validate_stored_output_references( + &bundle.precompute_plan, + &query_plan, + ) + .unwrap(); + // V1 has one stored output per definition; arbitrary output IDs are + // rejected before writer/reader agreement is considered. + let mut rebound_writer = bundle.precompute_plan.clone(); + rebound_writer.schemas[0] + .stored_output_reference + .stored_output_id = asap_types::sds::StoredOutputId(123); + assert!(rebound_writer + .validate_against_catalog(&bundle.summary_catalog) + .is_err()); } #[test] fn precompute_catalog_validates_without_backend_projection() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("catalog", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7023,7 +7086,7 @@ pub(crate) mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7048,7 +7111,7 @@ pub(crate) mod tests { #[test] fn compiles_one_decision_into_matching_collector_and_backend_views() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7064,7 +7127,7 @@ pub(crate) mod tests { assert_eq!( bundle .summary_catalog - .materializations + .definitions .keys() .cloned() .collect::>(), @@ -7115,7 +7178,7 @@ pub(crate) mod tests { bundle.transmission_plan.validate_frame(&wrong_version), Err(TransmissionPlanError::InvalidFrame(_)) )); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.definitions.len(), 1); assert_eq!( bundle .query_plan @@ -7187,7 +7250,7 @@ pub(crate) mod tests { #[test] fn backend_local_hll_and_envelope_ingest_are_supported() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7217,7 +7280,7 @@ pub(crate) mod tests { #[test] fn backend_local_precompute_contract_has_no_collector_producers() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7366,9 +7429,9 @@ pub(crate) mod tests { bundle.precompute_plan.schemas[0].window.pane_origin_ms, Some(7_000) ); - let definition = &bundle.summary_catalog.materializations - [&asap_types::sds::SummaryDefinitionId::from(config.policy_fingerprint())]; - assert_eq!(definition.pane_origin_ms, Some(7_000)); + assert!(bundle.summary_catalog.definitions.contains_key( + &asap_types::sds::SummaryDefinitionId::from(config.policy_fingerprint()) + )); assert_eq!( bundle .query_plan @@ -7405,9 +7468,9 @@ pub(crate) mod tests { deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); let compiled = - PhysicalPlanCompiler.compile_promql(request(query_id, promql), deployment); + DeploymentPlanCompiler.compile_promql(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); - assert_eq!(plan.summary_catalog.materializations.len(), 1, "{promql}"); + assert_eq!(plan.summary_catalog.definitions.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); assert!(plan.collector_plans.is_empty(), "{promql}"); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -7446,7 +7509,7 @@ pub(crate) mod tests { .clone() .into_physical_compilation_request() .unwrap(); - let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); + let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan @@ -7459,7 +7522,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(native, environment) .expect("native fixture compiles"); assert!(plan.precompute_plan.materializations.is_empty()); @@ -7486,7 +7549,7 @@ pub(crate) mod tests { .clone() .into_physical_compilation_request() .unwrap(); - let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); + let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan @@ -7499,7 +7562,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(native, environment) .expect("native demo compiles"); @@ -7529,12 +7592,12 @@ pub(crate) mod tests { .remove(0), ); } - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(compilation_request, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 4); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.definitions.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); assert_eq!(bundle.precompute_plan.producers.len(), 2); @@ -7573,10 +7636,10 @@ pub(crate) mod tests { guarantee: left_root.guarantee.clone(), }); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(compilation_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); - assert_eq!(bundle.summary_catalog.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.definitions.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!( bundle @@ -7608,9 +7671,8 @@ pub(crate) mod tests { .values() .filter_map(|node| match node { crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => Some( - bundle.summary_catalog.data_descriptors[&bundle - .summary_catalog - .materializations[&binding.materialization] + bundle.summary_catalog.data_descriptors[&bundle.summary_catalog.definitions + [&binding.materialization] .data_descriptor_id] .time_series_metric() .unwrap(), @@ -7623,7 +7685,7 @@ pub(crate) mod tests { #[test] fn precompute_schema_must_match_materialization_semantics() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7655,7 +7717,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let materialization = bundle.precompute_plan.materializations.first().unwrap(); assert_eq!(materialization.window_size, 60); assert_eq!( @@ -7709,7 +7771,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( bundle.lifecycle_estimates[0].window_realization_id, expected_id @@ -7760,7 +7822,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(workload, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(workload, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 2); } @@ -7772,14 +7834,14 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::Pane { pane_secs: 7 }; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - assert!(PhysicalPlanCompiler.compile_promql(request, env).is_err()); + assert!(DeploymentPlanCompiler.compile_promql(request, env).is_err()); } #[test] fn missing_window_implementation_evidence_fails_closed() { let mut request = request("q-window", "quantile_over_time(0.99, m[1m])"); request.queries[0].window_realization_candidates.clear(); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect_err("Planner must not receive a zero-cost invented window"); assert!(matches!(error, CompileError::Lifecycle { .. })); @@ -7787,7 +7849,7 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_schema_or_producer_drift() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7811,7 +7873,7 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_empty_or_duplicate_schema_ids() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7858,7 +7920,7 @@ pub(crate) mod tests { }), ) .expect("selection accepts evidence before freshness validation"); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(100_000)) .expect_err("stale certificate must fail"); assert!(matches!(error, CompileError::InvalidEvidence { .. })); @@ -7879,7 +7941,7 @@ pub(crate) mod tests { ) .expect("selection occurs before deployment-time freshness validation"); assert!(matches!( - PhysicalPlanCompiler.compile_promql(topk, environment(10_000)), + DeploymentPlanCompiler.compile_promql(topk, environment(10_000)), Err(CompileError::InvalidEvidence { .. }) )); @@ -7888,7 +7950,7 @@ pub(crate) mod tests { .cost .observed_at_unix_ms = 10_001; assert!(matches!( - PhysicalPlanCompiler.compile_promql(window, environment(10_000)), + DeploymentPlanCompiler.compile_promql(window, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7907,10 +7969,10 @@ pub(crate) mod tests { }), ) .expect("selection accepts valid evidence"); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect("certified TopK compiles"); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.definitions.len(), 1); assert_eq!( bundle.collector_plans[0].materializations[0] .evidence_source @@ -7924,7 +7986,7 @@ pub(crate) mod tests { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); request.planner_revision = "different".into(); assert!(matches!( - PhysicalPlanCompiler.compile_promql(request, environment(10_000)), + DeploymentPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::PlannerRevision { .. }) )); } @@ -7939,7 +8001,7 @@ pub(crate) mod tests { .summary_lifecycle_inputs .evidence_valid_for_ms = 10; assert!(matches!( - PhysicalPlanCompiler.compile_promql(request, environment(10_000)), + DeploymentPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7965,7 +8027,7 @@ pub(crate) mod tests { #[test] fn runtime_policy_encoding_is_checked() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7997,7 +8059,7 @@ pub(crate) mod tests { absolute_threshold: 0.0, gos: None, }); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect("delta-capable physical plan"); let rule = &bundle.transmission_plan.rules[0]; @@ -8032,7 +8094,7 @@ pub(crate) mod tests { #[test] fn runtime_adaptation_requires_fresh_exact_evidence_and_successor_version() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 1856672cd..26bf26af2 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -389,7 +389,7 @@ impl ErpPlanningInput { return Err("ERP evidence catalog differs from the active catalog".into()); } let materialization = catalog - .materializations + .definitions .get(&populations.summary_definition_id) .ok_or("ERP evidence summary is absent from the active catalog")?; let summary = catalog @@ -1480,13 +1480,12 @@ mod tests { .unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = catalog.reference().unwrap(); - observed.summary_definition_id = *catalog.materializations.keys().next().unwrap(); + observed.summary_definition_id = *catalog.definitions.keys().next().unwrap(); observed.input_semantics = asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; policy.observed_populations = Some(observed.clone()); policy.resolve_population_data_descriptor(Some(&catalog)); - let expected = - &catalog.materializations[&observed.summary_definition_id].data_descriptor_id; + let expected = &catalog.definitions[&observed.summary_definition_id].data_descriptor_id; assert_eq!( &policy.resolved_data_descriptor.as_ref().unwrap().id, expected diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 28f82a57c..187235955 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -39,7 +39,7 @@ pub fn install_selected_dag( precompute_sinks, }, }; - installed.validate()?; + installed.binding.validate(&installed.document.decode()?)?; Ok(installed) } diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs index 4a620b94d..71426ad69 100644 --- a/control_plane/src/physical/realization.rs +++ b/control_plane/src/physical/realization.rs @@ -4,7 +4,7 @@ //! selected logical roots or infer a pane width from a query's slide. use super::compiler::{ CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, - PhysicalPlanCompiler, QueryCompilationInput, + DeploymentPlanCompiler, QueryCompilationInput, }; use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; use asap_aware_mapping::cost_model::Cost; @@ -51,9 +51,9 @@ impl RealizationProvider for ExistingRealizations { frontend: super::compiler::QueryFrontend, ) -> Result { if frontend == super::compiler::QueryFrontend::MetricsQl { - PhysicalPlanCompiler.compile_metricsql(request, environment) + DeploymentPlanCompiler.compile_metricsql(request, environment) } else { - PhysicalPlanCompiler.compile_promql(request, environment) + DeploymentPlanCompiler.compile_promql(request, environment) } } diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 71f7bfa24..963ebccac 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -9,7 +9,7 @@ mod status; pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; #[cfg(test)] -use super::compiler::PhysicalPlanCompiler; +use super::compiler::DeploymentPlanCompiler; use std::collections::{BTreeMap, BTreeSet}; @@ -1034,7 +1034,7 @@ mod tests { ); entries.push(second); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request.clone(), env) .unwrap(); let costs = manifest(&plan, &request.queries).unwrap(); @@ -1091,7 +1091,7 @@ mod tests { .as_ref() .unwrap() .len(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), environment.clone()) .unwrap(); assert!(identities.insert(plan.envelope.plan_id)); @@ -1146,7 +1146,7 @@ mod tests { let quotes = candidates .iter() .map(|candidate| { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -1186,10 +1186,10 @@ mod tests { let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(candidates.len() >= 2); - let local = PhysicalPlanCompiler + let local = DeploymentPlanCompiler .compile_promql(candidates[0].clone(), environment.clone()) .unwrap(); - let native = PhysicalPlanCompiler + let native = DeploymentPlanCompiler .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert_ne!(local.envelope.plan_id, native.envelope.plan_id); @@ -1245,7 +1245,7 @@ mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(exact.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &exact.queries).unwrap(); @@ -1397,7 +1397,7 @@ mod tests { fn second_consumer_adds_reads_not_another_shared_state() { let (request, env) = fixture().into_physical_compilation_request().unwrap(); let first = manifest( - &PhysicalPlanCompiler + &DeploymentPlanCompiler .compile_promql(request.clone(), env.clone()) .unwrap(), &request.queries, @@ -1438,7 +1438,7 @@ mod tests { &shared.exact_composition_costs, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(shared.clone(), env) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index bf6b73187..19c12c689 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1029,6 +1029,10 @@ mod catalog_binding_tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1048,6 +1052,7 @@ mod catalog_binding_tests { plan_id: 7, plan_version: 2, clickhouse_context: None, + selected_dags: Default::default(), entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), }, catalog, @@ -1157,6 +1162,10 @@ mod catalog_binding_tests { SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); let (mut counter_plan, _) = fixture(); binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); + binding(&mut counter_plan).stored_output_reference = + asap_types::sds::StoredOutputReference::for_definition( + counter.policy_fingerprint().into(), + ); as_rate_plan(counter_plan) .validate_against_catalog(&counter_catalog) .unwrap(); @@ -1199,6 +1208,10 @@ mod tests { Ok(MaterializationBinding { full_window_slide_ms: None, materialization: PolicyFingerprint(7).into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + PolicyFingerprint(7).into(), + ), output_grouping: PhysicalGrouping::PerEntity, window_ms: 300_000, pane_origin_ms: Some(0), @@ -1326,6 +1339,7 @@ mod tests { tables: Default::default(), accuracy: planner_types::types::AccuracyTarget::Exact, }), + selected_dags: Default::default(), entries: [base, metricsql, clickhouse] .into_iter() .map(|entry| (QueryPlan::catalog_key(entry.language, "shared"), entry)) diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index 7e64c0089..a8ffa3132 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -585,37 +585,46 @@ mod hybrid_tests { ) .unwrap(); let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let entry = - crate::query_plan::compile_bound_composable_mapped( - "hybrid".into(), - query.into(), - &selected, - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |node, _| { - let (_, _, spatial_filter) = - crate::physical::compiler::raw_materialization_input_contract(node) - .map_err(QueryPlanError::Invalid)?; - Ok(MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint( - if spatial_filter.is_empty() { 7 } else { 8 }, - ) - .into(), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), + let entry = crate::query_plan::compile_bound_composable_mapped( + "hybrid".into(), + query.into(), + &selected, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::Reject, + |node, _| { + let (_, _, spatial_filter) = + crate::physical::compiler::raw_materialization_input_contract(node) + .map_err(QueryPlanError::Invalid)?; + Ok(MaterializationBinding { + full_window_slide_ms: None, + item_labels: Vec::new(), + materialization: asap_types::PolicyFingerprint(if spatial_filter.is_empty() { + 7 + } else { + 8 }) - }, - |_, _| {}, - ) - .unwrap(); + .into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + asap_types::PolicyFingerprint(if spatial_filter.is_empty() { + 7 + } else { + 8 + }) + .into(), + ), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + }) + }, + |_, _| {}, + ) + .unwrap(); assert_eq!(entry.materialization_bindings().len(), 2); assert!(!entry.nodes.values().any(|node| matches!( node, @@ -668,7 +677,7 @@ mod hybrid_tests { #[cfg(test)] mod planner_workload_tests { use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { let mut fixture: serde_json::Value = serde_json::from_str(include_str!( @@ -683,7 +692,7 @@ mod planner_workload_tests { let (request, environment) = snapshot .into_physical_compilation_request() .unwrap_or_else(|error| panic!("{query}: {error}")); - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(request, environment) .unwrap_or_else(|error| panic!("{query}: {error}")) } @@ -769,7 +778,7 @@ mod planner_workload_tests { let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.allow_mixed_summary_and_exact_execution); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.query_plan.entries.len(), 24); diff --git a/crates/asap_sketch_codec/Cargo.toml b/crates/asap_sketch_codec/Cargo.toml new file mode 100644 index 000000000..c2d728895 --- /dev/null +++ b/crates/asap_sketch_codec/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "asap_sketch_codec" +version.workspace = true +edition.workspace = true + +[dependencies] +asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch = "main" } +prost = "0.13" diff --git a/crates/asap_sketch_codec/src/lib.rs b/crates/asap_sketch_codec/src/lib.rs new file mode 100644 index 000000000..279efe168 --- /dev/null +++ b/crates/asap_sketch_codec/src/lib.rs @@ -0,0 +1,84 @@ +//! Runtime-independent decoding of the sketchlib protobuf envelope. + +use asap_sketchlib::proto::sketchlib::{ + sketch_envelope::SketchState, DdSketchState, KllState, SketchEnvelope, +}; +use asap_sketchlib::DdSketch; +use prost::Message; + +pub fn envelope_state(bytes: &[u8]) -> Result, String> { + SketchEnvelope::decode(bytes) + .map(|envelope| envelope.sketch_state) + .map_err(|error| format!("decode SketchEnvelope: {error}")) +} + +pub fn ddsketch_state(bytes: &[u8]) -> Result<(DdSketchState, f64), String> { + let envelope = + SketchEnvelope::decode(bytes).map_err(|error| format!("decode SketchEnvelope: {error}"))?; + match envelope.sketch_state { + Some(SketchState::Ddsketch(state)) => Ok((state, envelope.sample_p)), + _ => Err("SketchEnvelope contains no DDSketch state".into()), + } +} + +pub fn reconstruct_ddsketch(bytes: &[u8]) -> Result<(DdSketch, f64), String> { + let (state, sample_p) = ddsketch_state(bytes)?; + if !state.alpha.is_finite() || !(0.0..1.0).contains(&state.alpha) || state.alpha == 0.0 { + return Err("DDSketch alpha must be finite and between zero and one".into()); + } + Ok(( + DdSketch::from_raw(state.alpha, state.store_counts, state.store_offset), + sample_p, + )) +} + +pub fn kll_state(bytes: &[u8]) -> Result { + let envelope = + SketchEnvelope::decode(bytes).map_err(|error| format!("decode SketchEnvelope: {error}"))?; + match envelope.sketch_state { + Some(SketchState::Kll(state)) => Ok(state), + _ => Err("SketchEnvelope contains no KLL state".into()), + } +} + +pub fn encode_ddsketch(sketch: &DdSketch) -> Vec { + let envelope = SketchEnvelope { + format_version: 1, + producer: None, + hash_spec: None, + sample_p: 0.0, + sketch_state: Some(SketchState::Ddsketch(DdSketchState { + alpha: sketch.wire_alpha(), + store_counts: sketch.store_counts.clone(), + store_offset: sketch.store_offset, + })), + }; + envelope.encode_to_vec() +} + +pub fn encode_kll(sketch: &asap_sketchlib::sketches::kll::KLL) -> Vec { + use asap_sketchlib::proto::sketchlib::CoinState; + let (state, bit_cache, remaining_bits) = sketch.wire_coin(); + SketchEnvelope { + format_version: 1, + producer: None, + hash_spec: None, + sample_p: 0.0, + sketch_state: Some(SketchState::Kll(KllState { + k: sketch.wire_k(), + m: sketch.wire_m(), + num_levels: sketch.wire_num_levels(), + levels: sketch.wire_levels(), + items: sketch.wire_items(), + coin: Some(CoinState { + state, + bit_cache, + remaining_bits, + }), + offset: 0.0, + value_scale: 0, + residuals: Vec::new(), + })), + } + .encode_to_vec() +} diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index 018635c1a..d53dd0807 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -37,7 +37,12 @@ impl DerivedInputIdentity { root: PostAsapNodeId, frontiers: &BTreeMap, ) -> Result { - if document.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + if ![ + crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, + crate::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION, + ] + .contains(&document.schema_version) + { return Err("unsupported derived program document version".into()); } let decoded = document.decode()?; @@ -106,7 +111,7 @@ impl DerivedInputIdentity { } edges.sort(); let bytes = serde_json::to_vec(&serde_json::json!({ - "version": 1, "payload": node.payload, + "version": 2, "payload": node.payload, "state": node.output_state, "schema": node.output_schema, "guarantee": node.guarantee, "inputs": edges, })) @@ -168,7 +173,7 @@ mod tests { let a = SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); - assert_eq!(a.materializations, b.materializations); + assert_eq!(a.definitions, b.definitions); assert_eq!(a.data_descriptors, b.data_descriptors); let mut renamed = derived.clone(); renamed.metric = "output_alias".into(); @@ -319,7 +324,6 @@ mod tests { config.policy_fingerprint(), SummaryDescriptor::from_config(&config).unwrap(), data, - config.window_layout )] ) .is_err()); diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index ed40942aa..d84764dce 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize}; pub struct QueryNodeId(pub u64); pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 2; +pub const MAINTENANCE_DAG_SCHEMA_VERSION: u32 = 3; /// Versioned, language-neutral Planner DAG persisted with an installed plan. /// Plan lifecycle belongs to the enclosing `PrecomputePlan`; this document @@ -183,12 +184,55 @@ pub struct InstalledPostAsapDag { impl InstalledPostAsapDag { pub fn validate(&self) -> Result<(), String> { - if self.document.schema_version != OWNED_POST_ASAP_DAG_SCHEMA_VERSION - || self.document.query_id.trim().is_empty() - { + if self.document.query_id.trim().is_empty() { return Err("invalid post-ASAP DAG document identity/version".into()); } - self.binding.validate(&self.document.decode()?) + if self.document.schema_version != MAINTENANCE_DAG_SCHEMA_VERSION { + return Err("unsupported maintenance DAG document version".into()); + } + self.binding.validate_maintenance(&self.document.decode()?) + } + + /// Project the selected semantic DAG onto the maintenance ancestors of its + /// stored outputs. + pub fn maintenance_projection(mut self) -> Result { + if self.document.schema_version != OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + return Err("selected DAG has an unsupported document version".into()); + } + self.binding.validate(&self.document.decode()?)?; + if self.binding.precompute_sinks.is_empty() { + return Err("cannot project a DAG without maintenance sinks".into()); + } + let mut included = self + .binding + .precompute_sinks + .iter() + .copied() + .collect::>(); + let mut frontier = self.binding.precompute_sinks.clone(); + while let Some(consumer) = frontier.pop() { + for edge in self + .document + .edges + .iter() + .filter(|edge| edge.consumer == consumer) + { + if included.insert(edge.producer) { + frontier.push(edge.producer); + } + } + } + self.document + .nodes + .retain(|node| included.contains(&node.id)); + self.document + .edges + .retain(|edge| included.contains(&edge.producer) && included.contains(&edge.consumer)); + self.binding.nodes.retain(|id, _| included.contains(id)); + self.document.root = *self.binding.precompute_sinks.first().unwrap(); + self.document.schema_version = MAINTENANCE_DAG_SCHEMA_VERSION; + self.validate()?; + Ok(self) } } @@ -224,6 +268,55 @@ impl BackendExecutableBinding { self.nodes.get(&id) } + pub fn validate_maintenance(&self, dag: &ExecutableDag) -> Result<(), String> { + let ids = dag + .nodes + .iter() + .map(|node| node.id) + .collect::>(); + if ids.is_empty() || self.nodes.keys().copied().collect::>() != ids { + return Err("maintenance binding does not cover its projected DAG".into()); + } + if self.precompute_sinks.is_empty() + || self.precompute_sinks.iter().any(|id| !ids.contains(id)) + { + return Err("maintenance projection has missing sinks".into()); + } + if self.precompute_sinks.iter().any(|id| { + !matches!( + self.node(*id), + Some(BackendNodeBinding::Materialization { .. }) + ) + }) { + return Err("maintenance sink lacks a stored-output binding".into()); + } + for node in &dag.nodes { + match (node.output_state.timing, self.node(node.id)) { + ( + ExecutionTiming::MaintenanceTime, + Some( + BackendNodeBinding::MaintenanceInput + | BackendNodeBinding::Materialization { .. }, + ), + ) => {} + _ => { + return Err(format!( + "query-owned node {} in maintenance projection", + node.id.0 + )) + } + } + } + if dag + .edges + .iter() + .any(|edge| !ids.contains(&edge.producer) || !ids.contains(&edge.consumer)) + { + return Err("maintenance projection has dangling edges".into()); + } + Ok(()) + } + pub fn validate(&self, dag: &ExecutableDag) -> Result<(), String> { let semantic = dag .nodes @@ -277,10 +370,33 @@ mod tests { fn send_sync() {} send_sync::(); let wire = serde_json::json!({ - "schema_version": 2, "query_id": "q", "nodes": [], "edges": [], "root": 0 + "schema_version": 1, "query_id": "q", "nodes": [], "edges": [], "root": 0 }); let document: OwnedPostAsapDag = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(serde_json::to_value(document).unwrap(), wire); assert_eq!(serde_json::to_value(QueryNodeId(9)).unwrap(), 9); } + + #[test] + fn installed_maintenance_dag_rejects_complete_dag_version() { + let installed = InstalledPostAsapDag { + document: OwnedPostAsapDag { + schema_version: OWNED_POST_ASAP_DAG_SCHEMA_VERSION, + query_id: "q".into(), + nodes: Vec::new(), + edges: Vec::new(), + root: PostAsapNodeId(0), + }, + binding: BackendExecutableBinding { + nodes: BTreeMap::new(), + query_sink: PostAsapNodeId(0), + query_plan_sink: QueryNodeId(0), + precompute_sinks: Vec::new(), + }, + }; + assert!(installed + .validate() + .unwrap_err() + .contains("unsupported maintenance DAG")); + } } diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs index 3631b6cef..1122c4200 100644 --- a/crates/asap_types/src/plan_publication.rs +++ b/crates/asap_types/src/plan_publication.rs @@ -33,6 +33,109 @@ pub struct PhysicalPlanInstallRequest { pub adaptation_evidence: Vec, } +/// A projected writer must refer to the query entry installed in the same +/// generation. +pub fn validate_maintenance_query_bindings( + precompute: &PrecomputePlan, + query: &QueryPlan, +) -> Result<(), String> { + for (query_id, installed) in &precompute.executable_dags { + installed.validate()?; + let mut entries = query + .entries + .values() + .filter(|entry| &entry.query_id == query_id); + let entry = entries + .next() + .ok_or("maintenance projection has no query entry")?; + if entries.next().is_some() { + return Err("maintenance projection has ambiguous query entries".into()); + } + if entry.root != installed.binding.query_plan_sink { + return Err("maintenance projection and query entry have different roots".into()); + } + let selected = query + .selected_dags + .get(query_id) + .ok_or("maintenance projection has no selected semantic provenance")?; + if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION + || selected.query_id != *query_id + { + return Err("selected semantic provenance has invalid identity/version".into()); + } + selected.decode()?; + if installed + .document + .nodes + .iter() + .any(|node| !selected.nodes.contains(node)) + || installed + .document + .edges + .iter() + .any(|edge| !selected.edges.contains(edge)) + { + return Err("maintenance projection differs from its selected DAG".into()); + } + } + Ok(()) +} + +/// Every query read must target the installed writer's exact stored output and +/// physical window contract. The catalog describes semantics; these choices +/// belong to the executable plans. +pub fn validate_stored_output_references( + precompute: &PrecomputePlan, + query: &QueryPlan, +) -> Result<(), String> { + let writers = precompute + .schemas + .iter() + .map(|schema| (schema.materialization, schema)) + .collect::>(); + let configs = precompute + .materializations + .iter() + .map(|config| (config.policy_fingerprint().into(), config)) + .collect::>(); + for entry in query.entries.values() { + for binding in entry.materialization_bindings() { + let writer = writers.get(&binding.materialization).ok_or_else(|| { + format!( + "query binding for definition {} has no precompute stored-summary writer", + binding.materialization.as_u64() + ) + })?; + if binding.stored_output_reference != writer.stored_output_reference { + return Err( + "query read and precompute writer have different stored outputs".into(), + ); + } + let config = configs.get(&binding.materialization).ok_or_else(|| { + format!( + "query binding for definition {} has no precompute configuration", + binding.materialization.as_u64() + ) + })?; + let full_slide = matches!( + config.window_layout, + crate::WindowMaterializationLayout::FullWindow + ) + .then_some(config.slide_interval.saturating_mul(1_000)); + if binding.full_window_slide_ms != full_slide { + return Err("query full-window cadence differs from precompute definition".into()); + } + if config.stored_window_ms() != binding.window_ms { + return Err("query pane differs from precompute stored window".into()); + } + if config.pane_origin_ms != binding.pane_origin_ms { + return Err("query pane origin differs from precompute definition".into()); + } + } + } + Ok(()) +} + impl PhysicalPlanPublication { /// Validate every plan against the shared catalog snapshot. pub fn validate(&self) -> Result<(), String> { @@ -49,36 +152,8 @@ impl PhysicalPlanPublication { self.query_plan .validate_against_catalog(catalog) .map_err(|e| e.to_string())?; - let materializations = self - .precompute_plan - .materializations - .iter() - .map(|config| (config.policy_fingerprint(), config)) - .collect::>(); - for entry in self.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("query binding has no precompute materialization")?; - let full_slide = matches!( - config.window_layout, - crate::WindowMaterializationLayout::FullWindow - ) - .then_some(config.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_slide { - return Err( - "query full-window cadence differs from precompute definition".into(), - ); - } - if config.stored_window_ms() != binding.window_ms { - return Err("query pane differs from precompute stored window".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } + validate_maintenance_query_bindings(&self.precompute_plan, &self.query_plan)?; + validate_stored_output_references(&self.precompute_plan, &self.query_plan)?; let mut collectors = std::collections::BTreeSet::new(); for collector in &self.collector_plans { if collector.envelope != self.precompute_plan.envelope diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 746649fc1..a6375da38 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -115,8 +115,7 @@ pub struct PrecomputePlan { pub schemas: Vec, pub producers: Vec, pub materializations: Vec, - /// Planner semantic DAGs and backend-owned placement for this generation. - /// Empty only for legacy/config-only construction paths. + /// Maintenance projections ending at stored outputs. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub executable_dags: BTreeMap, } @@ -219,6 +218,8 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StateSchemaContract { + #[serde(alias = "state_reference")] + pub stored_output_reference: crate::sds::StoredOutputReference, pub schema_id: String, pub schema_version: u32, pub materialization: crate::sds::SummaryDefinitionId, @@ -333,6 +334,9 @@ impl PrecomputePlan { ); let value_projection = materialization.effective_value_projection().clone(); Ok(StateSchemaContract { + stored_output_reference: crate::sds::StoredOutputReference::for_definition( + fingerprint.into(), + ), schema_id: state_schema_id(fingerprint), schema_version: 1, materialization: fingerprint.into(), @@ -787,11 +791,15 @@ impl PrecomputePlan { } } let mut schema_ids = BTreeSet::new(); + let mut stored_outputs = BTreeSet::new(); for schema in &self.schemas { if schema.schema_id.trim().is_empty() || !schema_ids.insert(schema.schema_id.as_str()) + || !stored_outputs.insert(schema.stored_output_reference.stored_output_id) || schema.schema_version == 0 || schema.encodings.is_empty() + || schema.stored_output_reference.validate().is_err() + || schema.stored_output_reference.definition_id != schema.materialization { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 201b9a5fa..1d474e0a6 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -14,7 +14,7 @@ impl PrecomputePlan { pub fn bind_catalog(&mut self, catalog: &SummaryCatalog) -> Result<(), PrecomputePlanError> { for config in &self.materializations { let id = SummaryDefinitionId::from(config.policy_fingerprint()); - catalog.materializations.get(&id).ok_or_else(|| { + catalog.definitions.get(&id).ok_or_else(|| { invalid(format!("missing catalog materialization {}", id.as_u64())) })?; } @@ -53,12 +53,12 @@ impl PrecomputePlan { .iter() .map(|m| SummaryDefinitionId::from(m.policy_fingerprint())) .collect(); - if ids != catalog.materializations.keys().copied().collect() { + if ids != catalog.definitions.keys().copied().collect() { return Err(invalid("catalog/reference/materialization sets differ")); } for config in &self.materializations { let id = SummaryDefinitionId::from(config.policy_fingerprint()); - let binding = &catalog.materializations[&id]; + let binding = &catalog.definitions[&id]; let expected = SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { @@ -66,9 +66,6 @@ impl PrecomputePlan { "summary operator/update contract differs from catalog", )); } - if binding.pane_origin_ms != config.pane_origin_ms { - return Err(invalid("pane origin differs from catalog definition")); - } let data = &catalog.data_descriptors[&binding.data_descriptor_id]; let expected_source = config.source_identity(); if config.table_name.is_some() diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 0d9e58457..2690d1655 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -290,7 +290,7 @@ fn validate_catalog_projection( )); } for id in materializations { - if !catalog.materializations.contains_key(&id) { + if !catalog.definitions.contains_key(&id) { return Err(TransmissionPlanError::Catalog(format!( "unknown materialization {}", id.as_u64() diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 7595ee934..9e3e04bfe 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -24,6 +24,10 @@ pub struct QueryPlan { pub plan_version: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub clickhouse_context: Option, + /// Selected semantic roots retained for provenance; serving executes + /// `entries` and never reconstructs a plan from these documents. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub selected_dags: BTreeMap, pub entries: BTreeMap, } @@ -52,6 +56,7 @@ impl QueryPlan { plan_id: 0, plan_version: 0, clickhouse_context: None, + selected_dags: BTreeMap::new(), entries: BTreeMap::new(), } } @@ -102,7 +107,7 @@ impl QueryPlan { )); } let available = catalog - .materializations + .definitions .keys() .copied() .map(Into::into) @@ -111,7 +116,7 @@ impl QueryPlan { for entry in self.entries.values() { for binding in entry.materialization_bindings() { let identity = catalog - .materializations + .definitions .get(&binding.materialization) .ok_or_else(|| { QueryPlanError::Invalid( @@ -124,20 +129,9 @@ impl QueryPlan { "zero physical pane duration".into(), )); } - if binding.full_window_slide_ms.is_some() - != matches!( - identity.window_layout, - crate::WindowMaterializationLayout::FullWindow - ) - || binding.full_window_slide_ms == Some(0) - { - return Err(QueryPlanError::Invalid( - "query storage layout differs from catalog definition".into(), - )); - } - if binding.pane_origin_ms != identity.pane_origin_ms { + if binding.full_window_slide_ms == Some(0) { return Err(QueryPlanError::Invalid( - "query pane origin differs from catalog definition".into(), + "query full-window cadence must be nonzero".into(), )); } } @@ -154,7 +148,7 @@ impl QueryPlan { "counter readout must directly consume one catalog materialization".into(), )); }; - let identity = &catalog.materializations[&binding.materialization]; + let identity = &catalog.definitions[&binding.materialization]; let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; if !matches!( descriptor.fidelity, @@ -178,6 +172,34 @@ impl QueryPlan { "non-bootstrap QueryPlan has zero plan_version".into(), )); } + for (query_id, selected) in &self.selected_dags { + if query_id != &selected.query_id { + return Err(QueryPlanError::Invalid(format!( + "selected DAG map key `{query_id}` differs from document query ID `{}`", + selected.query_id + ))); + } + if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION + { + return Err(QueryPlanError::Invalid(format!( + "selected DAG `{query_id}` has unsupported schema version {}", + selected.schema_version + ))); + } + selected.decode().map_err(|error| { + QueryPlanError::Invalid(format!("selected DAG `{query_id}` is invalid: {error}")) + })?; + let matching_entries = self + .entries + .values() + .filter(|entry| entry.query_id == *query_id) + .count(); + if matching_entries != 1 { + return Err(QueryPlanError::Invalid(format!( + "selected DAG `{query_id}` must correspond to exactly one query entry; found {matching_entries}" + ))); + } + } if let Some(context) = &self.clickhouse_context { for (template, identities) in &context.window_templates { if !template.starts_with("moving-window-v1:") || identities.is_empty() { @@ -400,6 +422,13 @@ impl QueryPlanEntry { } } if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.stored_output_reference.validate().is_err() + || binding.stored_output_reference.definition_id != binding.materialization + { + return Err(QueryPlanError::Invalid( + "read binding has invalid stored output or definition".into(), + )); + } if binding.readout_lookback_ms == Some(0) { return Err(QueryPlanError::Invalid( "zero semantic readout lookback".into(), @@ -436,6 +465,8 @@ pub enum FallbackPolicy { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MaterializationBinding { + #[serde(alias = "state_reference")] + pub stored_output_reference: crate::sds::StoredOutputReference, /// Complete-window storage advances independently of its stored extent. /// None denotes disjoint pane storage. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index c06216148..896c9556c 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1,5 +1,6 @@ -//! Shared SDS metadata contracts. Summary payload bytes remain storage-engine -//! owned; catalogs and inventories contain identities and state references only. +//! Shared contracts for summary definitions, stored-summary metadata, and plan +//! output bindings. Payload bytes remain storage-engine owned and logically +//! belong to the stored summary identified by this metadata. pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; use crate::{AggregationType, PrecomputeMaterialization}; @@ -53,6 +54,42 @@ impl From for crate::PolicyFingerprint { } } +/// Identity of one persisted producer output within an installed plan version. +/// V1 derives it from the definition ID because the runtime index is keyed by +/// definition; a future schema may allocate independent output IDs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct StoredOutputId(pub u64); + +/// Typed join key carried by both the writer and every bound reader. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoredOutputReference { + #[serde(alias = "state_slot_id")] + pub stored_output_id: StoredOutputId, + pub definition_id: SummaryDefinitionId, +} + +impl StoredOutputReference { + /// V1 binding for the single stored output of a definition. + pub fn for_definition(definition_id: SummaryDefinitionId) -> Self { + Self { + stored_output_id: StoredOutputId(definition_id.as_u64()), + definition_id, + } + } + + pub fn validate(&self) -> Result<(), SdsError> { + if *self == Self::for_definition(self.definition_id) { + Ok(()) + } else { + Err(SdsError( + "stored output differs from its V1 definition binding".into(), + )) + } + } +} + descriptor_id!(SummaryDescriptorId); descriptor_id!(DataDescriptorId); @@ -275,6 +312,8 @@ pub enum InstanceLifecycle { #[serde(deny_unknown_fields)] pub struct SummaryInstance { pub instance_id: SummaryInstanceId, + #[serde(alias = "state_slot_id")] + pub stored_output_id: StoredOutputId, #[serde(alias = "materialization_id")] pub summary_definition_id: SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, @@ -282,6 +321,10 @@ pub struct SummaryInstance { pub time_range: HalfOpenTimeRange, pub group_values: BTreeMap, pub catalog_generation: CatalogGeneration, + /// Source generation selected by an explicit compatibility decision when + /// an unchanged definition reuses a committed payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reused_from_generation: Option, pub placement: SummaryPlacement, pub state_reference: SummaryStateReference, pub status: SummaryInstanceStatus, @@ -292,6 +335,13 @@ pub struct SummaryInstance { impl SummaryInstance { pub fn validate(&self) -> Result<(), SdsError> { + if self.stored_output_id + != StoredOutputReference::for_definition(self.summary_definition_id).stored_output_id + { + return Err(SdsError( + "summary instance has an invalid stored output".into(), + )); + } if self.time_range.start_ms >= self.time_range.end_ms { return Err(SdsError( "summary instance time range must be non-empty".into(), @@ -309,9 +359,23 @@ impl SummaryInstance { "summary instance placement must be resolved".into(), )); } + let payload_generation = if let Some(source) = &self.reused_from_generation { + validate_catalog_generation(source)?; + if source.plan_id != self.catalog_generation.plan_id + || source.plan_version >= self.catalog_generation.plan_version + { + return Err(SdsError( + "stored summary has invalid reuse provenance".into(), + )); + } + source.plan_version + } else { + self.catalog_generation.plan_version + }; if self.state_reference.store.is_empty() || self.state_reference.key.is_empty() || self.state_reference.state_schema_version == 0 + || self.state_reference.generation != payload_generation { return Err(SdsError( "summary instance has invalid state reference".into(), @@ -364,6 +428,38 @@ impl ObservedSummaryInventory { } Ok(()) } + + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), SdsError> { + self.validate()?; + let generation = catalog + .reference() + .map_err(|error| SdsError(error.to_string()))?; + for instance in self.instances.values() { + if instance.catalog_generation != generation { + return Err(SdsError( + "summary instance belongs to another plan generation".into(), + )); + } + let definition = catalog + .definitions + .get(&instance.summary_definition_id) + .ok_or_else(|| SdsError("summary instance has no catalog definition".into()))?; + if instance.summary_descriptor_id != definition.summary_descriptor_id + || instance.data_descriptor_id != definition.data_descriptor_id + || instance.state_reference.state_schema_version + != catalog.summary_descriptors[&definition.summary_descriptor_id] + .state_schema_version + { + return Err(SdsError( + "summary instance differs from its catalog definition".into(), + )); + } + } + Ok(()) + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -1201,6 +1297,7 @@ mod tests { fn observed_instance(lifecycle: InstanceLifecycle) -> SummaryInstance { SummaryInstance { instance_id: SummaryInstanceId::new("instance-1").unwrap(), + stored_output_id: StoredOutputId(7), summary_definition_id: SummaryDefinitionId(crate::PolicyFingerprint(7)), summary_descriptor_id: descriptor( 200, @@ -1223,6 +1320,7 @@ mod tests { plan_version: 2, snapshot_sha256: "abc".into(), }, + reused_from_generation: None, placement: SummaryPlacement { producer_id: "producer".into(), storage_node_id: "store".into(), @@ -1231,7 +1329,7 @@ mod tests { store: "summary-store".into(), key: "state/1".into(), state_schema_version: 1, - generation: 1, + generation: 2, sequence: 3, checksum: None, }, @@ -1260,6 +1358,48 @@ mod tests { inventory.validate().unwrap(); } + #[test] + fn stored_output_and_payload_version_must_match_instance_definition() { + let mut instance = observed_instance(InstanceLifecycle::Persistent); + instance.stored_output_id = StoredOutputId(8); + assert!(instance.validate().is_err()); + instance.stored_output_id = StoredOutputId(7); + instance.state_reference.generation = 3; + assert!(instance.validate().is_err()); + let mut reference = StoredOutputReference::for_definition(instance.summary_definition_id); + reference.stored_output_id = StoredOutputId(8); + assert!(reference.validate().is_err()); + } + + #[test] + fn stored_output_reference_accepts_legacy_state_slot_field() { + let reference: StoredOutputReference = serde_json::from_value(json!({ + "state_slot_id": 7, + "definition_id": 7 + })) + .unwrap(); + assert_eq!(reference.stored_output_id, StoredOutputId(7)); + reference.validate().unwrap(); + assert_eq!( + serde_json::to_value(reference).unwrap(), + json!({"stored_output_id": 7, "definition_id": 7}) + ); + } + + #[test] + fn reused_payload_requires_an_older_compatible_plan_generation() { + let mut instance = observed_instance(InstanceLifecycle::Persistent); + let mut source = instance.catalog_generation.clone(); + source.plan_version -= 1; + instance.reused_from_generation = Some(source.clone()); + instance.state_reference.generation = source.plan_version; + instance.validate().unwrap(); + instance.reused_from_generation.as_mut().unwrap().plan_id += 1; + assert!(instance.validate().is_err()); + instance.reused_from_generation = Some(instance.catalog_generation.clone()); + assert!(instance.validate().is_err()); + } + #[test] fn invalid_range_and_lease_are_rejected() { let mut instance = observed_instance(InstanceLifecycle::Ephemeral { diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4d1ac0628..8212e0ad7 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -1,7 +1,7 @@ //! Authoritative descriptor snapshot for a compiled physical plan. //! -//! Execution plans keep their compatibility fields during migration. This -//! catalog owns semantic definitions, not producer placement or pane state. +//! The catalog owns semantic definitions; executable plans own writer and +//! reader bindings, while runtime inventory owns placement and pane state. use std::collections::BTreeMap; @@ -10,30 +10,17 @@ use crate::sds::{ SummaryDescriptor, SummaryDescriptorId, }; use crate::PolicyFingerprint; -use crate::WindowMaterializationLayout; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 2; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 3; -/// Stable materialization identity binds operator and population descriptors. -/// Concrete intervals, groups and completeness belong to runtime instances. +/// Canonical definition binds operator and population descriptors. Writer +/// layout and concrete state belong to installed plans and runtime instances. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct SummaryDefinitionIdentity { +pub struct SummaryDefinition { pub summary_descriptor_id: SummaryDescriptorId, pub data_descriptor_id: DataDescriptorId, - /// Backend-selected physical representation. It is catalog-visible so - /// producers, readers, lifecycle management, and recovery agree on the - /// concrete state being referenced. - pub window_layout: WindowMaterializationLayout, - /// Pane boundary selected from the shared consumer workload. Legacy - /// snapshots deserialize as unknown and fail closed at pane-only reads. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -44,7 +31,7 @@ pub struct SummaryCatalog { pub plan_version: u64, pub summary_descriptors: BTreeMap, pub data_descriptors: BTreeMap, - pub materializations: BTreeMap, + pub definitions: BTreeMap, } #[derive(Debug, thiserror::Error)] @@ -53,9 +40,9 @@ pub enum SummaryCatalogError { SchemaVersion(u32), #[error("invalid summary catalog descriptor: {0}")] Descriptor(String), - #[error("materialization {0} has conflicting descriptor bindings")] - ConflictingMaterialization(u64), - #[error("materialization {0} references a missing descriptor")] + #[error("definition {0} has conflicting descriptor bindings")] + ConflictingDefinition(u64), + #[error("definition {0} references a missing descriptor")] MissingDescriptor(u64), #[error("catalog reference does not identify the supplied snapshot")] ReferenceMismatch, @@ -123,53 +110,16 @@ impl SummaryCatalog { .with_partitioning(config.partitioning) .with_population_key_encoding(config.population_key_encoding) .with_timestamp_column(config.table_timestamp_column.clone()); - Ok(( - config.policy_fingerprint(), - summary, - data, - config.window_layout.clone(), - config.pane_origin_ms, - )) + Ok((config.policy_fingerprint(), summary, data)) }) .collect::, SummaryCatalogError>>()?; - Self::build_with_origins(plan_id, plan_version, entries) + Self::build(plan_id, plan_version, entries) } pub fn build( plan_id: u64, plan_version: u64, - entries: impl IntoIterator< - Item = ( - PolicyFingerprint, - SummaryDescriptor, - DataDescriptor, - WindowMaterializationLayout, - ), - >, - ) -> Result { - Self::build_with_origins( - plan_id, - plan_version, - entries - .into_iter() - .map(|(fingerprint, summary, data, layout)| { - (fingerprint, summary, data, layout, None) - }), - ) - } - - fn build_with_origins( - plan_id: u64, - plan_version: u64, - entries: impl IntoIterator< - Item = ( - PolicyFingerprint, - SummaryDescriptor, - DataDescriptor, - WindowMaterializationLayout, - Option, - ), - >, + entries: impl IntoIterator, ) -> Result { let mut catalog = Self { schema_version: SUMMARY_CATALOG_SCHEMA_VERSION, @@ -177,28 +127,26 @@ impl SummaryCatalog { plan_version, summary_descriptors: BTreeMap::new(), data_descriptors: BTreeMap::new(), - materializations: BTreeMap::new(), + definitions: BTreeMap::new(), }; - for (fingerprint, summary, data, window_layout, pane_origin_ms) in entries { - let materialization = SummaryDefinitionId::from(fingerprint); + for (fingerprint, summary, data) in entries { + let definition = SummaryDefinitionId::from(fingerprint); summary .validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; data.validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let binding = SummaryDefinitionIdentity { + let binding = SummaryDefinition { summary_descriptor_id: summary.id().clone(), data_descriptor_id: data.id().clone(), - window_layout, - pane_origin_ms, }; if catalog - .materializations - .get(&materialization) + .definitions + .get(&definition) .is_some_and(|old| old != &binding) { - return Err(SummaryCatalogError::ConflictingMaterialization( - materialization.as_u64(), + return Err(SummaryCatalogError::ConflictingDefinition( + definition.as_u64(), )); } catalog @@ -207,7 +155,7 @@ impl SummaryCatalog { catalog .data_descriptors .insert(binding.data_descriptor_id.clone(), data); - catalog.materializations.insert(materialization, binding); + catalog.definitions.insert(definition, binding); } catalog.validate()?; Ok(catalog) @@ -237,7 +185,7 @@ impl SummaryCatalog { )); } } - for (id, binding) in &self.materializations { + for (id, binding) in &self.definitions { if !self .summary_descriptors .contains_key(&binding.summary_descriptor_id) @@ -259,13 +207,13 @@ impl SummaryCatalog { let mut pending = std::collections::BTreeMap::new(); let mut consumers: std::collections::BTreeMap<_, Vec<_>> = std::collections::BTreeMap::new(); - for (id, binding) in &self.materializations { + for (id, binding) in &self.definitions { let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { DataSourceIdentity::Derived { input } => input.inputs.clone(), _ => Default::default(), }; for source in &dependencies { - if !self.materializations.contains_key(source) { + if !self.definitions.contains_key(source) { return Err(SummaryCatalogError::Descriptor( "derived input references missing summary".into(), )); @@ -377,7 +325,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); assert_eq!(catalog.data_descriptors.len(), 3); - assert_eq!(catalog.materializations.len(), 3); + assert_eq!(catalog.definitions.len(), 3); } #[test] @@ -409,7 +357,7 @@ mod tests { SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); + assert_eq!(catalog.definitions.len(), 2); assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); } @@ -441,7 +389,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 2); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); + assert_eq!(catalog.definitions.len(), 2); } // Construction order cannot affect the published snapshot bytes. @@ -459,20 +407,23 @@ mod tests { } #[test] - fn catalog_persists_definition_pane_origin() { + fn catalog_definitions_do_not_store_writer_layout() { let mut materialization = config("requests", "", 60); materialization.pane_origin_ms = Some(7_000); let id = SummaryDefinitionId::from(materialization.policy_fingerprint()); let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); - assert_eq!(catalog.materializations[&id].pane_origin_ms, Some(7_000)); + assert!(catalog.definitions.contains_key(&id)); + assert!( + !serde_json::to_value(&catalog).unwrap()["definitions"][id.as_u64().to_string()] + .as_object() + .unwrap() + .contains_key("pane_origin_ms") + ); - let mut legacy = serde_json::to_value(&catalog).unwrap(); - legacy["materializations"][id.as_u64().to_string()] - .as_object_mut() - .unwrap() - .remove("pane_origin_ms"); - let decoded: SummaryCatalog = serde_json::from_value(legacy).unwrap(); - assert_eq!(decoded.materializations[&id].pane_origin_ms, None); + let mut invalid = serde_json::to_value(&catalog).unwrap(); + invalid["definitions"][id.as_u64().to_string()]["pane_origin_ms"] = + serde_json::json!(7_000); + assert!(serde_json::from_value::(invalid).is_err()); } // The same materialization cannot silently rebind to another population. @@ -492,24 +443,14 @@ mod tests { 1, 1, [ - ( - first.policy_fingerprint(), - summary.clone(), - data[0].clone(), - first.window_layout.clone(), - ), - ( - first.policy_fingerprint(), - summary, - data[1].clone(), - first.window_layout.clone(), - ), + (first.policy_fingerprint(), summary.clone(), data[0].clone()), + (first.policy_fingerprint(), summary, data[1].clone()), ], ) .unwrap_err(); assert!(matches!( error, - SummaryCatalogError::ConflictingMaterialization(_) + SummaryCatalogError::ConflictingDefinition(_) )); } @@ -557,7 +498,7 @@ mod tests { #[test] fn empty_catalog_is_valid() { let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - assert!(catalog.materializations.is_empty()); + assert!(catalog.definitions.is_empty()); catalog.validate().unwrap(); } } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 60d4549a0..7482c5210 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] # Internal crates (workspace) asap_types.workspace = true +asap_sketch_codec = { path = "../crates/asap_sketch_codec" } # Phase 9: the control plane is now an in-process library inside the # backend binary. Wiring up the in-process OpAMP server + capability-map # exposure is a follow-up after Phase 4 (centralized series_id @@ -72,9 +73,6 @@ asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch # existing PUBLIC `from_legacy_matrix`/`sketch_matrix`/`topk_heap_items` API. # Already in the lock as a transitive dep of `asap_sketchlib`. rmp-serde = "1.3" -# Shared collector ingest implementation. This follows ASAPCollector's -# current main branch alongside the direct Sketchlib dependency above. -asap-precompute-rs = { git = "https://github.com/ProjectASAP/ASAPCollector", branch = "main" } # Persistence layer (SketchStore parts / manifest / Tier-2 cache) moka = { version = "0.12", features = ["sync"] } memmap2 = "0.9" diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 81781851b..eb057edc9 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -89,6 +89,7 @@ async fn main() { tables: HashMap::from([("raw_samples".into(), schema)]), accuracy: AccuracyTarget::Epsilon(0.01), }), + selected_dags: Default::default(), entries: BTreeMap::new(), }; let active = validate_and_build_runtime_plan( diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 8de15bfd4..230fde049 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -2565,71 +2565,20 @@ fn decode_modified_otlp_sketch_bytes( match encoding { ENCODING_PROTO => match algorithm { - // Phase 3 step 3: DDSketch and KLL envelope-parsing / - // sketch reconstruction route through the shared - // `edge_runtime_adapter`, which delegates to - // `asap-precompute-rs`'s `Sketch` trait. Backend's - // accumulator wraps the result. Byte parity with Go is - // covered by `asap_sketchlib` PRs #40 (DDSketch) and #41 - // (KLL). - // - // HLL / CountSketch / CountMinSketch byte parity is - // tracked under ProjectASAP/ASAPCollector#243 — until it - // lands those three sketches keep using the backend's - // existing per-accumulator decoder. + // The neutral codec accepts both full envelopes and supported bare + // states. Query accumulators retain their family-specific readouts. SketchAlgorithm::DDSketch => { - use crate::precompute_engine::operators::edge_runtime_adapter::{ - reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, - }; - // Prefer the asap-precompute-rs runtime path (envelope- - // wrapped bytes, the canonical edge-framework wire format). - // If the input is a bare `DdSketchState` (as some unit-test - // / pre-envelope agent payloads still emit, mirrored by the - // PR #14 contract on `from_sketchlib_proto_bytes`), the - // adapter returns an error decoding the envelope — fall - // back to the backend's native decoder which already - // accepts both shapes. - match reconstruct_via_runtime(RtSketchType::DDSketch, bytes) { - Ok(ReconstructedSketch::DdSketch(inner)) => { - // The runtime reconstruction discards the envelope's - // sample_p; re-read it from the same full-frame bytes - // so a sampled series rescales its Count by 1/p. - let sample_p = DDSketchAccumulator::sample_p_from_envelope_bytes(bytes); - Ok(Box::new(DDSketchAccumulator { inner, sample_p })) - } - Ok(_) => { - Err("edge_runtime_adapter returned non-DDSketch reconstruction".into()) - } - Err(_) => Ok(Box::new(DDSketchAccumulator::from_sketchlib_proto_bytes( - bytes, - )?)), - } - } - SketchAlgorithm::Kll => { - use crate::precompute_engine::operators::edge_runtime_adapter::{ - reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, + let (inner, sample_p) = asap_sketch_codec::reconstruct_ddsketch(bytes)?; + let sample_p = if sample_p.is_finite() && sample_p > 0.0 && sample_p < 1.0 { + sample_p + } else { + 1.0 }; - // Same envelope-vs-bare-state handling as DDSketch above. - // Backend's KLL accumulator owns the wire-format-aligned - // `KllSketch` rather than the high-throughput `KLL` - // that asap-precompute-rs's `KLLWrapper` wraps internally - // — when the adapter succeeds, bridge by re-feeding the - // wrapper's snapshot bytes through backend's existing - // decoder. The envelope work (decode + state extraction - // + reconstruction) has already happened in the runtime - // adapter; this final step just reshapes into backend's - // accumulator type. On envelope-decode failure (bare - // state bytes) fall through to the native decoder. - match reconstruct_via_runtime(RtSketchType::KLLSketch, bytes) { - Ok(ReconstructedSketch::Kll { snapshot_bytes }) => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&snapshot_bytes)?, - )), - Ok(_) => Err("edge_runtime_adapter returned non-KLL reconstruction".into()), - Err(_) => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - } + Ok(Box::new(DDSketchAccumulator { inner, sample_p })) } + SketchAlgorithm::Kll => Ok(Box::new( + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, + )), SketchAlgorithm::Cms => Ok(Box::new( CountMinSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), @@ -3849,7 +3798,7 @@ mod sid_resolution_tests { async fn delta_apply_rotates_per_series_base_at_window_boundary() { use crate::precompute_engine::operators::DDSketchAccumulator; use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; - use asap_sketchlib::proto::sketchlib::DdSketchState; + use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; let (state, drain) = make_state().await; @@ -3881,10 +3830,13 @@ mod sid_resolution_tests { ); // ── Window 1: full frame. Base buckets [10, 0, 5]. ── - let full_w1 = DdSketchState { - alpha: 0.01, - store_counts: vec![10, 0, 5], - store_offset: 0, + let full_w1 = SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(DdSketchState { + alpha: 0.01, + store_counts: vec![10, 0, 5], + store_offset: 0, + })), + ..Default::default() } .encode_to_vec(); route_modified_otlp_sketches_to_precompute( diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index b6d3bfd73..7459c6e24 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -1628,6 +1628,9 @@ mod tests { let binding = asap_types::query_plan::MaterializationBinding { full_window_slide_ms: None, materialization: asap_types::PolicyFingerprint(policy).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + asap_types::PolicyFingerprint(policy).into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 382ec0b43..1608d3e09 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2552,6 +2552,7 @@ mod tests { plan_id: 7, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: Default::default(), }), storage_routing: Arc::new( @@ -5735,43 +5736,10 @@ pub fn validate_and_build_runtime_plan( .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; } - for entry in request.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let materialization = request - .precompute_plan - .materializations - .iter() - .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) - .ok_or_else(|| "query binding has no precompute definition".to_string())?; - if binding.window_ms != materialization.stored_window_ms() { - return Err( - "query physical pane duration differs from installed precompute definition" - .into(), - ); - } - if binding.pane_origin_ms != materialization.pane_origin_ms { - return Err( - "query physical pane origin differs from installed precompute definition" - .into(), - ); - } - // `full_window_slide_ms` is `#[serde(default)]`, so a publication from an - // older controller -- or one replayed from a stored artifact -- arrives as - // `None` on a FullWindow materialization. Without this gate the readout - // silently takes the overlap-merging path and counts observations twice, - // which is exactly what the full-window binding exists to prevent. - let full_window_slide_ms = matches!( - materialization.window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ) - .then_some(materialization.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_window_slide_ms { - return Err( - "query window layout differs from installed precompute definition".into(), - ); - } - } - } + asap_types::plan_publication::validate_stored_output_references( + &request.precompute_plan, + &request.query_plan, + )?; let runtime_materializations = request .precompute_plan .runtime_materializations() @@ -5799,6 +5767,10 @@ pub fn validate_and_build_runtime_plan( .query_plan .validate(&typed_fps) .map_err(|error| format!("QueryPlan validation error: {error}"))?; + asap_types::plan_publication::validate_maintenance_query_bindings( + &request.precompute_plan, + &request.query_plan, + )?; let storage_routing = match request.storage_routing.as_ref() { Some(value) => Arc::new( crate::storage_engines::types::BackendStorageRouting::from_json_payload(value) @@ -6046,12 +6018,30 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon .producers .iter() .map(|producer| { - ( - asap_types::sds::SummaryDefinitionId::from(producer.materialization), - producer.producer_id.clone(), - ) + let definition = asap_types::sds::SummaryDefinitionId::from(producer.materialization); + active + .precompute_plan + .schemas + .iter() + .find(|schema| schema.materialization == definition) + .map(|schema| { + ( + definition, + ( + schema.stored_output_reference.stored_output_id, + producer.producer_id.clone(), + ), + ) + }) }) - .collect(); + .collect::>>(); + let Some(producers) = producers else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"status":"error","error":"precompute producer has no stored-output binding"})), + ) + .into_response(); + }; let reporter = std::env::var("HOSTNAME").unwrap_or_else(|_| "asapquery-backend".into()); match state.summary_store.observed_summary_inventory( &reporter, @@ -6841,6 +6831,48 @@ mod catalog_install_tests { ) } + #[test] + fn complete_dag_cannot_be_installed_as_maintenance() { + let mut request = request(); + let query_id = request + .precompute_plan + .executable_dags + .iter() + .next() + .map(|(id, _)| id.clone()) + .expect("fixture has a maintained summary"); + request + .precompute_plan + .executable_dags + .get_mut(&query_id) + .unwrap() + .document = request.query_plan.selected_dags[&query_id].clone(); + assert!(install(request) + .unwrap_err() + .contains("unsupported maintenance DAG")); + } + + #[test] + fn selected_dag_identity_is_validated_even_without_using_its_projection() { + let mut request = request(); + let query_id = request + .query_plan + .selected_dags + .keys() + .next() + .cloned() + .expect("fixture has selected DAG provenance"); + request + .query_plan + .selected_dags + .get_mut(&query_id) + .unwrap() + .query_id = "different-query".into(); + assert!(install(request) + .unwrap_err() + .contains("differs from document query ID")); + } + #[test] fn invalid_clickhouse_entry_cannot_change_active_generation() { let active = install(request()).expect("baseline plan installs"); @@ -6949,7 +6981,7 @@ mod catalog_install_tests { }) .expect("demo has maintained summaries"); binding.window_ms += 1; - assert!(install(request).unwrap_err().contains("pane duration")); + assert!(install(request).unwrap_err().contains("query pane differs")); } #[test] diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 91c109e81..ecf778a12 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -52,7 +52,7 @@ struct Args { /// Versioned canonical QueryWorkload + DataWorkload and backend-local /// implementation evidence. The ASAPQuery profile invokes the pinned - /// Planner and PhysicalPlanCompiler at startup when this is supplied. + /// Planner and DeploymentPlanCompiler at startup when this is supplied. #[arg(long)] planning_snapshot: Option, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index e8f416e55..44f7d67db 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -304,8 +304,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { }) } payload => Err(format!( - "maintenance operator {:?} has no summary-state implementation", - payload + "maintenance operator {payload:?} has no summary-state implementation" )), } } @@ -2157,6 +2156,27 @@ mod tests { asap_types::PolicyFingerprint(value).into() } + fn maintenance_only( + mut dag: ExecutableDag, + mut binding: BackendExecutableBinding, + ) -> (ExecutableDag, BackendExecutableBinding) { + let retained = dag + .nodes + .iter() + .filter(|node| { + node.output_state.timing + == planner_types::post_asap::ExecutionTiming::MaintenanceTime + }) + .map(|node| node.id) + .collect::>(); + dag.nodes.retain(|node| retained.contains(&node.id)); + dag.edges + .retain(|edge| retained.contains(&edge.producer) && retained.contains(&edge.consumer)); + binding.nodes.retain(|id, _| retained.contains(id)); + dag.root = binding.precompute_sinks[0]; + (dag, binding) + } + #[test] fn cohort_lineage_is_order_independent_and_binds_every_input() { use crate::storage_engines::sketch_db::index::FrozenExactWindows; @@ -2500,6 +2520,7 @@ mod tests { ); scheduled_binding.query_sink = PostAsapNodeId(4); scheduled_binding.query_plan_sink = control_plane::query_plan::QueryNodeId(4); + let (dag, scheduled_binding) = maintenance_only(dag, scheduled_binding); let scheduled_adapter = OperatorAdapter { binding: &scheduled_binding, ..adapter @@ -2535,7 +2556,8 @@ mod tests { persistence::config::SketchStorePersistenceConfig, SketchStore, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; - let document = OwnedPostAsapDag::from_executable("immutable-chain".into(), &dag).unwrap(); + let mut document = + OwnedPostAsapDag::from_executable("immutable-chain".into(), &dag).unwrap(); let mut durable_configs = configs.to_vec(); durable_configs[1].derived_input = Some( asap_types::derived_input::DerivedInputIdentity::from_dag( @@ -2545,6 +2567,7 @@ mod tests { ) .unwrap(), ); + document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; let mut durable_binding = scheduled_binding.clone(); durable_binding.nodes.insert( PostAsapNodeId(3), @@ -2905,7 +2928,7 @@ mod tests { planner_types::pre_asap::ColumnRef::SampleValue, ); } - let document = + let mut document = OwnedPostAsapDag::from_executable("two-source-fixture".into(), &dag).unwrap(); let mut target = configs[1].clone(); if complete_groups { @@ -2922,6 +2945,7 @@ mod tests { ) .unwrap(), ); + document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; let mut binding = binding.clone(); for (node, summary_definition) in [ (1, first_id), @@ -3870,10 +3894,17 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; + let (dag, binding) = maintenance_only(dag, binding); bundle.precompute_plan.executable_dags = BTreeMap::from([( "retry".into(), InstalledPostAsapDag { - document: OwnedPostAsapDag::from_executable("retry".into(), &dag).unwrap(), + document: { + let mut document = + OwnedPostAsapDag::from_executable("retry".into(), &dag).unwrap(); + document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; + document + }, binding, }, )]); @@ -3999,6 +4030,7 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], }; + let (dag, binding) = maintenance_only(dag, binding); let source = sum(2.0); let adapter = OperatorAdapter { binding: &binding, @@ -4074,6 +4106,7 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; + let (dag, binding) = maintenance_only(dag, binding); let adapter = OperatorAdapter { binding: &binding, inputs: MaintenanceInputs::Live { diff --git a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs index 86b95a5ba..2874f5102 100644 --- a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs @@ -56,45 +56,11 @@ impl DatasketchesKLLAccumulator { /// DataCollector's `kllprocessor` emits when /// `encoding = KLL_SKETCH_ENCODING_PROTO`. /// - /// ⚠ This is a **lossy statistical reconstruction**, not a - /// bit-identical round-trip: the `KllState` proto carries the - /// retained items in level order plus an explicit `levels[]` - /// boundary array, but sketch-core's `KllSketch` backend types - /// keep their level structure private. Rather than touch - /// upstream `asap_sketchlib` to add a typed-state constructor, - /// we build a fresh `DatasketchesKLLAccumulator` with the same - /// `k` and replay every retained item through `update()`. - /// Quantile estimates on the reconstructed sketch are - /// approximately equivalent to the source's — within KLL's - /// own rank-error bound, which is the same bound the source - /// already inherited — so Phase 1 hot-path queries that hit - /// the reconstructed sketch return answers the user would - /// already have accepted from the source. Bit-identical - /// reconstruction is tracked as a sketchlib upstream follow-up. + /// The neutral codec decodes the sketchlib envelope. + /// The level-aware constructor below preserves the supplied retained + /// sample layout without replaying updates. pub fn from_sketchlib_proto_bytes(buffer: &[u8]) -> Result> { - use asap_sketchlib::proto::sketchlib::{sketch_envelope, KllState, SketchEnvelope}; - use prost::Message; - - // DataCollector's kllprocessor wraps the state in a - // `SketchEnvelope{kll: KllState}` via sketchlib-go's - // `SerializePortableFO` + `proto.Marshal`. Try envelope first, - // fall back to bare `KllState` for callers (e.g. unit tests) - // that encode the state directly. Mirrors the PR #14 fix on - // `CountMinSketchAccumulator::from_sketchlib_proto_bytes`. - let state = match SketchEnvelope::decode(buffer) { - Ok(env) => match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(st)) => st, - Some(other) => { - return Err(format!( - "SketchEnvelope contains non-KLL sketch: {:?}", - std::mem::discriminant(&other) - ) - .into()); - } - None => KllState::decode(buffer).map_err(|e| format!("decode KllState: {e}"))?, - }, - Err(_) => KllState::decode(buffer).map_err(|e| format!("decode KllState: {e}"))?, - }; + let state = asap_sketch_codec::kll_state(buffer)?; if state.k < 8 { return Err(format!("KllState.k must be >= 8 (got {})", state.k).into()); } @@ -394,6 +360,16 @@ impl MergeableAccumulator for DatasketchesKLLAccumul mod tests { use super::*; + fn encode_state(state: asap_sketchlib::proto::sketchlib::KllState) -> Vec { + use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; + use prost::Message; + SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec() + } + #[test] fn test_datasketches_kll_creation() { let kll = DatasketchesKLLAccumulator::new(200); @@ -590,7 +566,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let acc = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes).expect("decode ok"); @@ -635,7 +611,7 @@ mod tests { residuals: vec![], }; let decoded = - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&state.encode_to_vec()).unwrap(); + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&encode_state(state)).unwrap(); assert_eq!(decoded.inner.count(), source.count() as u64); for q in [0.0, 0.1, 0.5, 0.9, 1.0] { assert_eq!(decoded.inner.quantile(q), source.quantile(q), "q={q}"); @@ -705,7 +681,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let result = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("k must be >= 8")); @@ -727,7 +703,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let result = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("levels length")); diff --git a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs index 926a1e883..0f63348b1 100644 --- a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs @@ -91,42 +91,7 @@ impl DDSketchAccumulator { /// DataCollector's `ddsketchprocessor` emits when /// `encoding = DD_SKETCH_ENCODING_PROTO`. pub fn from_sketchlib_proto_bytes(buffer: &[u8]) -> Result> { - use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; - use prost::Message; - - // DataCollector's ddsketchprocessor wraps the state in a - // `SketchEnvelope{ddsketch: DdSketchState}` via sketchlib-go's - // `SerializePortableFO` + `proto.Marshal`. Try envelope first, - // fall back to bare `DdSketchState` for callers (e.g. unit - // tests) that encode the state directly. Mirrors the PR #14 - // fix on `CountMinSketchAccumulator::from_sketchlib_proto_bytes`. - // Capture the envelope's `sample_p` alongside the state so a `Count` - // query can rescale by `1/p`. Bare `DdSketchState` bytes (no envelope) - // carry no sampling info → `sample_p` 1.0 (no rescale). - let (state, sample_p) = match SketchEnvelope::decode(buffer) { - Ok(env) => { - let sp = env.sample_p; - match env.sketch_state { - Some(sketch_envelope::SketchState::Ddsketch(st)) => (st, sp), - Some(other) => { - return Err(format!( - "SketchEnvelope contains non-DDSketch sketch: {:?}", - std::mem::discriminant(&other) - ) - .into()); - } - None => ( - DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, - 1.0, - ), - } - } - Err(_) => ( - DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))?, - 1.0, - ), - }; + let (state, sample_p) = asap_sketch_codec::ddsketch_state(buffer)?; if !(state.alpha > 0.0 && state.alpha < 1.0) { return Err(format!( "DDSketchState alpha {} out of range (expected 0 < alpha < 1)", @@ -340,14 +305,18 @@ mod tests { // asap_sketchlib#57); the proto now carries only // `alpha`/`store_counts`/`store_offset`. fn encode_state(alpha: f64, store_counts: Vec, store_offset: i32) -> Vec { - use asap_sketchlib::proto::sketchlib::DdSketchState; + use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; let state = DdSketchState { alpha, store_counts, store_offset, }; - state.encode_to_vec() + SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec() } #[test] diff --git a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs b/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs deleted file mode 100644 index fba53304a..000000000 --- a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs +++ /dev/null @@ -1,391 +0,0 @@ -//! Edge-runtime adapter — Phase 3 step 3 of the ASAP edge-framework -//! migration (`docs/design-asap-edge-framework.md`, ADR-0002). -//! -//! Routes the **shared** ingest work (envelope wire-format parsing, -//! per-sketch state extraction, sketch reconstruction, sketch merge) -//! through the [`asap_precompute_rs`] crate so the same code runs in -//! agents (Rust edge runtime) and the backend (this repo). What stays -//! in this repo: the QUERY-side engine — PromQL aggregation, storage, -//! and query planning. See README §"Ingest path consumes -//! asap-precompute-rs" for the contract. -//! -//! # What's shared -//! -//! - **Envelope wire-format parsing.** The runtime view -//! [`SketchEnvelope`] (renamed from a backend-internal struct to -//! asap-precompute-rs's canonical type) decodes the on-the-wire -//! `asap_sketchlib::proto::sketchlib::SketchEnvelope` proto plus the -//! surrounding metadata (window bounds, `agg_id`, encoding tag, -//! sketch type, host-neutral labels, metric name, count, -//! temporality). -//! - **Per-sketch state extraction.** [`unwrap_envelope_state`] dispatches -//! the [`asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState`] -//! oneof into a typed `*State` proto for the requested sketch type, -//! producing the same diagnostic shape every accumulator's -//! `from_sketchlib_proto_bytes` used to repeat by hand. -//! - **Sketch reconstruction (DDSketch + KLL today).** -//! [`reconstruct_via_runtime`] constructs an asap-precompute-rs -//! `*Wrapper`, runs `Sketch::apply_delta(envelope_bytes)` (which is -//! the Layer-3 runtime's reconstruct-from-bytes path), then extracts -//! the underlying `asap_sketchlib::sketches::*` state via -//! `wrapper.inner().clone()`. DDSketch and KLL byte parity holds in -//! `asap_sketchlib::main` (PRs #40, #41); HLL / CountSketch / -//! CountMinSketch are tracked under -//! ProjectASAP/ASAPCollector#243 — until those land we keep the -//! backend's per-accumulator decoders for the three sketches and -//! only delegate envelope-parsing. -//! - **Cross-runtime sketch merge.** [`merge_via_runtime`] uses -//! asap-precompute-rs's `Sketch::merge` + `Sketch::snapshot` round- -//! trip so the merge logic lives in one place. Identical results to -//! `asap_sketchlib::sketches::*::merge_refs` because both call into -//! the same underlying merge implementation. -//! -//! # What stays put -//! -//! - The backend's **per-accumulator query-side surface** (`AggregateCore`, -//! `query_statistic`, `MergeableAccumulator`, ...) — query-side and -//! not what asap-precompute-rs is for. -//! - **Sparse delta application** (`apply_proto_delta_bytes` on the -//! backend's accumulators) — `asap_sketchlib` doesn't yet expose the -//! `compute_delta` family (Go's `sketchlib-go` has it; tracked -//! upstream), so asap-precompute-rs's wrappers fall back to "always -//! full" delta encoding. Backend's typed-delta apply is independent -//! and stays. - -use asap_precompute_rs::{ - envelope::ProtoSketchEnvelope, sketches::DDSketchWrapper, sketches::KLLWrapper, Sketch, -}; -use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; -use prost::Message; - -/// Re-export of asap-precompute-rs's runtime view of the -/// `SketchEnvelope` proto (the prost-generated wire-format type plus -/// surrounding host-neutral metadata: window bounds, `agg_id`, -/// encoding tag, sketch-type tag, labels, metric name, count, -/// temporality). -/// -/// Kept as a re-export so all backend code that touches the runtime -/// envelope reaches for the canonical type from the edge-framework -/// runtime — preventing a "backend-flavored" runtime view from -/// drifting alongside the agent-flavored one. -pub use asap_precompute_rs::envelope::{Encoding, SketchEnvelope, SketchType}; - -/// Re-export the [`asap_precompute_rs::Sketch`] trait family so backend -/// code that wants to operate on envelope bytes via the host-neutral -/// runtime trait (`snapshot`, `apply_delta`, `merge`, `reset`) imports -/// from one place. -pub use asap_precompute_rs::{CardinalitySketch, FrequencySketch, QuantileSketch}; - -/// Decode the wire-format `asap_sketchlib` `SketchEnvelope` proto from -/// `bytes` and return the inner [`SketchState`] oneof variant. -/// -/// Mirrors the per-accumulator `match SketchEnvelope::decode(buffer) -/// → Some(SketchState::X(st)) → st` ladder that every -/// `from_sketchlib_proto_bytes` used to inline. The fall-back to -/// decoding `bytes` as the bare typed-state proto is preserved at the -/// caller so the existing PR #14 contract continues to work for unit -/// tests that encode the state directly (without an envelope wrapper). -/// -/// This is the **single shared envelope-unwrap path** between -/// asap-precompute-rs and backend ingest. asap-precompute-rs's -/// per-wrapper `decode_envelope` does the same prost-decode + -/// oneof-extraction work; the difference is that asap-precompute-rs -/// then constructs a typed `asap_sketchlib::sketches::*` from the -/// state, which the backend may or may not want depending on the -/// downstream caller. -pub fn unwrap_envelope_state( - bytes: &[u8], -) -> Result, Box> { - let env = - ProtoSketchEnvelope::decode(bytes).map_err(|e| format!("decode SketchEnvelope: {e}"))?; - Ok(env.sketch_state) -} - -/// Result of [`reconstruct_via_runtime`] — backend uses the inner -/// `asap_sketchlib::sketches::*` to construct its own accumulator. -pub enum ReconstructedSketch { - /// Reconstructed [`asap_sketchlib::DdSketch`] state. - DdSketch(asap_sketchlib::DdSketch), - /// Reconstructed KLL — asap-precompute-rs's [`KLLWrapper`] owns - /// the high-throughput `asap_sketchlib::sketches::kll::KLL` - /// internally; backend's KLL accumulator wraps the wire-format- - /// aligned `KllSketch` instead. We surface the wrapper's - /// re-snapshot bytes so the caller can route them through - /// backend's existing `KllSketch::deserialize_msgpack` / - /// proto-state path or replay items via `KllSketch::update()`. - Kll { - /// Bytes of the reconstructed sketch's snapshot — same shape - /// as the input envelope, validated round-trip. - snapshot_bytes: Vec, - }, -} - -/// Reconstruct an `asap_sketchlib` sketch from envelope bytes by -/// delegating envelope parsing + sketch construction to -/// asap-precompute-rs's `Sketch` trait family. -/// -/// The function is sketch-type-aware because the wrappers' constructor -/// parameters (alpha for DDSketch, k+seed for KLL, ...) live partly in -/// the envelope state proto. We peek the state, construct a wrapper -/// with matching parameters, then call [`Sketch::apply_delta`] which -/// runs asap-precompute-rs's canonical decode + reconstruct pathway. -/// -/// Today wires DDSketch (byte parity per asap_sketchlib#40) and KLL -/// (byte parity per asap_sketchlib#41). HLL / CountSketch / -/// CountMinSketch are tracked under ProjectASAP/ASAPCollector#243 — -/// callers fall back to backend's per-accumulator decoder for those. -pub fn reconstruct_via_runtime( - sketch_type: SketchType, - envelope_bytes: &[u8], -) -> Result> { - match sketch_type { - SketchType::DDSketch => { - // Peek the state to learn alpha, then construct the - // wrapper with matching alpha so `apply_delta`'s merge - // step doesn't trip on `DdSketch::merge`'s alpha-equality - // guard. - let state = unwrap_envelope_state(envelope_bytes)?; - let alpha = match &state { - Some(SketchState::Ddsketch(s)) => s.alpha, - Some(_) => { - return Err("envelope is not a DDSketch".into()); - } - None => return Err("envelope has no sketch_state".into()), - }; - if !(alpha > 0.0 && alpha < 1.0) { - return Err(format!("DDSketch alpha {alpha} out of (0,1)").into()); - } - let mut wrapper = DDSketchWrapper::new(alpha); - wrapper - .apply_delta(envelope_bytes) - .map_err(|e| format!("DDSketchWrapper apply_delta: {e}"))?; - Ok(ReconstructedSketch::DdSketch(wrapper.inner().clone())) - } - SketchType::KLLSketch => { - // KLL: peek `k`, construct an empty wrapper, apply. - let state = unwrap_envelope_state(envelope_bytes)?; - let k = match state { - Some(SketchState::Kll(s)) => { - if s.k > i32::MAX as u32 { - return Err(format!("KllState.k too large: {}", s.k).into()); - } - s.k as i32 - } - Some(_) => return Err("envelope is not a KLL".into()), - None => return Err("envelope has no sketch_state".into()), - }; - let mut wrapper = KLLWrapper::new(k, None); - wrapper - .apply_delta(envelope_bytes) - .map_err(|e| format!("KLLWrapper apply_delta: {e}"))?; - // Re-snapshot via the wrapper's `Sketch::snapshot` — - // canonical asap-precompute-rs encode of the reconstructed - // state. Backend's KLL accumulator can then re-decode via - // its existing `from_sketchlib_proto_bytes` path; the - // edge-runtime adapter has done the envelope + state - // unwrap, the wire-format reshape, and the reconstruction - // round-trip. - let snapshot_bytes = wrapper - .snapshot() - .map_err(|e| format!("KLLWrapper snapshot: {e}"))?; - Ok(ReconstructedSketch::Kll { snapshot_bytes }) - } - SketchType::HLLSketch | SketchType::CountSketch | SketchType::CountMinSketch => { - Err(format!( - "reconstruct_via_runtime({sketch_type:?}): byte parity for \ - HLL / CountSketch / CountMinSketch not yet in upstream \ - asap_sketchlib — tracked at ProjectASAP/ASAPCollector#243. \ - Caller must fall back to backend's per-accumulator decoder." - ) - .into()) - } - SketchType::Unspecified => Err("reconstruct_via_runtime: SketchType::Unspecified".into()), - } -} - -/// Snapshot a backend-side `asap_sketchlib::DdSketch` through -/// asap-precompute-rs's `Sketch` trait — the canonical encode path -/// shared with the agent runtime. Used by the round-trip test -/// (`tests/edge_runtime_adapter.rs`). -pub fn snapshot_ddsketch_via_runtime( - sk: &asap_sketchlib::DdSketch, -) -> Result, Box> { - let mut wrapper = DDSketchWrapper::new(sk.alpha); - // Bridge into the wrapper by merging in the existing sketch. - // We can't move-construct the wrapper from a non-empty `DdSketch`, - // but `Sketch::apply_delta` against the existing snapshot bytes - // is equivalent. - if sk.total_count() > 0 { - // Re-encode the source's state into the canonical envelope - // shape that asap-precompute-rs's wrapper recognizes, then - // round-trip through `apply_delta`. Mirrors the agent runtime's - // own merge path. - let bridge_bytes = encode_ddsketch_envelope(sk); - wrapper - .apply_delta(&bridge_bytes) - .map_err(|e| format!("DDSketchWrapper apply_delta (bridge): {e}"))?; - } - wrapper - .snapshot() - .map_err(|e| format!("DDSketchWrapper snapshot: {e}").into()) -} - -/// Encode a backend-side `DdSketch` as the same `SketchEnvelope` proto -/// shape that asap-precompute-rs's `DDSketchWrapper::snapshot` emits. -/// -/// Matches asap-precompute-rs's `DDSketchWrapper::build_state` + -/// `encode_envelope` byte-for-byte — they call into the same -/// `asap_sketchlib::proto::sketchlib::*` types. Lives here so the -/// backend's existing accumulators don't need to import the wrapper -/// internals. -pub fn encode_ddsketch_envelope(sk: &asap_sketchlib::DdSketch) -> Vec { - use asap_sketchlib::proto::sketchlib::{ - sketch_envelope, DdSketchState, SketchEnvelope as ProtoEnvelope, - }; - let state = DdSketchState { - // Use `wire_alpha` so the bytes match Go's - // `sketchlib-go::DDSketch.SerializePortable` (PR - // asap_sketchlib#40 closes this). - alpha: sk.wire_alpha(), - store_counts: sk.store_counts.clone(), - store_offset: sk.store_offset, - // The DataPoint-level scalars (count/sum/min/max) were dropped from - // `DDSketchState` (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); - // the bucket counts carry all reconstructable state. - }; - let env = ProtoEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - // No edge sampling on this path: 0.0 is the proto3 default (dual-read as - // 1.0) so the encoded envelope stays byte-identical. - sample_p: 0.0, - sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), - }; - let mut buf = Vec::with_capacity(env.encoded_len()); - env.encode(&mut buf).expect("prost encode"); - buf -} - -#[cfg(test)] -/// Merge two `DdSketch` instances by routing through asap-precompute-rs's -/// runtime `Sketch::merge`. The result is byte-identical to -/// `asap_sketchlib::DdSketch::merge_refs(&[a, b])` because -/// both paths call the same underlying merge logic. -/// -/// Used by the cross-runtime parity test -/// (`tests/edge_runtime_adapter.rs::ddsketch_merge_via_runtime_matches_native`). -pub fn merge_ddsketches_via_runtime( - a: &asap_sketchlib::DdSketch, - b: &asap_sketchlib::DdSketch, -) -> Result> { - if (a.alpha - b.alpha).abs() > f64::EPSILON { - return Err(format!( - "merge_ddsketches_via_runtime: alpha mismatch ({} vs {})", - a.alpha, b.alpha - ) - .into()); - } - let mut wrapper_a = DDSketchWrapper::new(a.alpha); - if a.total_count() > 0 { - let bridge = encode_ddsketch_envelope(a); - wrapper_a - .apply_delta(&bridge) - .map_err(|e| format!("merge_ddsketches_via_runtime/a: {e}"))?; - } - let mut wrapper_b = DDSketchWrapper::new(b.alpha); - if b.total_count() > 0 { - let bridge = encode_ddsketch_envelope(b); - wrapper_b - .apply_delta(&bridge) - .map_err(|e| format!("merge_ddsketches_via_runtime/b: {e}"))?; - } - // `Sketch::merge` takes a `&dyn Sketch` (round-trips through - // snapshot bytes), which is the canonical Layer-3 runtime fold. - wrapper_a - .merge(&wrapper_b) - .map_err(|e| format!("DDSketchWrapper merge: {e}"))?; - Ok(wrapper_a.inner().clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// asap-precompute-rs's wrapper produces an envelope; backend's - /// shared envelope-unwrap path returns the matching oneof variant. - /// The dedup target. - #[test] - fn unwrap_envelope_state_matches_wrapper_output() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=10 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot ok"); - let state = unwrap_envelope_state(&bytes).expect("decode ok"); - match state { - Some(SketchState::Ddsketch(s)) => { - // `count` was dropped from `DdSketchState` - // (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); - // a non-empty sketch carries it in the bucket store. - assert!(s.store_counts.iter().sum::() > 0); - assert!(s.alpha > 0.0 && s.alpha < 1.0); - } - other => panic!("expected DDSketch state, got {other:?}"), - } - } - - /// asap-precompute-rs's wrapper produces an envelope; the runtime - /// adapter's reconstruction returns a backend-shaped - /// `asap_sketchlib::DdSketch` whose serialized bytes - /// (re-encoded through the same envelope shape) match the - /// original. - #[test] - fn ddsketch_round_trip_through_runtime_adapter() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=100 { - w.update(i as f64); - } - let original_bytes = w.snapshot().expect("snapshot ok"); - let reconstructed = - reconstruct_via_runtime(SketchType::DDSketch, &original_bytes).expect("reconstruct ok"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - ReconstructedSketch::Kll { .. } => panic!("got KLL, expected DDSketch"), - }; - assert_eq!(dd.total_count(), 100); - let re_encoded = encode_ddsketch_envelope(&dd); - assert_eq!( - re_encoded, original_bytes, - "round-trip via runtime adapter must be byte-identical" - ); - } - - /// Construct two non-overlapping DDSketches, merge via the runtime - /// adapter, and verify counts add up. Uses asap-precompute-rs's - /// `Sketch::merge` + `Sketch::snapshot` round-trip. - #[test] - fn ddsketch_merge_via_runtime_combines_counts() { - let mut a = DDSketchWrapper::new(0.01); - for i in 1..=10 { - a.update(i as f64); - } - let mut b = DDSketchWrapper::new(0.01); - for i in 11..=20 { - b.update(i as f64); - } - // Reach into the wrapper's inner via snapshot/decode. - let a_inner = - match reconstruct_via_runtime(SketchType::DDSketch, &a.snapshot().unwrap()).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let b_inner = - match reconstruct_via_runtime(SketchType::DDSketch, &b.snapshot().unwrap()).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let merged = merge_ddsketches_via_runtime(&a_inner, &b_inner).expect("merge ok"); - assert_eq!(merged.total_count(), 20); - } -} diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index af284459e..9df95ba19 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -4,7 +4,6 @@ pub mod count_sketch_accumulator; pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; -pub mod edge_runtime_adapter; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 4cbd57a66..8d20c07f5 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -74,7 +74,9 @@ where key.summary_definition, sink_node.0 ))); } - binding.validate(dag).map_err(ScheduleError::Invalid)?; + binding + .validate_maintenance(dag) + .map_err(ScheduleError::Invalid)?; if !binding.precompute_sinks.contains(&sink_node) || !matches!( binding.node(sink_node), @@ -226,6 +228,28 @@ mod tests { } } + fn maintenance_only( + mut dag: ExecutableDag, + mut binding: BackendExecutableBinding, + sink: PostAsapNodeId, + ) -> (ExecutableDag, BackendExecutableBinding) { + let retained = dag + .nodes + .iter() + .filter(|node| { + node.output_state.timing + == planner_types::post_asap::ExecutionTiming::MaintenanceTime + }) + .map(|node| node.id) + .collect::>(); + dag.nodes.retain(|node| retained.contains(&node.id)); + dag.edges + .retain(|edge| retained.contains(&edge.producer) && retained.contains(&edge.consumer)); + binding.nodes.retain(|id, _| retained.contains(id)); + dag.root = sink; + (dag, binding) + } + fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), @@ -346,9 +370,10 @@ mod tests { root: PostAsapNodeId(3), }; let execute = |dag: &ExecutableDag| { + let (dag, binding) = maintenance_only(dag.clone(), binding(), PostAsapNodeId(3)); execute_precompute_sink( - dag, - &binding(), + &dag, + &binding, PostAsapNodeId(3), key(3), &Subtract, @@ -383,27 +408,16 @@ mod tests { }; let registry = Registry::default(); let sink = Sink::default(); - let first = execute_precompute_sink( - &dag, - &binding(), - PostAsapNodeId(3), - key(3), - ®istry, - &sink, - ) - .unwrap(); + let (dag, binding) = maintenance_only(dag, binding(), PostAsapNodeId(3)); + let first = + execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) + .unwrap(); assert_eq!(*first, 6); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); - let replay = execute_precompute_sink( - &dag, - &binding(), - PostAsapNodeId(3), - key(3), - ®istry, - &sink, - ) - .unwrap(); + let replay = + execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) + .unwrap(); assert!(Arc::ptr_eq(&first, &replay)); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); } @@ -442,6 +456,7 @@ mod tests { bindings .nodes .insert(PostAsapNodeId(0), BackendNodeBinding::QueryInput); + let (dag, bindings) = maintenance_only(dag, bindings, PostAsapNodeId(3)); let registry = FrontierRegistry(Registry::default()); let sink = Sink::default(); let result = @@ -488,7 +503,7 @@ mod tests { }; assert!(matches!( execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink), - Err(ScheduleError::Invalid(message)) if message.contains("query-time node") + Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") )); assert!(matches!( execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(0), ®istry, &sink), diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index bc4d4741d..87c751e06 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -558,7 +558,7 @@ mod tests { config.pane_origin_ms = Some(0); config.table_timestamp_column = Some("timestamp_ms".into()); let sds = SummaryCatalog::from_materializations(41, 1, &[config.clone()]).unwrap(); - let materialization = *sds.materializations.keys().next().unwrap(); + let materialization = *sds.definitions.keys().next().unwrap(); let read = QueryNodeId(0); let readout = QueryNodeId(1); let input_schema = relation_schema(&[ @@ -590,6 +590,7 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization, + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition(materialization), output_grouping: PhysicalGrouping::Reduce(Vec::new()), item_labels: Vec::new(), window_ms: 1_000, @@ -728,6 +729,7 @@ mod tests { tables, accuracy: planner_types::types::AccuracyTarget::Exact, }), + selected_dags: Default::default(), entries: BTreeMap::from([( QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &canonical_sql), entry, diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index ba07c0b9b..19486299c 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -24,7 +24,7 @@ pub(crate) fn resolve( id: SummaryDefinitionId, ) -> Result, EngineError> { let identity = catalog - .materializations + .definitions .get(&id) .ok_or_else(|| miss(format!("unknown materialization {}", id.fingerprint().0)))?; let summary = catalog @@ -256,11 +256,11 @@ mod tests { fn resolves_without_descriptor_copies() { let bundle = fixture(); let catalog = &bundle.summary_catalog; - let id = *catalog.materializations.keys().next().unwrap(); + let id = *catalog.definitions.keys().next().unwrap(); let result = resolve(catalog, id).unwrap(); assert!(std::ptr::eq( result.summary, - &catalog.summary_descriptors[&catalog.materializations[&id].summary_descriptor_id] + &catalog.summary_descriptors[&catalog.definitions[&id].summary_descriptor_id] )); let mut broken = catalog.clone(); broken.data_descriptors.clear(); @@ -270,8 +270,8 @@ mod tests { #[test] fn rejects_operator_fidelity_mismatch_during_resolution() { let mut catalog = catalog_fixture(); - let id = *catalog.materializations.keys().next().unwrap(); - let descriptor_id = catalog.materializations[&id].summary_descriptor_id.clone(); + let id = *catalog.definitions.keys().next().unwrap(); + let descriptor_id = catalog.definitions[&id].summary_descriptor_id.clone(); catalog .summary_descriptors .get_mut(&descriptor_id) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index b5b822641..0ba9c966c 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -960,6 +960,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: MATERIALIZATION.into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + MATERIALIZATION.into(), + ), output_grouping: PhysicalGrouping::Reduce(vec!["job".into()]), window_ms: AT, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 577409c86..59bffd71d 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -194,6 +194,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 41b41ab77..a0327fb8e 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -154,8 +154,7 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { .catalog .as_ref() .and_then(|catalog| { - let definition = - catalog.materializations.get(&binding.materialization)?; + let definition = catalog.definitions.get(&binding.materialization)?; catalog .data_descriptors .get(&definition.data_descriptor_id)? @@ -883,6 +882,10 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, @@ -1074,7 +1077,7 @@ mod tests { #[test] fn compiled_window_schedules_execute_exact_ranges() { use crate::precompute_engine::window_manager::WindowManager; - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; for evaluation_secs in [20, 45, 60, 120, 90] { for phase_ms in [0, 5_000] { for full in [false, true] { @@ -1102,7 +1105,7 @@ mod tests { asap_types::WindowMaterializationLayout::FullWindow ) == full }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; let manager = WindowManager::with_layout( config.window_size, @@ -1173,7 +1176,7 @@ mod tests { // Compile the two readouts, store one pane series, and execute the actual ratio. #[test] fn compiled_shared_sum_panes_preserve_each_lookback() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -1184,7 +1187,7 @@ mod tests { entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let config = &plan.precompute_plan.materializations[0]; let policy = config.policy_fingerprint(); @@ -1314,6 +1317,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1411,6 +1418,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index f48ba405d..13a19d368 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -444,14 +444,22 @@ fn validate_binding_phase( impl QueryExecutionContext<'_> { /// Resolve exactly one compiler-bound materialization. This is the formal - /// QueryPlan path: fingerprint -> SID is the only lookup; metadata checks - /// are integrity checks and never broaden the candidate set. + /// QueryPlan path: the validated stored output resolves to its definition's + /// generation-scoped SID index. Metadata checks never broaden that set. pub fn read_bound_materialization( &self, binding: &asap_types::query_plan::MaterializationBinding, ) -> Result, GroupState)>, SummaryExecutorError> { use asap_types::query_plan::PhysicalGrouping; + if binding.stored_output_reference.validate().is_err() + || binding.stored_output_reference.definition_id != binding.materialization + { + return Err(SummaryExecutorError::Unsupported( + "read binding has invalid stored output", + )); + } + let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { start_ms: i64::try_from(self.t0_ms).map_err(|_| { @@ -1453,6 +1461,9 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + asap_types::PolicyFingerprint(7).into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(7_000), @@ -1888,6 +1899,9 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: Some(20_000), materialization: fp.into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + fp.into(), + ), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 60_000, @@ -1954,6 +1968,9 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: None, materialization: fp.into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + fp.into(), + ), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 8b9f5b57a..8027ccdc6 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -60,6 +60,10 @@ pub(super) fn entry( ) .then_some(config.slide_interval * 1000), materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + config.policy_fingerprint().into(), + ), output_grouping: grouping, item_labels: config.aggregated_labels.labels.clone(), window_ms: config.stored_window_ms(), @@ -128,6 +132,7 @@ pub(super) fn install( plan_id: 1, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: entries .into_iter() .map(|e| (e.canonical_query.clone(), e)) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e0c9ba55c..751b38774 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -618,6 +618,9 @@ pub struct SketchStore { instances: RwLock>, /// Interns immutable SDS descriptors across all Series IDs and panes. descriptors: SummaryDescriptorRegistry, + /// Previous payload generations admitted by an explicit definition + /// compatibility check during plan installation. + compatible_source_generations: RwLock>, /// sid → item_label (the data-point attribute NAME, e.g. "service" /// or "endpoint") for CountMin/CountSketch sids registered in /// per-item mode. Its presence is what makes a CMS sid answerable by @@ -873,6 +876,26 @@ impl SketchStore { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }; + let previous = self.descriptors.authoritative_snapshot(); + let previous_sources = self.compatible_source_generations.read().unwrap().clone(); + let mut compatible_sources = BTreeMap::new(); + if let Some((old_catalog, old_generation)) = &previous { + if old_catalog.plan_id == catalog.plan_id + && old_catalog.plan_version < catalog.plan_version + { + for (id, definition) in &catalog.definitions { + if old_catalog.definitions.get(id) == Some(definition) { + compatible_sources.insert( + *id, + previous_sources + .get(id) + .cloned() + .unwrap_or_else(|| (**old_generation).clone()), + ); + } + } + } + } let closed = self .persistence_metadata .read() @@ -882,6 +905,9 @@ impl SketchStore { .transpose() .map_err(|error| error.to_string())? .flatten(); + // Publish the compatibility decision first so a reader that observes + // the successor catalog can also resolve its admitted source payload. + *self.compatible_source_generations.write().unwrap() = compatible_sources; self.descriptors .install_catalog(Arc::clone(&catalog)) .map_err(|error| error.to_string())?; @@ -903,7 +929,7 @@ impl SketchStore { .ok_or("summary admission requires an installed catalog")?; if coordinates.iter().any(|coordinate| { !catalog - .materializations + .definitions .contains_key(&coordinate.summary_definition_id) }) { return Err("summary admission references an uninstalled definition".into()); @@ -1128,12 +1154,17 @@ impl SketchStore { .map(|set| set.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); + let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidates .into_iter() .filter(|sid| { instances.get(sid).is_some_and(|binding| { - Self::instance_visible_in_generation(binding, generation.as_deref()) + Self::instance_visible_for_read( + binding, + generation.as_deref(), + &compatible_sources, + ) }) }) .collect() @@ -1143,11 +1174,32 @@ impl SketchStore { binding: &SdsBinding, generation: Option<&CatalogGeneration>, ) -> bool { + match generation { + Some(generation) => binding.catalog_generation.as_deref() == Some(generation), + None => !matches!( + binding.data_descriptor.source, + asap_types::sds::DataSourceIdentity::Derived { .. } + ), + } + } + + fn instance_visible_for_read( + binding: &SdsBinding, + generation: Option<&CatalogGeneration>, + compatible_sources: &BTreeMap, + ) -> bool { + if Self::instance_visible_in_generation(binding, generation) { + return true; + } + let Some(generation) = generation else { + return false; + }; + let definition = SummaryDefinitionId::from(binding.metadata.policy_fp); !matches!( binding.data_descriptor.source, asap_types::sds::DataSourceIdentity::Derived { .. } - ) || generation - .is_some_and(|generation| binding.catalog_generation.as_deref() == Some(generation)) + ) && compatible_sources.get(&definition) == binding.catalog_generation.as_deref() + && binding.catalog_generation.as_deref() != Some(generation) } #[cfg(test)] @@ -1196,7 +1248,7 @@ impl SketchStore { &self, reporter_id: &str, storage_node_id: &str, - producers: &BTreeMap, + producers: &BTreeMap, inventory_version: u64, observed_at_ms: i64, ) -> Result { @@ -1212,24 +1264,25 @@ impl SketchStore { snapshot_sha256: reference.snapshot_sha256, }; let instances = self.instances.read().unwrap(); + let compatible_sources = self.compatible_source_generations.read().unwrap(); let durable = self.persistence_read.read().unwrap().clone(); let mut reported = BTreeMap::new(); for (series_id, binding) in instances.iter() { - if !Self::instance_visible_in_generation(binding, Some(&generation)) { + if !Self::instance_visible_for_read(binding, Some(&generation), &compatible_sources) { continue; } + let reused_from_generation = (binding.catalog_generation.as_deref() + != Some(&generation)) + .then(|| binding.catalog_generation.as_deref().cloned()) + .flatten(); let summary_definition_id = SummaryDefinitionId::from(binding.metadata.policy_fp); if binding.metadata.policy_fp.is_unset() - || !catalog - .materializations - .contains_key(&summary_definition_id) + || !catalog.definitions.contains_key(&summary_definition_id) { continue; } - let producer_id = producers - .get(&summary_definition_id) - .map(String::as_str) - .ok_or_else(|| { + let (stored_output_id, producer_id) = + producers.get(&summary_definition_id).ok_or_else(|| { format!( "materialization {} has no producer in the active PrecomputePlan", summary_definition_id.as_u64() @@ -1269,14 +1322,16 @@ impl SketchStore { .map_err(|error| error.to_string())?; let instance = SummaryInstance { instance_id: instance_id.clone(), + stored_output_id: *stored_output_id, summary_definition_id, summary_descriptor_id: binding.summary_descriptor.id().clone(), data_descriptor_id: binding.data_descriptor.id().clone(), time_range: HalfOpenTimeRange { start_ms, end_ms }, group_values, catalog_generation: generation.clone(), + reused_from_generation: reused_from_generation.clone(), placement: SummaryPlacement { - producer_id: producer_id.into(), + producer_id: producer_id.clone(), storage_node_id: storage_node_id.into(), }, state_reference: SummaryStateReference { @@ -1286,7 +1341,9 @@ impl SketchStore { window.0, window.1 ), state_schema_version: binding.summary_descriptor.state_schema_version, - generation: generation.plan_version, + generation: reused_from_generation + .as_ref() + .map_or(generation.plan_version, |source| source.plan_version), sequence: window.1, checksum: None, }, @@ -1356,7 +1413,9 @@ impl SketchStore { observed_at_ms, instances: reported, }; - inventory.validate().map_err(|error| error.to_string())?; + inventory + .validate_against_catalog(&catalog) + .map_err(|error| error.to_string())?; Ok(inventory) } @@ -2410,6 +2469,7 @@ impl SketchStore { .map(|sids| sids.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); + let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidate_sids .iter() @@ -2418,7 +2478,11 @@ impl SketchStore { .get(sid) .map(|m| { required_keys.is_subset(&m.group_by_keys) - && Self::instance_visible_in_generation(m, generation.as_deref()) + && Self::instance_visible_for_read( + m, + generation.as_deref(), + &compatible_sources, + ) }) .unwrap_or(false) }) @@ -2810,7 +2874,7 @@ impl SketchStore { .authoritative_snapshot() .ok_or("derived reactivation requires an authoritative catalog")?; if binding.metadata.policy_fp != definition.fingerprint() - || !catalog.materializations.contains_key(&definition) + || !catalog.definitions.contains_key(&definition) { return Err("derived reactivation differs from its installed definition".into()); } @@ -2830,9 +2894,7 @@ impl SketchStore { .descriptors .authoritative_snapshot() .ok_or("series reactivation requires an authoritative catalog")?; - if *old_definition != Some(definition) - || !catalog.materializations.contains_key(&definition) - { + if *old_definition != Some(definition) || !catalog.definitions.contains_key(&definition) { return Err("series reactivation does not match the installed materialization".into()); } let old_generation = old_generation @@ -3366,7 +3428,7 @@ impl SketchStore { ) { (Some(definition), Some(generation), Some((catalog, installed_generation))) => { if generation != installed_generation - || !catalog.materializations.contains_key(definition) + || !catalog.definitions.contains_key(definition) { tracing::warn!( sid = rec.sid, @@ -3773,7 +3835,11 @@ mod tests { let producers = BTreeMap::from([( SummaryDefinitionId::from(fingerprint), - "producer-a".to_string(), + ( + asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + .stored_output_id, + "producer-a".to_string(), + ), )]); let inventory = store .observed_summary_inventory("backend-a", "store-a", &producers, 1, 100) @@ -3782,6 +3848,11 @@ mod tests { assert_eq!(inventory.instances.len(), 2); let instance = inventory.instances.values().next().unwrap(); assert_eq!(instance.summary_definition_id.fingerprint(), fingerprint); + assert_eq!( + instance.stored_output_id, + asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + .stored_output_id + ); assert_eq!(instance.status, SummaryInstanceStatus::Ready); assert_eq!(instance.completeness, InstanceCompleteness::Unknown); assert!(!instance.group_values.is_empty()); @@ -3805,8 +3876,7 @@ mod tests { inventory.instances.keys().collect::>(), next_inventory.instances.keys().collect::>() ); - let catalog_identity = - &plan.summary_catalog.materializations[&instance.summary_definition_id]; + let catalog_identity = &plan.summary_catalog.definitions[&instance.summary_definition_id]; assert_eq!( instance.summary_descriptor_id, catalog_identity.summary_descriptor_id @@ -3835,7 +3905,11 @@ mod tests { store.register(meta_with_policy(42, fingerprint)); let producers = BTreeMap::from([( SummaryDefinitionId::from(fingerprint), - "producer-a".to_string(), + ( + asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + .stored_output_id, + "producer-a".to_string(), + ), )]); let inventory = store .observed_summary_inventory("backend-a", "store-a", &producers, 1, 100) @@ -5019,6 +5093,59 @@ mod tests { assert!(store.series_ids_for_policy(fingerprint).is_empty()); } + #[test] + fn unchanged_definition_explicitly_reuses_a_committed_previous_generation_payload() { + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let store = SketchStore::new(); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(509, fingerprint)); + store.append_sample(509, BTreeMap::new(), (0, 10_000), sample(1)); + let mut next = plan.summary_catalog; + next.plan_version += 1; + store.install_summary_catalog(Arc::new(next)).unwrap(); + assert_eq!(store.series_ids_for_policy(fingerprint), vec![509]); + let output = asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + .stored_output_id; + let inventory = store + .observed_summary_inventory( + "backend-a", + "store-a", + &BTreeMap::from([( + SummaryDefinitionId::from(fingerprint), + (output, "producer-a".into()), + )]), + 1, + 100, + ) + .unwrap(); + let reused = inventory.instances.values().next().unwrap(); + assert_eq!(reused.stored_output_id, output); + assert_eq!( + reused.reused_from_generation.as_ref().unwrap().plan_version + 1, + reused.catalog_generation.plan_version + ); + let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( + plan.precompute_plan.envelope.plan_id, + plan.precompute_plan.envelope.plan_version + 2, + &[], + ) + .unwrap(); + store + .install_summary_catalog(Arc::new(incompatible)) + .unwrap(); + assert!(store.series_ids_for_policy(fingerprint).is_empty()); + } + #[test] fn catalog_reactivation_uses_new_physical_series_without_old_disk_payload() { use crate::drivers::ingest::series_resolver::SeriesIdResolver; @@ -5179,7 +5306,14 @@ mod tests { assert!(!store.completed_windows.read().unwrap().contains_key(&850)); std::fs::remove_dir(writer.path()).unwrap(); store.seal_finite_summary_input(&generation).unwrap(); - let producers = BTreeMap::from([(fingerprint.into(), "producer".to_string())]); + let producers = BTreeMap::from([( + fingerprint.into(), + ( + asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + .stored_output_id, + "producer".to_string(), + ), + )]); let inventory = store .observed_summary_inventory("backend", "store", &producers, 1, 30_000) .unwrap(); @@ -5415,7 +5549,14 @@ mod tests { .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let definition_id = SummaryDefinitionId::from(fingerprint); - let producers = BTreeMap::from([(definition_id, "producer-a".to_string())]); + let producers = BTreeMap::from([( + definition_id, + ( + asap_types::sds::StoredOutputReference::for_definition(definition_id) + .stored_output_id, + "producer-a".to_string(), + ), + )]); let tmp = tempfile::TempDir::new().unwrap(); let disk = tmp.path().to_path_buf(); let mut metadata = meta_with_policy(506, fingerprint); diff --git a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs index 26341f942..894b15ee6 100644 --- a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs +++ b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs @@ -283,9 +283,8 @@ impl SummaryState { SummaryState::Dd(sk) => { match encoding { // PROTO_DELTA: dispatch on the payload SHAPE, mirroring the - // edge's own `DDSketchWrapper::apply_delta` - // (asap-precompute-rs/src/sketches/ddsketch.rs) — which - // tries the full-envelope decode first, then falls back to + // supported DDSketch frame decoder, which tries the + // full-envelope decode first, then falls back to // the bucket-delta proto. Two wire shapes can arrive on the // ProtoDelta channel: // diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 83778690e..ffefbd250 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -157,15 +157,12 @@ impl SummaryDescriptorRegistry { ); } let materialization = asap_types::sds::SummaryDefinitionId::from(metadata.policy_fp); - let identity = catalog - .materializations - .get(&materialization) - .ok_or_else(|| { - format!( - "materialization {} is absent from the installed SummaryCatalog", - materialization.as_u64() - ) - })?; + let identity = catalog.definitions.get(&materialization).ok_or_else(|| { + format!( + "materialization {} is absent from the installed SummaryCatalog", + materialization.as_u64() + ) + })?; Some(( catalog.summary_descriptors[&identity.summary_descriptor_id].clone(), catalog.data_descriptors[&identity.data_descriptor_id].clone(), @@ -399,7 +396,6 @@ mod tests { asap_types::PolicyFingerprint(7), summary.clone(), data.clone(), - asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, )], ) .unwrap(); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c679d0bcd..d42457fe7 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -748,6 +748,7 @@ mod tests { plan_id, plan_version, clickhouse_context: None, + selected_dags: Default::default(), entries: Default::default(), }), storage_routing: Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), diff --git a/data_plane/src/tests/test_utilities/planning.rs b/data_plane/src/tests/test_utilities/planning.rs index f05dac22e..4a30ba1f0 100644 --- a/data_plane/src/tests/test_utilities/planning.rs +++ b/data_plane/src/tests/test_utilities/planning.rs @@ -1,6 +1,6 @@ //! Synthetic complete quotes for deployment fixtures; never used in production. use control_plane::physical::compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }; pub(crate) fn quoted_snapshot( @@ -20,9 +20,9 @@ pub(crate) fn quoted_snapshot( .enumerate() .filter_map(|(index, candidate)| { let plan = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index 9c8e442f0..cc2d34b06 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -20,7 +20,9 @@ use asap_otel_proto::tonic::metrics::v1::{ KllSketch as OtelKllSketch, KllSketchDataPoint, KllSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::{HllVariant as ProtoHllVariant, HyperLogLogState, KllState}; +use asap_sketchlib::proto::sketchlib::{ + sketch_envelope, HllVariant as ProtoHllVariant, HyperLogLogState, KllState, SketchEnvelope, +}; use asap_sketchlib::{CountMinSketch, CountSketch, HllSketch, HllVariant, MessagePackCodec}; use prost::Message; use serde_json::Value; @@ -317,7 +319,12 @@ fn kll_export(metric: &str, timestamp_ns: u64, raw: &[f64]) -> ExportMetricsServ attributes: labels(), start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec(), encoding: KllSketchEncoding::Proto as i32, flags: 0, series_id: 0, diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 55d9b165f..daa4d60f0 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -46,7 +46,7 @@ fn quote_snapshot_for_frontend_test( metricsql: bool, ) -> control_plane::physical::compiler::BackendLocalPlanningInput { use control_plane::physical::{ - compiler::{PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, + compiler::{DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; let (request, environment) = snapshot @@ -59,9 +59,9 @@ fn quote_snapshot_for_frontend_test( .into_iter() .filter_map(|candidate| { let plan = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let unit_cost = if preferred { 1.0 } else { 1e12 }; @@ -313,17 +313,13 @@ fn is_warm(response: &Value) -> bool { }) } -// Measured ERP parameters must reach the real accumulator and answer held-out +// Confidence-sized KLL parameters must reach the real accumulator and answer // raw samples through the installed QueryPlan, without native fallback. +// Uncertified ERP maxima have separate exact-routing process coverage. #[tokio::test] -#[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] -async fn erp_measured_kll_collector_to_query_oracle() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; +async fn certified_kll_state_to_query_oracle() { + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; const QUERY: &str = "quantile_over_time(0.9, erp_latency[5s])"; - let artifact: Value = serde_json::from_str(include_str!( - "../../control_plane/tests/fixtures/erp-kll-measured.json" - )) - .unwrap(); let mut fixture: Value = serde_json::from_str(include_str!( "../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) @@ -332,15 +328,19 @@ async fn erp_measured_kll_collector_to_query_oracle() { entry["query"] = QUERY.into(); entry["requirements"]["accuracy"] = serde_json::json!({"explicit": {"Epsilon": 0.06}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + // This collector fixture exports KLL state only. An empty ERP artifact + // keeps runtime capability filtering while requiring theoretical sizing. fixture["implementation"]["erp"] = serde_json::json!({ - "distribution": artifact["records"][0]["distribution"], - "artifact": artifact, "implementation": "lib", "error_metric": "max_rank_err", + "distribution": {"workload": {"external": {"dataset": "kll-process-fixture"}}}, + "artifact": {"schema_version": 1, "producer_version": "test", "records": []}, + "implementation": "lib", "error_metric": "max_rank_err", "min_trials": 10, "expected_updates": 1000.0, "expected_queries": 10.0, "expected_merges": 0.0, "retention_seconds": 60.0, "cpu_weight": 1.0, "byte_second_weight": 1e-9, "mode": "hybrid", "runtime": {"allowed_algorithms": ["Kll"], "max_memory_bytes": null} }); let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let window_model = snapshot.physical_inputs.window_cost_model.clone(); let (mut request, mut environment) = snapshot.into_physical_compilation_request().unwrap(); request.allow_mixed_summary_and_exact_execution = false; request.queries[0].group_by_labels = vec!["service".into()]; @@ -355,11 +355,34 @@ async fn erp_measured_kll_collector_to_query_oracle() { environment.target = control_plane::physical::compiler::PhysicalDeploymentTarget::DistributedCollectors; environment.target_collector_ids = vec!["erp-collector".into()]; - let plan = PhysicalPlanCompiler + control_plane::physical::compiler::prepare_window_implementations( + &mut request.queries[0], + &window_model, + environment.target, + request.query_retention_margin_ms, + ) + .unwrap(); + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); + let k = plan.precompute_plan.materializations[0].parameters["k"] + .as_u64() + .unwrap() as u32; + let guarantee = asap_aware_mapping::DefaultAccuracyModel::sketch_guarantee( + &planner_types::post_asap::SketchAlgorithm::Kll, + &planner_types::post_asap::SketchParams::Kll { k }, + &planner_types::post_asap::SketchQuery::Quantile { q: 0.9 }, + ) + .unwrap(); + assert!(asap_aware_mapping::AccuracyModel::satisfies( + &asap_aware_mapping::DefaultAccuracyModel, + &guarantee, + &planner_types::types::AccuracyTarget::EpsilonDelta { + epsilon: 0.06, + delta: 0.01 + } + )); let collector = serde_json::to_value(&plan.collector_plans[0]).unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -416,8 +439,7 @@ async fn erp_measured_kll_collector_to_query_oracle() { .unwrap() .as_millis() as i64; let base = now - now.rem_euclid(5000) - 20000; - // A different deterministic stream from training seed 42; the oracle - // evaluates rank error, not the unrelated relative error of the value. + // The oracle evaluates rank error, not relative error of the value. let raw: Vec = (0..1000) .map(|i| ((i * 7919 + 17) % 1009) as f64 / 1009.0) .collect(); @@ -457,7 +479,7 @@ async fn erp_measured_kll_collector_to_query_oracle() { } }) .await - .expect("ERP plan must answer without exact fallback"); + .expect("certified KLL plan must answer without exact fallback"); let estimate = first_value(&response, "value").expect("numeric estimate"); let rank = raw.iter().filter(|v| **v <= estimate).count() as f64 / raw.len() as f64; assert!( @@ -475,45 +497,20 @@ fn erp_collector_kll_export(plan: &Value, end_ms: u64, raw: &[f64], sequence: u6 ResourceMetrics, ScopeMetrics, }, }; - use asap_precompute_rs::Precompute; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; - let decoded = asap_precompute_rs::CollectorPlan::from_json( - &serde_json::to_vec(plan).unwrap(), - "erp-collector", - ) - .unwrap(); - let config = decoded - .to_precompute_config_set() + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_value(plan.clone()).unwrap(); + assert_eq!(decoded.materializations.len(), 1); + let k = decoded.materializations[0].parameters["k"] + .as_u64() .unwrap() - .configs - .remove(0); - let k = config.sketch_params["k"] as i32; - assert_eq!(k, 32); - let runtime = asap_precompute_rs::precompute::PrecomputeImpl::new( - Some(config), - Some(Box::new(move || { - Box::new(asap_precompute_rs::sketches::KLLWrapper::new(k, Some(123))) - })), - Some(Box::new(asap_precompute_rs::sketches::KLLObserver)), - ); + .try_into() + .unwrap(); + let mut sketch = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(k, 123); for value in raw { - runtime - .observe(&asap_precompute_rs::Observation::new( - end_ms - 500, - "erp_latency", - vec![], - vec![asap_precompute_rs::KeyValue::new("service", "erp")], - asap_precompute_rs::ObservationValue { - kind: asap_precompute_rs::ObservationValueKind::Float, - float: *value, - ..Default::default() - }, - )) - .unwrap(); + sketch.update(value); } - let envelopes = runtime.tick(end_ms); - assert_eq!(envelopes.len(), 1); - let wire = SketchEnvelope::decode(envelopes[0].payload.as_slice()).unwrap(); + let wire = SketchEnvelope::decode(asap_sketch_codec::encode_kll(&sketch).as_slice()).unwrap(); let Some(sketch_envelope::SketchState::Kll(state)) = wire.sketch_state else { panic!("KLL state required") }; @@ -576,7 +573,12 @@ fn erp_collector_kll_export(plan: &Value, end_ms: u64, raw: &[f64], sequence: u6 attributes, start_time_unix_nano: (end_ms - 5000) * 1_000_000, time_unix_nano: end_ms * 1_000_000, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec(), encoding: KllSketchEncoding::Proto as i32, flags: 0, series_id: 0, @@ -602,7 +604,7 @@ async fn registered_temporal_topk_count_sketch_heap() { } async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlgorithm) { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; use planner_types::post_asap::{CompositionOperator, SketchQuery, SummaryFamilyType}; const QUERY: &str = "topk(3, count_over_time(top_endpoint_qps[5s]))"; struct Evidence; @@ -660,7 +662,7 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg &Evidence, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -922,7 +924,7 @@ async fn run_shared_dashboard(multi_pane: bool) { .into_iter() .enumerate() .map(|(index, candidate)| { - let plan = control_plane::physical::compiler::PhysicalPlanCompiler + let plan = control_plane::physical::compiler::DeploymentPlanCompiler .compile_promql(candidate.clone(), environment.clone()) .unwrap(); let manifest = @@ -957,6 +959,12 @@ async fn run_shared_dashboard(multi_pane: bool) { assert!(plan.cost_comparison.is_some()); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!(plan.query_plan.entries.len(), 3); + assert!(plan + .precompute_plan + .executable_dags + .values() + .all(|installed| installed.document.schema_version + == asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION)); if multi_pane { assert!(plan.lifecycle_estimates[0] .window_realization_id diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 29a5b1789..9c1082c54 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -16,7 +16,6 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_precompute_rs::Precompute; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope as ProtoEnvelope}; use control_plane::opamp::{ opamp_proto, CollectorPlanStatus, CollectorPlanStatusKind, COLLECTOR_PLAN_CAPABILITY, @@ -76,49 +75,20 @@ fn ddsketch_export( plan: &serde_json::Value, sequence: u64, ) -> Vec { - let decoded = asap_precompute_rs::CollectorPlan::from_json( - &serde_json::to_vec(plan).unwrap(), - "whole-e2e-collector", - ) - .unwrap(); - let mut configs = decoded.to_precompute_config_set().unwrap().configs; - assert_eq!( - configs.len(), - 1, - "two query roots must create only one producer" - ); - let config = configs.remove(0); - assert_eq!(config.sketch_params["relative_accuracy"], alpha); - let runtime = asap_precompute_rs::precompute::PrecomputeImpl::new( - Some(config), - Some(Box::new(move || { - Box::new(asap_precompute_rs::sketches::DDSketchWrapper::new(alpha)) - })), - Some(Box::new(asap_precompute_rs::sketches::DDSketchObserver)), - ); + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_value(plan.clone()).unwrap(); + assert_eq!(decoded.materializations.len(), 1); + let mut sketch = asap_sketchlib::DdSketch::new(alpha); for value in values { - runtime - .observe(&asap_precompute_rs::Observation::new( - timestamp_ns / 1_000_000 - 500, - metric, - vec![], - vec![asap_precompute_rs::KeyValue::new("service", "whole-e2e")], - asap_precompute_rs::ObservationValue { - kind: asap_precompute_rs::ObservationValueKind::Float, - float: *value, - ..Default::default() - }, - )) - .unwrap(); + sketch.update(*value); } - let envelopes = runtime.tick(timestamp_ns / 1_000_000); - assert_eq!(runtime.stats().input_observations, values.len() as u64); - assert_eq!(envelopes.len(), 1); - assert_eq!(envelopes[0].count, values.len() as u64); - let wire = ProtoEnvelope::decode(envelopes[0].payload.as_slice()).unwrap(); - let Some(sketch_envelope::SketchState::Ddsketch(state)) = wire.sketch_state else { - panic!("expected actual Collector DDSketch state") - }; + assert_eq!(sketch.total_count(), values.len() as u64); + let sketch_bytes = asap_sketch_codec::encode_ddsketch(&sketch); + let wire = ProtoEnvelope::decode(sketch_bytes.as_slice()).unwrap(); + assert!(matches!( + wire.sketch_state, + Some(sketch_envelope::SketchState::Ddsketch(_)) + )); let materialization = plan["materializations"][0]["materialization"] .as_u64() .unwrap(); @@ -166,7 +136,7 @@ fn ddsketch_export( attributes, start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: sketch_bytes, encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, @@ -297,12 +267,9 @@ async fn respond_next_collector_plan( .expect("collector-plan custom message"); assert_eq!(custom.capability, COLLECTOR_PLAN_CAPABILITY); assert_eq!(custom.r#type, COLLECTOR_PLAN_MESSAGE); - let decoded = asap_precompute_rs::collector_plan::CollectorPlan::from_json( - &custom.data, - "whole-e2e-collector", - ) - .expect("actual Collector validator accepts the emitted plan"); - assert_eq!(decoded.to_precompute_config_set().unwrap().configs.len(), 1); + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_slice(&custom.data).expect("decode backend CollectorPlan"); + assert_eq!(decoded.materializations.len(), 1); let plan: serde_json::Value = serde_json::from_slice(&custom.data).expect("decode collector physical plan"); let plan_id = plan["envelope"]["plan_id"] @@ -395,7 +362,6 @@ async fn quote_workload( } #[tokio::test] -#[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] async fn production_control_plane_to_data_plane_otlp_to_promql() { let control_binary = std::env::var("ASAP_E2E_CONTROL_PLANE_BIN") .expect("ASAP_E2E_CONTROL_PLANE_BIN is set by scripts/e2e.sh whole"); @@ -521,6 +487,12 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { "backend_compat": control_plane::physical::compiler::BACKEND_COMPAT, "apply_timeout_ms": 10000 }); + let workload: serde_json::Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + request["data_workload"] = workload["data_workload"].clone(); + request["data_workload"]["data_ingestion_interval"]["value"] = 1_000.into(); let mut second = request["queries"][0].clone(); second["query_id"] = "whole-process-e2e-median".into(); second["query_string"] = "quantile_over_time(0.5, whole_process_e2e_latency_ms[1s])".into(); @@ -730,9 +702,14 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!( - still_active, physical_plan_status, + still_active["plans"], physical_plan_status["plans"], "failed rollout changed active plan" ); + assert_eq!( + still_active["materializations"][0]["materialization"], + physical_plan_status["materializations"][0]["materialization"], + "failed rollout changed the installed materialization" + ); let still_warm: serde_json::Value = client .get(format!("{data_base}/api/v1/query")) .query(&[ diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index d35f6cae4..220cdef85 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -18,7 +18,7 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::DdSketchState; +use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; struct ChildGuard(Child); @@ -53,7 +53,12 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec) -> Vec }], start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec(), encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 84e7af7ea..1a291c496 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -10,7 +10,7 @@ //! //! The control plane drives the plan: a PromQL query and an accuracy target //! go through `BackendLocalPlanningInput::planning_request` → -//! `PhysicalPlanCompiler::compile`, and the resulting materializations are +//! `DeploymentPlanCompiler::compile`, and the resulting materializations are //! projected into a physical-plan artifact with QueryPlan/SummaryCatalog //! bindings, then staged and activated before ingest. //! @@ -157,7 +157,7 @@ use asap_otel_proto::tonic::metrics::v1::{ Metric, ResourceMetrics, ScopeMetrics, }; use asap_sketchlib::proto::sketchlib::{ - CountMinState, CountSketchState, CounterType, DdSketchState, + sketch_envelope, CountMinState, CountSketchState, CounterType, DdSketchState, SketchEnvelope, }; use prost::Message; @@ -172,7 +172,7 @@ use prost::Message; /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( "../../docs/examples/asapquery-compatibility-demo-snapshot.json" @@ -202,7 +202,7 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec, store_offset: i32) } } +fn encode_dd_full_state(state: DdSketchState) -> Vec { + SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec() +} + /// Build an OTLP `ExportMetricsServiceRequest` wrapping a single DDSketch /// data point. fn build_dd_sketch_export( @@ -745,7 +754,7 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); // ── 3. POST the sketch DP via OTLP HTTP ──────────────────────────── // @@ -785,7 +794,7 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -877,7 +886,7 @@ async fn controller_plan_to_query_full_roundtrip_kll() { .as_f64() .expect("planner sized a relative-accuracy quantile summary"); let dd_state = build_dd_sketch_state(alpha, vec![5u64, 10, 15, 20], -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -897,7 +906,7 @@ async fn controller_plan_to_query_full_roundtrip_kll() { "request_size_bytes", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -1725,7 +1734,7 @@ async fn shadow_mode_does_not_change_served_ddsketch_quantile() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -1745,7 +1754,7 @@ async fn shadow_mode_does_not_change_served_ddsketch_quantile() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -1835,7 +1844,7 @@ async fn live_serve_actually_answers_ddsketch_quantile() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -1855,7 +1864,7 @@ async fn live_serve_actually_answers_ddsketch_quantile() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -1997,7 +2006,7 @@ async fn live_serve_hll_global_count_merges_across_sids() { #[test] fn probe_queryplan() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; for q in [ "sum by (service) (quantile_over_time(0.99, http_latency_ms[1s]))", "quantile_over_time(0.99, http_latency_ms[1s])", @@ -2012,7 +2021,7 @@ fn probe_queryplan() { fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); let snap: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (req, env) = snap.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(req, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(req, env).unwrap(); eprintln!("PROBE {q}"); for (id, e) in plan.query_plan.entries.iter() { eprintln!( diff --git a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs deleted file mode 100644 index e4623e6bb..000000000 --- a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Phase 3 step 3 acceptance tests: backend ingest **consumes** -//! `asap-precompute-rs` for the shared envelope-parsing, -//! sketch-reconstruction, and merge logic. -//! -//! Each test produces an envelope via `asap-precompute-rs`'s wrappers -//! (the canonical Rust edge runtime) and routes the bytes through the -//! backend's runtime adapter (`precompute_operators::edge_runtime_adapter`). -//! Successful round-trips prove that the asap-precompute-rs `Sketch` -//! trait family is sitting in the backend's ingest path — i.e. the -//! shared logic actually runs in this repo, not just in agents. -//! -//! Sketch coverage: -//! - **DDSketch**: round-trip + structural assertions are live — DDSketch -//! is a deterministic histogram, so `snapshot → reconstruct → snapshot` -//! is byte-identical (`asap_sketchlib`#40). -//! - **KLL**: structural envelope compatibility is live. Byte identity is -//! not a supported contract because reconstruction replays retained items -//! through randomized, lossy compaction. -//! - **HLL + CountSketch + CountMinSketch**: the shared runtime adapter does -//! not support these families; their production decoders are tested at the -//! backend accumulator boundary instead. - -use asap_precompute_rs::sketches::{DDSketchWrapper, KLLWrapper}; -use asap_precompute_rs::Sketch; - -use data_plane::precompute_engine::operators::edge_runtime_adapter::{ - encode_ddsketch_envelope, reconstruct_via_runtime, snapshot_ddsketch_via_runtime, - unwrap_envelope_state, ReconstructedSketch, SketchType, -}; -use data_plane::storage_engines::types::AggregateCore; - -// --- DDSketch ----------------------------------------------------- - -/// Round-trip: an envelope produced by asap-precompute-rs's -/// `DDSketchWrapper` is reconstructed by the backend's runtime adapter -/// to a backend-shaped `DdSketch`, re-encoded through the same -/// envelope shape, and the resulting bytes match the original. -/// -/// This proves the **shared envelope wire format** flows through both -/// crates with byte parity — the dedup target. -#[test] -fn ddsketch_envelope_round_trip_through_backend_adapter() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=200 { - w.update(i as f64); - } - let original = w.snapshot().expect("DDSketchWrapper snapshot"); - assert!(!original.is_empty()); - - let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &original) - .expect("runtime adapter reconstruction"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!("expected DDSketch reconstruction"), - }; - // `count` is recovered from the bucket store now that the scalar was - // dropped (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57). - assert_eq!( - dd.total_count(), - 200, - "count preserved through runtime adapter" - ); - - let re_encoded = encode_ddsketch_envelope(&dd); - assert_eq!( - re_encoded, original, - "envelope round-trip via asap-precompute-rs runtime must be byte-identical" - ); -} - -/// Structural: an envelope produced by asap-precompute-rs's -/// `DDSketchWrapper` is unwrapped via the backend's -/// `unwrap_envelope_state` (which itself goes through asap-precompute-rs's -/// `ProtoSketchEnvelope`), and the typed inner state has the expected -/// count + alpha shape. -#[test] -fn ddsketch_envelope_structural_assertions() { - use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; - - let mut w = DDSketchWrapper::new(0.005); - w.update(1.0); - w.update(2.0); - w.update(3.0); - let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes) - .expect("unwrap") - .expect("state"); - match state { - SketchState::Ddsketch(s) => { - // `count` was dropped from `DdSketchState` - // (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); it is - // recovered by summing the bucket store counts. - assert_eq!(s.store_counts.iter().sum::(), 3, "structural count"); - assert!( - s.alpha > 0.0 && s.alpha < 1.0, - "alpha within (0,1): got {}", - s.alpha - ); - } - other => panic!("expected DDSketch state, got {other:?}"), - } -} - -/// End-to-end: DDSketch envelope → backend `AggregateCore` (via the -/// runtime adapter), then the backend's `query_statistic` API answers a -/// quantile query on the reconstructed accumulator. This proves the -/// **adapter integration is live** — backend ingest goes through -/// asap-precompute-rs and the resulting accumulator works on the -/// query-side surface. -#[test] -fn ddsketch_envelope_ends_up_in_backend_accumulator() { - use data_plane::precompute_engine::operators::DDSketchAccumulator; - - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=100 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot"); - - let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &bytes).expect("reconstruct"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let acc = DDSketchAccumulator { - inner: dd, - sample_p: 1.0, - }; - - let q = acc - .query_statistic( - asap_types::Statistic::Quantile, - &None, - &[("quantile".to_string(), "0.5".to_string())] - .into_iter() - .collect(), - ) - .expect("quantile query"); - assert!( - (q - 50.0).abs() / 50.0 < 0.05, - "median estimate close to 50: got {q}" - ); - let count = acc - .query_statistic(asap_types::Statistic::Count, &None, &Default::default()) - .expect("count query"); - assert_eq!(count as u64, 100); -} - -/// Snapshot a backend-side `DdSketch` *back through* -/// asap-precompute-rs's `Sketch::snapshot` and assert byte-equality -/// with the canonical envelope bytes. Closes the round-trip -/// (encode side) — proves backend can EMIT the same wire bytes as -/// asap-precompute-rs. -#[test] -fn ddsketch_backend_sketch_snapshots_to_canonical_envelope_bytes() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=50 { - w.update(i as f64); - } - let canonical = w.snapshot().expect("snapshot"); - let dd = match reconstruct_via_runtime(SketchType::DDSketch, &canonical).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let via_runtime = snapshot_ddsketch_via_runtime(&dd).expect("runtime snapshot"); - assert_eq!( - via_runtime, canonical, - "snapshot via runtime adapter is byte-identical to source envelope" - ); -} - -// --- KLL ---------------------------------------------------------- - -/// Structural: KLL envelope unwraps to the expected oneof variant via -/// the shared `unwrap_envelope_state` helper. -#[test] -fn kll_envelope_structural_assertions() { - use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; - - let mut w = KLLWrapper::new(200, Some(7)); - for i in 1..=10 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes) - .expect("unwrap") - .expect("state"); - match state { - SketchState::Kll(s) => { - assert_eq!(s.k, 200, "structural k"); - assert_eq!(s.items.len(), 10, "all 10 items retained"); - } - other => panic!("expected KLL state, got {other:?}"), - } -} diff --git a/data_plane/tests/edge_sketch_codec.rs b/data_plane/tests/edge_sketch_codec.rs new file mode 100644 index 000000000..4f94d9fee --- /dev/null +++ b/data_plane/tests/edge_sketch_codec.rs @@ -0,0 +1,75 @@ +//! Portable edge sketch envelopes reconstruct through the neutral codec and +//! remain readable by the backend's query accumulators. + +use asap_sketch_codec::{encode_ddsketch, reconstruct_ddsketch}; +use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; +use data_plane::storage_engines::types::AggregateCore; + +#[test] +fn ddsketch_full_envelope_round_trips_without_collector_runtime() { + let mut source = asap_sketchlib::DdSketch::new(0.01); + for value in 1..=200 { + source.update(value as f64); + } + let bytes = asap_sketch_codec::encode_ddsketch(&source); + assert!(matches!( + asap_sketch_codec::envelope_state(&bytes).unwrap(), + Some(SketchState::Ddsketch(_)) + )); + let (decoded, _) = reconstruct_ddsketch(&bytes).unwrap(); + assert_eq!(decoded.total_count(), 200); + assert_eq!(encode_ddsketch(&decoded), bytes); +} + +#[test] +fn ddsketch_bare_state_is_rejected_and_envelope_supports_query_readout() { + let mut source = asap_sketchlib::DdSketch::new(0.01); + for value in 1..=100 { + source.update(value as f64); + } + let envelope = asap_sketch_codec::encode_ddsketch(&source); + let Some(SketchState::Ddsketch(state)) = asap_sketch_codec::envelope_state(&envelope).unwrap() + else { + panic!("DDSketch state required") + }; + let bare = prost::Message::encode_to_vec(&state); + assert!(asap_sketch_codec::reconstruct_ddsketch(&bare).is_err()); + let (decoded, _) = asap_sketch_codec::reconstruct_ddsketch(&envelope).unwrap(); + let accumulator = data_plane::precompute_engine::operators::DDSketchAccumulator { + inner: decoded, + sample_p: 1.0, + }; + let median = accumulator + .query_statistic( + asap_types::Statistic::Quantile, + &None, + &[("quantile".to_string(), "0.5".to_string())] + .into_iter() + .collect(), + ) + .unwrap(); + assert!((median - 50.0).abs() / 50.0 < 0.05); +} + +#[test] +fn kll_envelope_keeps_level_layout_for_backend_readout() { + let mut source = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(200, 7); + for value in 1..=50 { + source.update(&(value as f64)); + } + let bytes = asap_sketch_codec::encode_kll(&source); + let state = asap_sketch_codec::kll_state(&bytes).unwrap(); + assert_eq!(state.k, 200); + assert_eq!(state.items.len(), 50); + let snapshot_bytes = bytes; + let accumulator = data_plane::precompute_engine::operators::DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&snapshot_bytes).unwrap(); + assert!(accumulator.get_quantile(0.5).is_finite()); +} + +#[test] +fn a_different_sketch_family_cannot_be_decoded_as_ddsketch() { + let mut kll = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(200, 7); + kll.update(&42.0); + let bytes = asap_sketch_codec::encode_kll(&kll); + assert!(asap_sketch_codec::reconstruct_ddsketch(&bytes).is_err()); +} diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index 6830697b5..b86830b94 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -19,7 +19,7 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::DdSketchState; +use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; use serde_json::Value; @@ -80,7 +80,12 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, values: &[f64]) -> Vec { }], start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec(), encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, diff --git a/data_plane/tests/support/current_series_process.rs b/data_plane/tests/support/current_series_process.rs index b0dabba0e..bc7f1140d 100644 --- a/data_plane/tests/support/current_series_process.rs +++ b/data_plane/tests/support/current_series_process.rs @@ -1,7 +1,7 @@ use super::*; use control_plane::physical::{ compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; @@ -68,7 +68,7 @@ async fn current_series_quantiles_topk_share_and_replace_values() { let quotes = candidates .into_iter() .filter_map(|candidate| { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), env.clone()) .ok()?; let warm = candidate.queries.iter().all(|query| { diff --git a/data_plane/tests/support/issue_701_702_process.rs b/data_plane/tests/support/issue_701_702_process.rs index cdd251578..d39c3f16a 100644 --- a/data_plane/tests/support/issue_701_702_process.rs +++ b/data_plane/tests/support/issue_701_702_process.rs @@ -2,7 +2,7 @@ use super::*; use control_plane::physical::{ compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; @@ -153,7 +153,7 @@ async fn run_warm_workload(queries: Vec<(String, u64, u64)>) { .into_iter() .filter_map(|candidate| { let plan = - match PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) { + match DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) { Ok(plan) => plan, Err(error) => { errors.push(error.to_string()); @@ -346,7 +346,7 @@ fn issue_701_702_uncertified_ratios_require_exact_fallback() { workload_cost::enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(!candidates.is_empty()); for candidate in candidates { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate, environment.clone()) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{query}"); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 403296140..cdc04f320 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -57,6 +57,7 @@ pub fn artifact_from_materializations( plan_id: 1, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: BTreeMap::new(), }; for config in &precompute.materializations { @@ -129,6 +130,10 @@ pub fn artifact_from_materializations( binding: MaterializationBinding { full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_definition( + config.policy_fingerprint().into(), + ), output_grouping, item_labels: config.aggregated_labels.labels.clone(), window_ms: config.slide_interval * 1000, diff --git a/docs/developer_docs/control-plane/architecture-naming-review.zh.md b/docs/developer_docs/control-plane/architecture-naming-review.zh.md index ef6ae4c64..5cedf4e26 100644 --- a/docs/developer_docs/control-plane/architecture-naming-review.zh.md +++ b/docs/developer_docs/control-plane/architecture-naming-review.zh.md @@ -37,7 +37,7 @@ | 所属边界 / 源码 | 原名称 → 最终名称 | |---|---| -| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`；`BackendLocalImplementation` → `BackendLocalPhysicalInputs`；`PlanningQuery` → `QueryCompilationInput`；`PlanningRequest` → `PhysicalCompilationRequest`；`PhysicalCompiler` → `PhysicalPlanCompiler`；`PhysicalPlan` → `CompiledPhysicalPlan` | +| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`；`BackendLocalImplementation` → `BackendLocalPhysicalInputs`；`PlanningQuery` → `QueryCompilationInput`；`PlanningRequest` → `PhysicalCompilationRequest`；`PhysicalCompiler` → `DeploymentPlanCompiler`；`PhysicalPlan` → `CompiledPhysicalPlan` | | 同上：查询语义 | `post_asap` → `selected_plan_root`；`source` → `legacy_query_source`；`window_secs` → `query_lookback_seconds`；`group_by` → `group_by_labels`；`accuracy` → `accuracy_target`；`lifecycle` → `summary_lifecycle_inputs`；`runtime_policy` → `materialization_runtime_policy` | | 同上：候选与证据 | `logical_selection` → `planner_selection_trace`；`materialization_policy` → `enabled_materialization_keys`；`evidence` → `topk_membership_evidence_by_query_id`；`hybrid_execution` → `allow_mixed_summary_and_exact_execution`；`synthesized_window_queries` 删除：编译器来源标记改为每个候选的 `derived` / `cohort_only`，不接受序列化输入 | | 同上：窗口与成本 | `WindowImplementationCandidate` → `WindowRealizationCandidate`；`ImplementationCostEvidence` → `WindowRealizationCostQuote`；`LifecycleCostEvidence` → `LifecycleUnitCosts`；`LifecyclePlanningInput` → `SummaryLifecyclePlanningInputs`；`window_implementations` → `window_realization_candidates` | @@ -63,7 +63,7 @@ |---|---|---| | 外部 ASAPPlanner | 解析与语义 IR，合法 summary/exact 候选，精度推理，逻辑选择；backend 提供具体成本和能力约束 | Planner 的语义选择与 backend 的物理候选比较是不同层次，不是两个重复 planner | | `control_plane::planner_selection` | 适配 Planner 的选择调用、精度及证据；输出语义 DAG 与诊断 trace | `selection` 必须说明是 logical 还是 physical；trace 不是决定执行行为的配置 | -| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择，以及绑定具体物理实现，生成多个一致的计划投影 | `PhysicalPlanCompiler` 比 `PhysicalCompiler` 清楚；整个模块当前职责仍比单纯 lowering 更宽 | +| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择，以及绑定具体物理实现，生成多个一致的计划投影 | `DeploymentPlanCompiler` 比 `PhysicalCompiler` 清楚；整个模块当前职责仍比单纯 lowering 更宽 | | `physical::workload_cost` | 枚举工作负载级候选、编译、生成报价清单、核验报价、选择最低成本可行候选 | manifest、quote、evaluation、selection report 不应相互替代 | | `physical::erp` | Error–Resource Profile 的部署适配、分布匹配、经验参数与资源估计 | `empirical_runtime_profile` 是错误展开；输入还包含策略与观测，不只一个 profile | | `control_plane::clickhouse` | SQL frontend、逻辑选择、物理绑定；支持已有 catalog 输入与自动生成 materialization 两条路径 | `ClickHouseSqlWorkload.sds` 实际是 `SummaryCatalog`；SQL 当前没有走同一套 `workload_cost::select` 整计划报价流程 | @@ -126,7 +126,7 @@ ClickHouse 的自动路径直接从 SQL 选择与绑定构建 `PhysicalPlanPubli | 当前名称 | 建议名称 / 约束 | |---|---| | `PlanningRequest` | `PhysicalCompilationRequest`；包含 workload 上下文，不是单 query | -| `PhysicalCompiler` | `PhysicalPlanCompiler` | +| `PhysicalCompiler` | `DeploymentPlanCompiler` | | `PhysicalPlan` | `CompiledPhysicalPlan`；候选和获选结果可继续复用此类型，无须新增 `SelectedPhysicalPlan` wrapper | | `logical_selection` | `planner_selection_trace`；说明只做诊断 | | `window_implementations` | `window_realization_candidates`；对象 `WindowImplementationCandidate` 也应相应命名 | diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 44f9e3ae8..7f029b8b5 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -5,8 +5,8 @@ ## Current implementation boundary -The compiler consumes ASAPPlanner types pinned to the revision exposed as -`physical::compiler::PLANNER_REVISION`, selects +The compiler consumes ASAPPlanner types from `main`, with the resolved revision +exposed as `physical::compiler::PLANNER_REVISION`, and selects from Planner's legal candidate space with backend-owned cost and evidence inputs, and emits one `CompiledPhysicalPlan`. The plan contains one SummaryCatalog plus CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan projections @@ -32,10 +32,10 @@ PhysicalCompilationRequest + DataWorkload + concrete implementation evidence | ^ | abstract candidates | complete physical costs v | -ASAPPlanner selection <---------- PhysicalPlanCompiler +ASAPPlanner selection <---------- DeploymentPlanCompiler | v -PhysicalPlanCompiler -------> CompiledPhysicalPlan +DeploymentPlanCompiler -------> CompiledPhysicalPlan | | | | v v v v Collector Precompute Backend Query @@ -112,7 +112,7 @@ identity. Physical identities never enter post-ASAP IR. ### Physical compiler ```rust -impl PhysicalPlanCompiler { +impl DeploymentPlanCompiler { pub fn compile_promql( &self, request: PhysicalCompilationRequest, @@ -338,7 +338,7 @@ identity. ### Add a deployment topology 1. Add a public `PhysicalDeploymentTarget` variant and its required target fields. -2. Teach `PhysicalPlanCompiler::compile_promql` how selected operators can be placed on it. +2. Teach `DeploymentPlanCompiler::compile_promql` how selected operators can be placed on it. 3. Reject plans requiring an unavailable stage/capability. 4. Verify the output contains one complete CollectorPlan for every producer and one SummaryCatalog and matching QueryPlan referencing all produced materializations. diff --git a/docs/developer_docs/control-plane/planning-terminology.md b/docs/developer_docs/control-plane/planning-terminology.md index 3a4dbcd97..b2076814b 100644 --- a/docs/developer_docs/control-plane/planning-terminology.md +++ b/docs/developer_docs/control-plane/planning-terminology.md @@ -23,7 +23,7 @@ flowchart TB LOGICAL["ASAPPlanner + selection adapter
Legal semantic DAG selection"] REQUEST["PhysicalCompilationRequest
QueryCompilationInput + enabled materialization keys"] WINDOWS["Generate window candidates
Cadence + evaluation phase + WindowCostModel"] - COMPILE["PhysicalPlanCompiler
Compile concrete candidate plans"] + COMPILE["DeploymentPlanCompiler
Compile concrete candidate plans"] MANIFEST["WorkloadCostManifest
Component implementations and pricing basis"] QUOTE["WorkloadQuote
Provider feasibility and component prices"] EVALUATE["CandidatePlanEvaluation
Select the lowest-cost feasible enumerated candidate"] diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 96491c76b..2600999f9 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -99,7 +99,7 @@ data_plane() { CURRENT_STAGE="data-plane/edge-runtime-wire" say "data-plane: edge runtime sketch envelope -> backend accumulator" - rust_test data_plane --test edge_runtime_consumes_precompute_rs + rust_test data_plane --test edge_sketch_codec CURRENT_STAGE="data-plane/production-process" say "data-plane: production binary -> modified OTLP -> SketchStore -> PromQL" diff --git a/tools/shared-workload/ACCURACY_E2E.md b/tools/shared-workload/ACCURACY_E2E.md index 73fc5f5aa..e7cd3619e 100644 --- a/tools/shared-workload/ACCURACY_E2E.md +++ b/tools/shared-workload/ACCURACY_E2E.md @@ -14,8 +14,8 @@ The repository's existing backend CI is unchanged. Runtime scope is **backend-local precompute**. No ASAPCollector service is started or called: the generator sends Remote Write directly to the backend. -The `asap-precompute-rs` build dependency and offline Google mapper source happen -to live in the ASAPCollector repository; they are not an extra running collector. +The optional offline Google mapper source lives in the ASAPCollector repository; +the backend build and runtime do not depend on that repository. ## Query matrix diff --git a/tools/test_shared_panes.py b/tools/test_shared_panes.py index 30b3cc73e..551b10d47 100644 --- a/tools/test_shared_panes.py +++ b/tools/test_shared_panes.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -"""Validate the unmerged Planner/backend pair without changing immutable IR pins.""" +"""Validate the unmerged Planner/backend pair against Planner main.""" import argparse import json from pathlib import Path -import re import shutil import subprocess import tempfile @@ -17,17 +16,6 @@ def main(): parser.add_argument("cargo_args", nargs=argparse.REMAINDER) args = parser.parse_args() backend = Path(__file__).resolve().parents[1] - manifest = (backend / "control_plane/Cargo.toml").read_text() - declaration = re.search(r"^planner-types(?:\.workspace)?\s*=\s*(.+)$", manifest, re.MULTILINE) - if declaration is None: - raise RuntimeError("planner-types dependency is missing") - if re.search(r"workspace\s*=\s*true", declaration.group(0)): - manifest = (backend / "Cargo.toml").read_text() - declaration = re.search(r"^planner-types\s*=\s*(.+)$", manifest, re.MULTILINE) - revision_match = re.search(r'rev\s*=\s*"([0-9a-f]+)"', declaration.group(1)) if declaration else None - if revision_match is None: - raise RuntimeError("planner-types must declare a pinned Git revision") - revision = revision_match.group(1) lock = backend / "Cargo.lock" original_lock = lock.read_bytes() with tempfile.TemporaryDirectory(prefix="asap-pane-reuse-") as temporary: @@ -39,7 +27,7 @@ def main(): old = 'asap-types = { path = "../types" }' if old not in source: raise RuntimeError("unexpected Planner dependency declaration") - cargo_toml.write_text(source.replace(old, 'asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "' + revision + '" }')) + cargo_toml.write_text(source.replace(old, 'asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", branch = "main" }')) config = root / "validation.toml" text = "" if args.sketchlib: From 070fad6e0f514056f0be25f1e43f7d16e19ccacf Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:05:38 +0000 Subject: [PATCH 02/25] fix: restrict foundation SDS recovery to the installed generation --- crates/asap_types/src/sds.rs | 30 +++---- .../storage_engines/sketch_db/index/mod.rs | 89 +++---------------- .../tests/support/durable_summary_process.rs | 72 ++++++++++++++- .../catalog-physical-plan-runtime.md | 21 +++++ 4 files changed, 118 insertions(+), 94 deletions(-) diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 896c9556c..88fadf379 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -321,8 +321,7 @@ pub struct SummaryInstance { pub time_range: HalfOpenTimeRange, pub group_values: BTreeMap, pub catalog_generation: CatalogGeneration, - /// Source generation selected by an explicit compatibility decision when - /// an unchanged definition reuses a committed payload. + /// Reserved wire field. Cross-version state adoption is not supported. #[serde(default, skip_serializing_if = "Option::is_none")] pub reused_from_generation: Option, pub placement: SummaryPlacement, @@ -359,23 +358,15 @@ impl SummaryInstance { "summary instance placement must be resolved".into(), )); } - let payload_generation = if let Some(source) = &self.reused_from_generation { - validate_catalog_generation(source)?; - if source.plan_id != self.catalog_generation.plan_id - || source.plan_version >= self.catalog_generation.plan_version - { - return Err(SdsError( - "stored summary has invalid reuse provenance".into(), - )); - } - source.plan_version - } else { - self.catalog_generation.plan_version - }; + if self.reused_from_generation.is_some() { + return Err(SdsError( + "cross-version state adoption is not supported".into(), + )); + } if self.state_reference.store.is_empty() || self.state_reference.key.is_empty() || self.state_reference.state_schema_version == 0 - || self.state_reference.generation != payload_generation + || self.state_reference.generation != self.catalog_generation.plan_version { return Err(SdsError( "summary instance has invalid state reference".into(), @@ -1387,13 +1378,16 @@ mod tests { } #[test] - fn reused_payload_requires_an_older_compatible_plan_generation() { + fn new_generation_rejects_cross_version_payload_adoption() { let mut instance = observed_instance(InstanceLifecycle::Persistent); let mut source = instance.catalog_generation.clone(); source.plan_version -= 1; instance.reused_from_generation = Some(source.clone()); instance.state_reference.generation = source.plan_version; - instance.validate().unwrap(); + assert!( + instance.validate().is_err(), + "cross-version adoption must be rejected" + ); instance.reused_from_generation.as_mut().unwrap().plan_id += 1; assert!(instance.validate().is_err()); instance.reused_from_generation = Some(instance.catalog_generation.clone()); diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 751b38774..d7b3e81cc 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -618,9 +618,6 @@ pub struct SketchStore { instances: RwLock>, /// Interns immutable SDS descriptors across all Series IDs and panes. descriptors: SummaryDescriptorRegistry, - /// Previous payload generations admitted by an explicit definition - /// compatibility check during plan installation. - compatible_source_generations: RwLock>, /// sid → item_label (the data-point attribute NAME, e.g. "service" /// or "endpoint") for CountMin/CountSketch sids registered in /// per-item mode. Its presence is what makes a CMS sid answerable by @@ -876,26 +873,6 @@ impl SketchStore { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }; - let previous = self.descriptors.authoritative_snapshot(); - let previous_sources = self.compatible_source_generations.read().unwrap().clone(); - let mut compatible_sources = BTreeMap::new(); - if let Some((old_catalog, old_generation)) = &previous { - if old_catalog.plan_id == catalog.plan_id - && old_catalog.plan_version < catalog.plan_version - { - for (id, definition) in &catalog.definitions { - if old_catalog.definitions.get(id) == Some(definition) { - compatible_sources.insert( - *id, - previous_sources - .get(id) - .cloned() - .unwrap_or_else(|| (**old_generation).clone()), - ); - } - } - } - } let closed = self .persistence_metadata .read() @@ -905,9 +882,6 @@ impl SketchStore { .transpose() .map_err(|error| error.to_string())? .flatten(); - // Publish the compatibility decision first so a reader that observes - // the successor catalog can also resolve its admitted source payload. - *self.compatible_source_generations.write().unwrap() = compatible_sources; self.descriptors .install_catalog(Arc::clone(&catalog)) .map_err(|error| error.to_string())?; @@ -1154,17 +1128,12 @@ impl SketchStore { .map(|set| set.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidates .into_iter() .filter(|sid| { instances.get(sid).is_some_and(|binding| { - Self::instance_visible_for_read( - binding, - generation.as_deref(), - &compatible_sources, - ) + Self::instance_visible_in_generation(binding, generation.as_deref()) }) }) .collect() @@ -1183,25 +1152,6 @@ impl SketchStore { } } - fn instance_visible_for_read( - binding: &SdsBinding, - generation: Option<&CatalogGeneration>, - compatible_sources: &BTreeMap, - ) -> bool { - if Self::instance_visible_in_generation(binding, generation) { - return true; - } - let Some(generation) = generation else { - return false; - }; - let definition = SummaryDefinitionId::from(binding.metadata.policy_fp); - !matches!( - binding.data_descriptor.source, - asap_types::sds::DataSourceIdentity::Derived { .. } - ) && compatible_sources.get(&definition) == binding.catalog_generation.as_deref() - && binding.catalog_generation.as_deref() != Some(generation) - } - #[cfg(test)] /// Live policy count — number of distinct fingerprints with at /// least one sid. Useful for telemetry / `/runtime` introspection @@ -1264,17 +1214,12 @@ impl SketchStore { snapshot_sha256: reference.snapshot_sha256, }; let instances = self.instances.read().unwrap(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let durable = self.persistence_read.read().unwrap().clone(); let mut reported = BTreeMap::new(); for (series_id, binding) in instances.iter() { - if !Self::instance_visible_for_read(binding, Some(&generation), &compatible_sources) { + if !Self::instance_visible_in_generation(binding, Some(&generation)) { continue; } - let reused_from_generation = (binding.catalog_generation.as_deref() - != Some(&generation)) - .then(|| binding.catalog_generation.as_deref().cloned()) - .flatten(); let summary_definition_id = SummaryDefinitionId::from(binding.metadata.policy_fp); if binding.metadata.policy_fp.is_unset() || !catalog.definitions.contains_key(&summary_definition_id) @@ -1329,7 +1274,7 @@ impl SketchStore { time_range: HalfOpenTimeRange { start_ms, end_ms }, group_values, catalog_generation: generation.clone(), - reused_from_generation: reused_from_generation.clone(), + reused_from_generation: None, placement: SummaryPlacement { producer_id: producer_id.clone(), storage_node_id: storage_node_id.into(), @@ -1341,9 +1286,7 @@ impl SketchStore { window.0, window.1 ), state_schema_version: binding.summary_descriptor.state_schema_version, - generation: reused_from_generation - .as_ref() - .map_or(generation.plan_version, |source| source.plan_version), + generation: generation.plan_version, sequence: window.1, checksum: None, }, @@ -2469,7 +2412,6 @@ impl SketchStore { .map(|sids| sids.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidate_sids .iter() @@ -2478,11 +2420,7 @@ impl SketchStore { .get(sid) .map(|m| { required_keys.is_subset(&m.group_by_keys) - && Self::instance_visible_for_read( - m, - generation.as_deref(), - &compatible_sources, - ) + && Self::instance_visible_in_generation(m, generation.as_deref()) }) .unwrap_or(false) }) @@ -5094,7 +5032,7 @@ mod tests { } #[test] - fn unchanged_definition_explicitly_reuses_a_committed_previous_generation_payload() { + fn new_generation_requires_fresh_state_for_unchanged_definition() { let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" @@ -5113,7 +5051,10 @@ mod tests { let mut next = plan.summary_catalog; next.plan_version += 1; store.install_summary_catalog(Arc::new(next)).unwrap(); - assert_eq!(store.series_ids_for_policy(fingerprint), vec![509]); + assert!( + store.series_ids_for_policy(fingerprint).is_empty(), + "new version must start cold" + ); let output = asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) .stored_output_id; let inventory = store @@ -5128,12 +5069,10 @@ mod tests { 100, ) .unwrap(); - let reused = inventory.instances.values().next().unwrap(); - assert_eq!(reused.stored_output_id, output); - assert_eq!( - reused.reused_from_generation.as_ref().unwrap().plan_version + 1, - reused.catalog_generation.plan_version - ); + assert!(inventory.instances.is_empty()); + store.register(meta_with_policy(510, fingerprint)); + store.append_sample(510, BTreeMap::new(), (0, 10_000), sample(2)); + assert_eq!(store.series_ids_for_policy(fingerprint), vec![510]); let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( plan.precompute_plan.envelope.plan_id, plan.precompute_plan.envelope.plan_version + 2, diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index b79aad913..defaf434b 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -18,7 +18,9 @@ async fn persisted_summary_restarts_without_live_reregistration() { serde_json::json!(5000); let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); + let plan = quote_snapshot_for_test(snapshot.clone()) + .compile_promql() + .unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -154,4 +156,72 @@ async fn persisted_summary_restarts_without_live_reregistration() { assert!(is_warm(&after), "{after}"); assert_eq!(after["data"]["result"], before["data"]["result"]); assert_eq!(after["data"]["result"][0]["value"][1], "15"); + drop(second); + + // Equal semantics do not grant a new plan version access to old disk state. + let mut next_snapshot = snapshot; + next_snapshot.environment.plan_version += 1; + let next = quote_snapshot_for_test(next_snapshot) + .compile_promql() + .unwrap(); + assert_eq!( + install.summary_catalog.definitions, + next.summary_catalog.definitions + ); + let next_install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: next.summary_catalog, + collector_plans: next.collector_plans, + precompute_plan: next.precompute_plan, + transmission_plan: next.transmission_plan, + query_plan: next.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + }; + std::fs::write(&artifact, serde_json::to_vec(&next_install).unwrap()).unwrap(); + let port = unused_port(); + let base = format!("http://127.0.0.1:{port}"); + let mut third = spawn(port); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut third.0).await; + let cold: Value = client + .get(format!("{base}/api/v1/query")) + .query(&[("query", query.as_str()), ("time", "5")]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(!is_warm(&cold), "new version adopted old state: {cold}"); + assert_eq!( + remote_write( + &client, + &base, + &WriteRequest { + timeseries: vec![series( + "asap_demo_gauge", + &[ + (1000, 2.0), + (2000, 4.0), + (3000, 6.0), + (4000, 8.0), + (5000, 10.0) + ] + )], + } + ) + .await, + 204 + ); + drain_precompute(&client, &base).await; + let rebuilt: Value = client + .get(format!("{base}/api/v1/query")) + .query(&[("query", query.as_str()), ("time", "5")]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(is_warm(&rebuilt), "{rebuilt}"); + assert_eq!(rebuilt["data"]["result"][0]["value"][1], "30"); } diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 5a8b044ea..a3eecb195 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -15,3 +15,24 @@ track the required code and recovery changes. The HTTP lifecycle is exposed through `/api/v1/physical-plan`, `/api/v1/physical-plan/activate`, `/api/v1/physical-plan/discard`, and `/api/v1/physical-plan/status`. + +## Foundation scope (#749) + +This branch validates that maintenance and query plans belong to one installed +catalog generation, retain the selected DAG provenance, and agree on writer/read +bindings and physical windows. It removes the Collector runtime dependency. + +The identity representation is transitional: catalog schema 3 still uses policy +fingerprints and couples the stored-output ID to that identifier. It cannot yet +represent independent hot/rebuild outputs with one semantic definition or prove +that equal metric names refer to the same logical dataset. Do not use this stage +as the completed SDS implementation. #774 supplies the Planner semantic export, +independent definition/output identities and explicit logical dataset binding; +#763/#765 supply the remaining storage and execution integration. These are +required before claiming the target contract or supporting multi-dataset reuse. + +Recovery at this stage is limited to the same installed catalog generation. +Installing a new plan version never adopts previous-version state, even for an +unchanged definition. New input must populate that version before it can serve +accelerated results; the query follows its installed fallback/unavailability +policy while cold. Cross-version adoption metadata is rejected. From 3c840ff9807be988278e0bc1eafe113d8c8715d5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:11:12 +0000 Subject: [PATCH 03/25] fix: allocate fresh physical series when a plan version changes --- .../storage_engines/sketch_db/index/mod.rs | 27 ++++++++++++++++--- .../catalog-physical-plan-runtime.md | 4 ++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index d7b3e81cc..c45f274b9 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -2806,15 +2806,17 @@ impl SketchStore { if matches!( binding.data_descriptor.source, asap_types::sds::DataSourceIdentity::Derived { .. } - ) { + ) || binding.catalog_generation.as_deref() + != self.active_catalog_generation().as_deref() + { let (catalog, generation) = self .descriptors .authoritative_snapshot() - .ok_or("derived reactivation requires an authoritative catalog")?; + .ok_or("series reactivation requires an authoritative catalog")?; if binding.metadata.policy_fp != definition.fingerprint() || !catalog.definitions.contains_key(&definition) { - return Err("derived reactivation differs from its installed definition".into()); + return Err("series reactivation differs from its installed definition".into()); } if binding.catalog_generation.as_deref() != Some(generation.as_ref()) { return Ok(Some(generation)); @@ -3368,6 +3370,18 @@ impl SketchStore { if generation != installed_generation || !catalog.definitions.contains_key(definition) { + // A new version must allocate a fresh physical series. Otherwise + // the persisted resolver can route fresh input back to this + // excluded series and its already-completed windows. + if generation.plan_id == installed_generation.plan_id + && generation.plan_version < installed_generation.plan_version + && catalog.definitions.contains_key(definition) + { + self.removed_sids + .write() + .unwrap() + .insert(rec.sid, (Some(Arc::clone(generation)), Some(*definition))); + } tracing::warn!( sid = rec.sid, "persisted summary catalog provenance differs; leaving state unbound" @@ -5055,6 +5069,13 @@ mod tests { store.series_ids_for_policy(fingerprint).is_empty(), "new version must start cold" ); + assert!( + store + .authorize_series_reactivation(509, fingerprint.into()) + .unwrap() + .is_some(), + "live version change must allocate a fresh physical series" + ); let output = asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) .stored_output_id; let inventory = store diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index a3eecb195..0fd76e35a 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -35,4 +35,6 @@ Recovery at this stage is limited to the same installed catalog generation. Installing a new plan version never adopts previous-version state, even for an unchanged definition. New input must populate that version before it can serve accelerated results; the query follows its installed fallback/unavailability -policy while cold. Cross-version adoption metadata is rejected. +policy while cold. Fresh writes allocate a new physical series instead of using +the previous generation's completed series, both after restart and during live +activation. Cross-version adoption metadata is rejected. From 70a8d8885ab44e7f0a4934f36c64f2c92d3ad37e Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:14:24 +0000 Subject: [PATCH 04/25] test: record foundation rebase and recovery regression evidence --- .../pr749-sds-foundation-2026-09-28/README.md | 42 ++++++++++++++++++ .../SHA256SUMS | 8 ++++ .../pr749-before-fix.log.gz | Bin 0 -> 722 bytes .../pr749-final-clippy.log.gz | Bin 0 -> 173 bytes .../pr749-final-data-plane.log.gz | Bin 0 -> 20271 bytes .../pr749-final-process.log.gz | Bin 0 -> 353 bytes .../pr749-library-tests.log.gz | Bin 0 -> 30272 bytes .../pr749-live-before-fix.log.gz | Bin 0 -> 589 bytes .../pr749-recovery-e2e.log.gz | Bin 0 -> 626 bytes .../pr749-store-before-fix.log.gz | Bin 0 -> 488 bytes 10 files changed, 50 insertions(+) create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/README.md create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/SHA256SUMS create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-before-fix.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-clippy.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-library-tests.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz create mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-store-before-fix.log.gz diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md new file mode 100644 index 000000000..f3d8413f4 --- /dev/null +++ b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md @@ -0,0 +1,42 @@ +# PR #749 foundation rebase and SDS review + +The implementation is based on main `b7c0f08a`, including the merged #737 design. +Commit `79eb631a` preserves the prior implementation tree while removing the +already-squashed documentation history. Fixes are in `070fad6e` and `3c840ff9`. + +## Corrections + +- An unchanged definition previously authorized reading an older generation. + The compatibility map and read path are removed; adoption metadata is rejected. +- After restart, the persisted resolver could send fresh input to an excluded, + completed old series. The old physical address is reserved and fresh writes + allocate a new series. +- A live version change now authorizes the same fresh allocation without first + requiring the old series to be removed. + +Metadata, store-visibility and live-reactivation regressions fail before their +fixes. The process test also exposed the completed-series rejection before the +fresh-allocation fix. Those failure logs are retained. + +## Validation + +On the corrected #749 code: + +- 112 type-library tests, 427 control-plane tests and 1,161 data-plane tests pass. +- The production-process test recovers the same version without re-ingestion, + verifies a new version is cold, then ingests fresh data and verifies its result. +- Strict all-target Clippy passes for those three packages. + +The data-plane suite and process test were rerun after the final allocation fix. +No production-cost or independent human approval claim is made. + +## Scope + +This is the plan/schema foundation described in #737's staged migration, not the +completed SDS implementation. Its policy-fingerprint-based schema remains +transitional. Canonical semantic definitions, independent deployed-output +identity and explicit logical dataset identity land in #774 using Planner #462; +subsequent PRs complete shared execution and storage integration. The developer +installation guide and PR description make that boundary explicit. + +Compressed logs are checked by `SHA256SUMS`. diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/SHA256SUMS b/docs/evaluation/pr749-sds-foundation-2026-09-28/SHA256SUMS new file mode 100644 index 000000000..b352d6229 --- /dev/null +++ b/docs/evaluation/pr749-sds-foundation-2026-09-28/SHA256SUMS @@ -0,0 +1,8 @@ +c6f94a832ce5fe9b7e37760c7261b327562b478297841c88b82f7214752bc707 pr749-before-fix.log.gz +cf40ddb67304c10117fd153087e39370accc5f352b420a644104ecfc94b84000 pr749-final-clippy.log.gz +2d8020ab240b66cfb189e00691c678619dbfd0efa3c3f21aae92e5efa8c76dc5 pr749-final-data-plane.log.gz +271c6756f25644a56cc2fb0fbee5c3b7a1dc98014f580f53a7ec6210227f0de3 pr749-final-process.log.gz +b068361a9e0e495b03251b457081a8c441bcbc7a57ede7bef1b3b1750e349b40 pr749-library-tests.log.gz +d1765747adb31af751181c16ada19eef7b35739c39d55f7ccc111e3f0916bc3b pr749-live-before-fix.log.gz +f91df10f88497203a438a9a5e1bc7f0655c20451a1055c44b09f2c29d6401085 pr749-recovery-e2e.log.gz +d0c1ede2fd39feaf9db03e1c6df2047ce8344043c6460e7d465f7a7c67217a30 pr749-store-before-fix.log.gz diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-before-fix.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-before-fix.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..72cd0016cf048fb0adfaa90d587421c4cc3c5a5f GIT binary patch literal 722 zcmV;@0xkU?iwFP!00002|HV~ZPunmQefO`p(jK7F@>QU1f)MH$NYnPffIUr9>e#of zM`CBTQ`G(XU55?|<6#m=&4cAQ$vNlVbL;@XnQ2;8E4={jcpLe3iw;(4oW&{ZFTHPF zkt7S{mt7SL(!{-57PU;`J7aIXMRbY9 ziaU(Phk4X+rIRO6&uBuSrSV8lt?}lSM3#G6eunu1W8{9-(Y({bE2ASpM73G$s?)t( zcZ*&uYgGjm2mhhs=~t(fG#< zanU_cBC+ZgG`8E}i{|N3HK~Tf(J-6v>M$EHW;gg51auY#Ahe?B$%;bbUAaKAmWPw5 zv>0r6rLe}ia)s7~)t0Yo!)3{3D9LDu;}|ZcpT3-bWNgk=-C1-6V-a0%fbP6Bori$N zm|t4t682sVy9fJ|{BU@b4xr^)k8P6( z!o2{LOw*xX0QDU1XfMI+>f7~A`C)qY`Q~bRc7Br00QE}IHf&IPSaGX(MOmTWj#Q39 zwFV9$gWht1Y`eE1E3a+=yZ4^}yV@5U2V+R7A@-eO#~4~`Y|#g9gZDm^XfZSu8yY({gSn!8_`J49@dZ1 zBL?oV6#=vTuFSL`!+)qWfp_(Q-MDU6?H`Y?^BEk-P#91`BXMrPd3z^?aG8vCku*I~ bULgwKW88!k$UsrBvIxlwz)jjb1pxp6ILk}F literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..fc9448e37e18360c571637f30f3d1c3b7c6181a3 GIT binary patch literal 20271 zcmV($K;yq3iwFP!00002|JA)olj}&5CV07>t_}i@~F&4wdnoq^X&jn(oWDZ{Pm6=G>N5Ss%X@d7o!(mDldu z>-V32{Pg|XfBVPz1AqO;|NOuI=l}744zBz7`!3(VxVreqPj5~4(s$1Je{^n_d-wN$ z|MH)IKj&rrk8z#<_}e%7zyH0gOMi03xBt~U-~X>~ZPy&i%6z zTt$TDYWgEC=dJ8d*~wMyVE_ue&yQ{yoV;!6mJJLoSJ4&ay$i5XEc@0qwCQ}mAI?LS z_f5x0TeI?+_I4=gMw`BB6+62R`bu`96O6Yp1%`KBNk4pXAM!({P=zK>tjc$;^4Uk} zszT^$*X4WX*RFnOI(lze^c8n5317s<&VN+d9|J8>Wcb!vNaG)Td2We#^Qt8>9?tZe z{Wcd-6#m80FB0Cy_=nfL8k{@>hz(!m{sI$oeJGE3*=^qCXP*sVwb6riAAXZV zrZa*yae&) zx328nZ@($4veDF&%i zHTxXMSdKp6DXWN-GiBOEo1M!Vpn%fl3zOc+%2g0tBF%hX5_2xPhRA{*GDskURkEJY zKw-2bAtWavBaYQGYKVQ2X^}Nm5q?f;dD~7@#_k{PKz}+h$pKdc_3h+{1c}gTbwAK0 zVY`32u2F#mYFv*?Xk)4rG)+&+e~y#Y!FYL&$j_kQukvh2J;{$`GM>R1e0i)%7%7Jm zhTV%OSA-TK16DvL)=E4A>Y7T{oDB@#5q&^3A{#s54S1FcHSLFPe?3z7XTK7uQ>gPq z&_H>mXPYo{UDtGglQqO2*RQ{Cx^8HDddkOq|G~nlD@gTq^kdj>O;;XEy2p$y`o%~S zM5#T37e-6Bpn2IMa79-|msl6g8zGdG1d`-4k7uyxh-JO+*)C+`Pp)3SA3mm?`jrsM zx*htoONuG2TMo}^?eYhFp=(J@(<)8Z`w2?)6!HnIQFO$nX|woSqp+4gE+-&*2QA63 zToesf@P_u-(c|#^67j2j-sZcqBAss;l19IT@d~0-&c8grcZVx@?RVEM(HNt0b@) zhinNc7S*~Y+2V15ywJul=*re9wxUgtnfF5ERf5*YLLrkUXRCxR%{;UN>sL?|r?^+9 z@H!QdWfP2&Qr;M8@Wh@QcywsLJ$<+l7Hir8TMD$Xb|V;s;Es-T!1H~0TdSdzHK~5Y zvKiXOH1`-;)2%7kw7BkEWL#`(i?ImP&J~MY7hV1)zB~EE!@v%fX^{U%;EBZd)t6ZUVrIQF!;?W>lyW$K*o;UUNjmgb>;5TE^IoI z0R_uv6%41`XBvyknm5SYaRp(2$q(1C`{onN4PAyAr$JN;8N(W1*scne@v zlcDCY7PBH+rOl75dm(D;I+IYJ9-9w?f6FPW<|xjmu6cEJeqkflWZf2wn~(T&m48IK z=QNOrWO`D7@Ct@1C}5c%y4Mnjhts61CE%EI)YrZDaMeHbW!>L_IkEQpp@U3UE1M}6 zN;))(vd`q;uq4@!EAG4ggC5Eyl>`ljF z%3FX(@qLSK8BB$JRk|lS^z{21$=SUFaXWhc`x6M|^s(zyph34b*5>zlog8*kS07oC zAK7F5t|Y@7O621YM$gC-L^iA6-%hD&Wlx(JLm}(-Kxi2*4KOzCIL5(m2d|NSniq(; zmsPfFU|}#@pf3$%GCm8^J=?;uYIX#OtdkeEy_E|$xQw?`N&HvT?~Z>Do;A99(%iKY z(+M#^M-RJ#jy*Z;nb-27)z4hX^Onrj!g$%9li7wBa+Aw&NTz#`| zD&@d_C3ImrcVOR>*b`xmuVJ(YSf{xIhV?_olrJJf)D*}OVgU>2PIRaWfwczGp2Kal zqn)$3;$>*upYnEPmzY$@s85!oPy@-y!EGM$%7FNaIM(w zKEg?y1uXmWi3v4r|7Up<6!TqP{mOz=2 zVGosm0XVkQL9PeH*YOw)XTLYQ2x-40dg?8wuJKcWHRL(xO8($iQPqeY-#WJY6-#s_ z>tFrd`xpC)Wq!K*H_zxb3$v@(<@bX)E%bqNJ8;q{V2gQcsh(nSegrU_KMCbMFC;z+ zw+V;i{rY90WgEhutUu?P6$AhJ{!hu9Ag(KWf+ll>dfT}o6-zh9XZP{DztR|IVg|TY z_aTh4SZRC9isw$<8FV?FCCxlTc|H%!Xpo&_!EZqnlFK`@W6^oz5jTQu?8M% z@o$p-x_{w#UIbRM)@VnjuD+F|d2mPou@4W-`gJ5P&O*^*wA3eWNniC%Bw96#-4KAU zJcH@FHfwfrZBeFmmD3X?9l3U$o}0ak#uwByj{LrNoS6$y8PX?+6_=bo4^;$CilX!{ zjLdtuT=uN|TOQ8NN#w-zJ<~+5+O$=;JUf=8fT1FR)*r7dobMw2A7oc{Qp6+vaQXRd zW!Pj$BExiH@hi{=IeXWUO&m6|YX?C{pYgNaJAfsdrKyj42`}#4$WYR zx_R-6D8edz0=d+p-&&gILV8$*$`fcA`m^?o%v=Y?b)YNMfL8lLXntg@&+A zj~6q+cps5yz5tGdiP>o801B5y!KngErCevnT`_!?cp~dgjn~8-Qj?m)XUXl)ZMxTg z)9a4{jdTLsI#R!KVb8fI8o6<2+&K**aJ|OQI?&EJVq>A~_YAF$wGZYAbe#P4k`_32 zxYwjlq#v};rO#)u^Wuzx?;`V6^Ja+)bnzCX%y#ZX@QFMm7GN<<&Bi)ZB-x(9S0*F1 z!rc}37S|eT2uNk=C6h$%U2^OusNkQS?|{5{MQ3E^6mDN}*a2TAAm{eN(VCxk%Cq)O zHJod|!4~uZDyBUf7I$2UgzWSWc_4Gi zqsl`jpk%zsO(7Mw>z7}J*j-w`lbOY)t|Pq{Hg)eS94mAhaUPH!T8Kg;@tXy!$Jrf^ z;-y!~0jsSVHan{F;PxMTWSnY<9KU#WvBp7y|KyCcP7Rlgs^=wO1+ng$^R=y{_A8-V;NY)-shNxW+&|Zg(!l>hIuZ}eCa|PY2!E+6 zcglyfxz5)R(XFNBZ$AWO2i?kn{LJ0T-4?|=Q}tHF|9`1L>~fgs@i zuH;|rsf`jJ92-9zfdA?va0ex)(Kr|foB*+cJ(Z;ff0rJg$heEZf3U+j^s zC#)&@PYwDafFga!B$BrArn+;BjEe#VTOZKIO3&7g=8EjOrWQPvrMc_2+s?ft29(e; z$_NlbFRxzd6`#)D%5--&7_zylwARohM^StYTyb7V3vm&u>EUkVK>rJz=aa1Zvp=>l zF23fJ_dsrZz|A~@f2B9@zlJC9FYp5X|IY&`4+_;_Y!B7VorcI8pFEUmb6-H@zhEn0 zxq}2=!bu)Nu3(C@0!3~_fAL=g*l>Q^t04)@CtBM=HFdgwQriQglR>y}`bK%qmpEkq z3z+x+B~1Id7Rm+|^y)1vwwzZ(US=_;0h>^vSi6&j7V>K7+pwIkl&zba=6|7CKEwJo zkFA(o+^(Gi5j^24`q_qi*xW6E0yNnU2B4r+Kxx7(kf{igWxkL#GNqNXu z9(GAKQ!Bw{33kLPT86cYil<}>YdDW)+B)+7mU=> z9I%&e^Zx6%63N|dS=j`HkeXdh=$f$z{K7PWqFCnG);b|=D*gokS9#K>08Q>jUjuZB zi+K^eK$>KSjH5#apAMYAYZR~SqHl{8$i5Z*=?xe@m-{&GM4x-CpeU)>h| zWfmY$Vf{Mj8xdQgRMrWQLM^&!;a9*#4*|r8 z(6&laX0xHrU-Po!kN{ghX?Kt^Q*#2ikfO;?B`^aGqef+i^D;K)K&~wK+Ws{&LSIbsGUBY3f0NnE&g}e%Fg;wcClOT$NGq45eKD9mEq)ORj zfXP**rAOV0fVS4lR_*~U7e~IoFuJ{R59^h~)f-UeT+!N?Au&Jhwtffni|*?Ha^L|F zcvj<*T#6|hNTi@n0hLmhnF23Wy-BE&x(A7De>Wmg55y-8voyxxI~YgYRC6_(&*6{j zTPIh+Gk3Sp;R)M-zU;>v=ff-rDJ_IFH^8$H%v{I+LNGpV_Voqe$ebxLIYhLYvCGzN zleA6X&ki;`hqKU?3xjS-h?Xrs46;hFwv4hzk4h3N$FZ6$!P3%a37Tl1zwjV}5lVvP zNm$4Tq(-NjWI4>@@=WwJ_oYmL5#=TXI4K|meR7llfhn)Tapa+Tty@q;Mr803NW+>a23M%UjcqT%Sm?dg<9H- z4J=h-48nAsW5o?V(LOdz6s$>39HF=R9PDsn*4#4Ta=!77}6uyP>0FxwhD+HcE!N;MIH=4fH zm+hxhfxB6gAa}2&OlO|Q-X4LlDc3@1*J(Zy&+!T1gke}KMc=ITQz7Z@;&VCvti>Pm z_Z7prFiObF*i#)|LptkbN0==uwh$3Ea_0WJ-5IF~qf^eJc-hV!n$G6!;Pu@lrY*)R z^SmW#Ifqrt2|3+EkN)PqtY0GGQ4=d$vVnO>na*S&tqzS#U6@?*Y*n3fv>Pz*=6ZNw zF|_Olwt+0G8>zaw;Nf37uhg0^D%GsnD{+pRxDj383VKl5D&x7iGD@Rw_&-Cgc-|5@Lr}q5fQ_Zs&>Pi$59XN5>ZgcYJrcd^d zJeg^&c|Tdi%XkwWSdnV_uubw}?}W|uP|-bct1>43W|pZHDS32mUA-U%w_)Jux}|h( z9{<$_SBWlt0up3cD2pxkzITr2KDYUP!eBT*?27!D`TdFbQ#@zM^jb+7=Ak$xPj|8v zm;!aEnxvc-VLZtM2bdH+(*vpL2i@hIJ^joyGGY&qS(r(NJ+S!KkHAdHuE6A!#V6JI zKQ>2Y|8Zyfk4^*Anc6&er?;*h%K?*=sY)LFb1@Yy+hgK&zN2@ui#KLN^h%upSS-YG zd;0L)2aa=n-pYb55~ClZQ8LZJa>dD}aZBM;d0q3vFOTrLNtqa-l1vy(3TGpbZ*ZdC z_7Ib5!?FaxZ8a5pFxc?RT)~sQnf>6kjB~8U(U*SV5rH18d2@qR6LiN~2ZU95+p=K+ ztb?+a!W8kT^$DaB6IvlvFx3<+W)S8Oy(BMbc|5Wb(3hSzut+=)pw^a`k(8^K4z4f# zl3;PoighE42#bsqygol-fhHw@VvNT zdx{HY){6Q%lCIegXA*xkhDME3jMqS54wvyTPh#hpHl6l<9d?Z|OzZ9O*-GKS@uNt& zAXC$t1@$vd&h!ZuQhATTx+K@R_H;cVJzU$+u|P8=yVIYDK0P#ZG84r^yDz)_055^o zt<^jg6QAFk3U==pJc9Igf+^;aJDtg9(@S${-E|a40;cJ~br8nv%>_j|M}^M|Qgooq zK`$V&0{@rHwJy4*Rm*PX+UD4h8k*8uTJIY~hpN3*1GR*wDq{+;a>4xImY(Ac_@Ng= z7Ew5o%I2KM8nS{ePOG^vs;YT|&PzgtX@G~T`&g}+^uT@`h~A4zWZ{_8Rf5)^;JXt$`ZknGgDc_Yd?PI=o;*AL-^6Bs)Cp=@F% zJJTlkM6Q8E#)vjkjs~%+dKas9)3oJFAUH^ALrU;^qX@=`GK|kBOFYsgbI;j@yNRoIC46p*A1tp7!NnDXbSru6rjUFFj5iYo_W6yaHT1hm?F8pjM zb&X*X{1zCrrf^F!$kZFXa~)7V+4p#tqQba!zL{upwLG^)_>@cD+05e=H$#^T?{5_M z3gyIwP|%7E2E@tO+R~#&Xo9EY&Vnol(R7EEo+T>~*1cPMB#@ab0lx-1S}_Cd2hI`E zen5-v0p078-Rg)fRU~-)aX+Dp>od%$;M>u2UbeN1#fVuizmSWS{0nl$gi{yH9&lBE z49zkr6FkL=zxg`ReREMXIB2L#mbsn1A1njX=+ zdtY4xQ~rY4h%TV-c=IU9c{p>Wbd*D4P6g^C4v<-w&2T(KBYU~ zpE#ONui%W=KsxtkCk?UdnWbc(#i=!*fM7ubPrf;#A7`knMl3fokABff%7lZW@5hKS z-dIVxOqeLX@h)VD{iu+@&f$n%6v;fZt{z(RHC>tkqn6=7x6yB^`j zyvW-)N+)vaOpENAzNbgay-@hP(T_D~mlpHrBA;_|WdY6Cbx6{YG@8og2X+>SnaAEC zS+K~Do1&wZ{7A{3vh;+3r6j7n|FEW@^7}2fa^^_5S%aj#*Pin`O5q5(C;ah*B59sO z)&esJ;XVDT+mcu!5&E6%CKE|2eH{E=a3>(LS=Re%0Qz8WjM0GQ8|j=>%frED=n0B< z*xa#db|4)lWlk@nO<_-migt*G10fmafoR@fQ*h27QQD}uiFy;~H9ZFHFRpGSp3VY> z!UIFraN6gDRS%W4$~u&;D&W&ZD0)N@{rB%4Vggyn{&j0a)kMayc1ipU>L>=}uuBPh zaxeDoK~~cSy8;_jFF{C}#WY8DjpxPhq}SKt0}5P190@7z@ZJ{pWJd`e#yzq*b#{Hu zHq1z2a2$BaT@hS5&bYyu64O1}E#)1DCfIXvMf_wN3{^w$U(phSV=bXjg-QNPc5{}- z_fV=e^=4mb`h?`*07v4y4#D*=+Zg?g;6;}=#`i@snt=YA9jRh-CabaZ90am`*`1Ev zaH8%e1eryQX7vc?KbPhi7*6gFex*q2}i31QFz^exqS&7S%Y7!aIEzB_#e)2Bz? zZSExyo_3QK5$U@8lw!EL|QUUTGx5O4*o4vzX!# z?kYJ6GZ@Pw1?VSWt!2ma?9>^;t}3MPhUDNjujlT7ap{Mh9yGFI>!M_VE?<#ETH`#@ zNf&83TsloSD6;VvJfpK~n)H(90YOSqA;;V6M(RW^re(?WgqvZ|E6`K8wrW1uR(*rn zR^KZo53!T4!UHFKDsW2)Aj_^~mmT9J)%nF0IbJzZp-u40XHoPnl=im!Wbnlruzt#> zbehkJ>tAJ_18@_ywq^@^t4Bfy43-PgL(XN4;$xEPcrFAvHI<1>i|#KZ5U#RhPRJfv zwVf;!cH#~YWih4>oOAzTnFDZ7;ll$rwzxC3ar!pKj2|nSwpSAJW8}{}q%?3cFzc%o zJ`&8aS)`N;vn|yP9dJ~V+icA6FideS5fj~EwYVQRy;WiB7jC+lJE=5N;&n8T4pn~4 zn!|zkvt#!`s7c7iZ>N125;SMcFghR?CfLFGq=ARuvjdkMmdC|tKMTIqz!v0*h(*NO zt84vwfYMB!aU2iC;Z%_crsdW8&5so(x zn;E+T{`lwuv1tYOO?VxeCj_bN4SCK(n`|Fs@HIu^4LMLN0dK+8zlE4jIT}= zb-8m(Jr0guX_Di|lDl-b!Jz{oxdX{LzHnhf@cXG)9}6xMQvobz`YT8Ht+m{pWY(Eb zO&x=vyl(Ll6htw|gDjOHSk?h11;8j@QBtdqM@?IIH^}AhZUIzU-W?jG4}h1KwU%sg z+A{mt3J`LV=3pTQbf%W=B57)p{kf+|VdI%x2nlewW+mKJz&aU-H*-uiTB4>DKvX(3xK9$Fyz zCjg3CfJN9%0w)Bcjg^ZN5DyS9GPm)wNlvB7Jn@t7=m~Vvbu?-UUy+opk{l3SgvO(> z(^L955mnsIUtS0#bu2_y51VbS0LbbG&Y{9kekKY?AY=V3RH2XYKLneS9sZ(qpq!Q4S1R*`$SIGf zUDJ(OCO#GHqpJsQqB3v-MW}5PS$B;I^zD3cTc~B~J&v^VsFxv;9DJ;jN_(e=&x-RS zr6A51S*HCzTpmdSOxs{02)dLOk71#c3QXNZ5ov=GF~(GIv;$ye4J9=+&BlzGkt;m~ zk5M^_YSn?F=+^ZB;sgepC<^|Rw~q53^hv;U!s8@0ZRez&+a7I*lHFxHR&~?4y7Ha7 zIB-@I`L#oLzK21bVawN^^}DeEO(01SAdP!&2=M8&6E#3 z7zXpkN4_ng-1yE_!9fBaxCR7#!7XgW4~l;H=Y%r%3S8gxh!>`1xVKnkzha=}=yOCk zxb?5^{RT&5J%PPGa#+Mv47ifGGA+BB!xi<8;@}$x)OH^7Tw9xc?nGR&{!UsFSAtiBmdG;i zQbUnqj4^DP`#UYzXWYe2i?9=lb{wV!21osl{`;nLSc0=i0{Lbaaw6lRIfAN!#7zrJ zo`_6cv6@M)s+0jM52xv@hsXD&Ip(IpaHU|M5Kxcd*mV6c(j>+poeZA_7-qr&X<>aC zg&10HB%^28=PFBoOXk|iU=NMhEZ)oe*Kr!XXL}l zQ5c8~aAU2Nm0P5C=@&lN5!Qm|Q9FBcSi>qyh~fxlIE*|CDfNsdmC5HmHCRgyQSg@| zyEY)I2=rSAFG(lZrbi@6La=u~kfv9=G>5fGsLTw44w8)1a6c`bf;-xLIv%Xm&Ip%i zMeyl3Iov5I%l)Lfpk=3uaymPAZg?J>q$xCaKAX^JW8pRgC^vX&q&Wk3-|vTR zgF~AZi#Qoh_R47tdENL6V>mrvW7c9a`H|1WEnOl8zWol`lFMn(@f-ytY%XsnG0+KO z=-7!Won;Zthu4tK+||!h3YMMOM5P1|V>G9;$0QNtN?u6+I^lm|#DnZ&Lh|xbX|nsc zCq|You4>f91Zxao!5`BAP~%0GKIQ(Tu6CV@A%E9DZ3VI9h$2tt76(>0)IOQHAu7N1 zFcJc|4WA@A8!Jnu%}8p1m>SSZO>`w@FcJN6UJmLVL4y1b4(y+reY(>%I>yRQd1qSf z?SS|MXJcn_F8`bEG}7JCZbqg~@`dUDyp^?22kkErK)3vS?wh$|v>#Vj^&9oCK!nJ*N{RJza>Q8h@#7@r5SazTArT<%# zkoXqIo_cn<{XP1R&65rn5@u+P%5^!*kax?iak8`0ux_Zuv8b!?Fk{j~s`(97>8O zU&N;XZSB}I%kf)oRqmSs=`Jp99{Se#FGI5r`ig9br1-ZG?R@l`_*`YhIPyf1~j|Fgy-=kUs zRG!v^;{_yJlG|GCr@G{(oG_e?OMLlM+`c5jbd>CT(7}N>hg`D-OWQHdBQDNtU%9-V z7$BW=m*$#Arz|=LCZ9temiDuc-MFR+1gJ`A#f}?+4qd+g7?ooE176~RlP-}bF*8kK z5%%IT`S>h19ANq3ko6=@giw;LUt|vn_viSd>od1*b&{j>g1F90ebI007St&ZYGI1~ zBXt83gI%}!m;#25uq{X7IM^tScR7)STZE(nW1;rGX|oq8R>XzyXIvPQ4A9vsZHk5% zk|YKKds4xUKGC%!3QTsg_@XSU?oXHfxJtj72O*-wv;!ejYDnEW%vFu0D<NgaH3vT<(TkV?p)V@tOcXFDi*ilHuLOg4Sv>@^l2@8OSA?@HH{3C zTD2yei=|(^igY_Vkos4BWT5Zy2}x01CrTk&A+3QpasUCy+zvT&-+~ z=K19y;X(#jrQL)LguC1X*Gur`I^%zD`3H|(2x+PicuDsn;k#(Yg3gcO%5tMWk+9gv zfq>oS+ppckP)jZi9P!ut2?Abg8A(&ukqk4PLf+bM>OfVITc7+U@+W6X(&CLS~c?4&JE-fsy`(AiXwAxKIR?`&GUUK9w8jRr&I2fR&50G&vG{NULL znL3===F>?ucROA!1r-l!Mmufo8hPuwT8y-sC;Q@mUdw!Z&oVQ78HmwGe8EbXecFtt zo%AN+$c!Zsi3suA0hK0~$cJle>Lu3jRqju2L%5oSWA$5Ur4k&sp|K+zLo%*Cv|k^K z+2C4TEFvBs^?>P>l5pYSKrZ6)t)!4T$4aQyC8Ip}eulUa6r5ulZ@^l5M)bS@BF}k{ zVCWS@X)BK~NR@MAzteEeMc)BUZG1Ymcep1LR}6Ct!#iz|*8@hGgEHb9t6KKuqIdHc?!~D^c2=qaxUw;9H#5Q*VH$6dI=>3Zelea7alqbUr63N zH4;g85)XA|p~NC^J9wsUUhsfBa z9HcvW@lnfs>%L4T-0lpoF<#HpcfDz8`dT-G3RhqDulIM%Sm$(xp_PdIo%*koaAj1H zl*O~_GF+?-wO#}MN-5R9gfW!^8G$-pp%eqd=F)D5<e<4L#55qh(rgHdam3ZiY22xfU9^OzZuv#tAG_ZT zLtQVkZbQUHd(#EqO`pGr7O{66n3BMHIzUSHY-o`9p*KxD3Qd=62PqH3>VY=VvZ=Xy zInY9pdIO5Dr_%e-EF{o)Ry6FS=TgLSHzYZj)sC9lv+yx}jxTuDXkkH=*A}vg&~PKo0-M?!&~LKtth)O!)M!UqN{1ynBh}>|n$Y)3W$*deVk`M23045|XkR z9%pD`oV046uY03iH?-_c_a%4UtK5~d@3wi$EC^gNBCAW~F5>Flhk5kH9lhcN;kE56 z!Z;6k>fanOnlSw5I{kgw@|tlbRi)hRw2TG}t>-K;AYS$Xq%?d?M6nG5rWX=PXx;+R zfKR`^7TR2>0DU*41ay?d#?PQ`H~ig^v$;77-Y#o^mE(rv$wXZc?FgFV-%lV5Grc{_Jk;3LPhtc zEfe=LKoL*oudn3DDp1PE#&PebeDItyyfmsDiU@Q`ZfRiNHqyyDR-2-Ou7FYQUq(%Z zrF${36}4LHk3{&zrKt2>`7&dA4i19I0Sc(MtsoYIh=xSE_g<9ABGmairlkdtg5!px zAP++8)+1PZ$&Se{X^tk^UgxZst0oin)FeF3b;=Ir9I1tm& zEsc(*9S)VWV+vRg9uu#@xnkmC)4<$m+(C{Wjm-O-tA8n6*EaMAN!8A~D_`C?2h~!w zbpjD?LmD2y>0hUUdVytkz`!F*Tz|{Uo?U#00gd_YP7~SSxOLNL#~f*`n((mFk~T7P z6Rmq5B#%)(CcD{n;&8a_OqB|Vxv^iRjUpNA=XB>8ldg->1E`T6KMZcGo$OfNp&wNo z`WXozHWq9rn}NkVY@R7YpKw7386r~K6K!LyjD&@y9o#i8A!UQv<@Zfao@V5UetzBy zJF=ELRh0TD$sW8WC5DZO@{#cuA5N-t47pOIKrQ;tunQbId?p%IO|a6R#jD~u+-P0ZjHN>nO3<#?F+83 z*@fm#R>_Q;eZxrLp1Zp&y<1hg!KC5M?Lri^bFb2s0Ceol=-T7P43aHk3Q7TUzT8@IIuEj_ufwBb>P={H}qFlqSLr`dQkHDiq9fZ_PbtaiB~3 zxPj)a)=W5SbP?>A*ienvh-}(qV{5TziS~)A1GG8mdSXiH*vIEkXNX@@x2C4_li`jevPUdK^fqkK3q*z<5OtGM0z4M1l7v{Cf zq6*#zSyyDiZm_nq#Hn+*{!;AU-p<9`Fa4>?x@4#EBxans7euzHKLjHr7J|_vn1Xl> z$TU<#Ymc*$y=sLXI!o-Ak`>qMSex22r$}ntSN=wN!TIcplJ2YeNCBfp1pt~ZuqNl; zd1Cu9I%Pq80)26O!W3xn;&c2bYNw_W@sOiahNyEXffgONEm4OtM~WZBWS+B8anJf; zhd;;a)``UG#X$IRh&mj*>{u2a%!-#WR$gvq_1(Qo)mFzBi4Z9kj3Zgtyx8omunwSz_Io5-Y+t_!j1N-8Kbq(Vm*Veq*8V+oi?n+$; z+y^vz9c6J-2MCb#m8DPWDw8uZZ|~3Go4zs{Nx@l%dBoa`l(@yQ)~K{)6>Dmou!^C! zd4P*mq)Of`!h@#@pNk>j(4t4DjVlpFkUZ(h*Zu2>#Bq3-lfyMYuN!URxO?KQMj;R{tS&XIz)s!emZx zK=-cw;E`q?<>ShGPXM597>N+2q8KY9tGE#sKWCsc0i-~8to85vxoH%SS0;lJ_iz==nIR3-fznkb_PmGCYtT*j7?&c0Gu?Yh_r$g%c z=A^}5;l{e>Gehy?;C3FTD4`B9N9bXEtOde<)^EQJGhX!4WAJ!FT4Wf zpU#b*-{t0d<4|=+^F|yk=2#I)z3Q>uUB81h@15#jMEG5ToRYBV-B3sYm1fJ?H^XZ% z#T9}6g--29*1P5!x`>Xs`msbZ#LSf5YfN70jj9`egegqwPcp6Gz8bJdljF+A@=r87 zMSMhUx8Prv8a$O_MEBxYINmh_7Qwdr=V2iiYmyHAmnu(>zq^d1Q>=1wWqvJ4|DWf1(3q*J=faw zbNak~3UoXi7%KN@2_!hLS=19Ssufv-X=@9>RYM5@=?G9krQMQDLb1fwG7*iS?J^HG z9jDg_hfwHwx6k$`c9JCoD2DL}5xv1{0aHYf(?Hbkjr^CxG4}qrG;@3r_a^u(68qxG zMmW(Vd+cO+7R}x_4aYYmr8+p1VO0b;-G@jlPZrq4b-?XQwW>5rw`9LFV}2&p@t^cGj>1YR*8-D^sgW* zVwkqQXSmf=eA2RBqAa!g1x^&aVz#u*3=Wy{~G^y%)tzl8IpM1G@WS;C_be zceQv~cl+`aD8Yj8I23v{6m$^`9v7}d|N0K26GC+4MzOw^Kmwab1M1;fG~H!vgj0Tq z-OI=-E$dJd&1)flE$o|k>b~K2>)~0ge)eyu@!+-*2Aqs{R130cMRZfUH_i8C{-7f` z*Mik#PEFDi8OYB+1Qu2QS3N(j0bGlAjb4wty2$sI`pg~sLpzND6A$e0f8|BV^A#j6 z*Ex{fWJwY&PFH@mbb5KqQEMp5-cCb#hXXyC})i|{ye3WCkBl%s@5Vvf`88+=K-x6=v;>rtM zI?Yad5yFE~a}pH?MIsKZRHnkH(!AHX1at`&yf@0p4;_&u!AEin#l2@%RN=Tna*l`i z;;JFSMl*ne+>(MEMqo$}14}*hu3s5{2qngmuRG-FbEKO2yxs9K676BvH81cFkHN0au~TCoIWyGxUwxS2HyOAxN{I-9@jaZCX(L1}ZXJE3 z&dV^nPNs(?*~5K^9M+b0c=(NW;Y1@`JP76!@|&vMvEL`fUW!759O)0%Y#c&2>Ubji zNP>J6{72Bj{2Xq+D-nwgz-)Y|moJ6ZzR_}4u;7QC<>+C?wZ?qpXS}6((r*D+07Uv) z#W_&WBJ`7a?Z&4mq|mg$eYt2t-cHr1@*hmTfQX~m-WqjVTV zroHR*o2wyQ#{QW+S+xqq2n(<5=KmaxnmFK;YxKq=W?Dmg>6?ETHzw4%{G({GSZ#3-8Y;=Z>2{G z$a_Kn^g?a`%t$JG3c#vpzk(dC8K=IuKp&P0Kpdx+oX~_YWfJlB4}dFDzW|95>*B}JVTLX#fXDw z{wsj`FC&&qsL-9i1R!%Md4%vU%!TO-f0YL_KyN!kiDgohBsg|3X{)g2%9(HOrO^$v z41AQ0cLQGBTDyi5#MgmoS~$U=Pf3c%*Z|Eubt8mD0HRUZHy}9dwsQT@7bLjYNOI+6 z?1!^zs^AvPwcA?zOpd`J1>eh<@+?#_ngWsrARb+9A!lt=rR~a9giobM5ZIPA0EDI9==Q+4QyNOE`m_MTx=Mrq;M7fpl#$F2UP-#98YZCg!wadTB zRGBK@vzJg(-TLN?Gd}aB>*`3vx~g2e6m4k6!LgJ;BWcz><^D9j;eDdr-uXhBg}o!J zHWglob(BxF@LbfMUG7O4fN_-MsvfQveW0dh$`z*>K+jg3f7~%;r3k4w8n6#(3o^S} z|B8rVKAF`&U!K*$vFQ`uK;I2}R(-S>b4su2S@2>Y!RIQc(qkq92L0OZgqt053S9`o zO8LT}g-#^jLclwhE|`6)Jy`k$0e}O+VvRi8I1vnp$ zgj>rrd-fwgR~NIxI7Zr+&wL_tK!q`oCo|CsBx|2o-yvf(9VD^G-ceOtLJb~vV@sfk z)c+6;N3H~2Tr9ZrY%GzE#5(`9WSqG`G`cGZDxn1JcqB7K6DNzL0Lx=7W5o=I8^Y3Y zu>phrPVU`o7%^EJ-9;S_N9zzWn^cks#k@;EN>W@A<)1w5ovvVd(ocIIWtzNmwvM+P z3xuCok<~)?L-ovo^lV`rE2BM6h=-o0qU7O{;*lhE!e0hXG$7#K376+Q1q`zB3!agq zeq*$n^eGm$_e^HQFz0DMTviNBAlZg%;uq5J&L>;?qJxCW{#yvaouE0IbiA_T7ZRs4 zj@dr{eE;$N`}dHSLE;`wDp68=8eY26pKACOrJu{&DoBzg;7?syzeHm$IfY4E6tw+m z6$rAnOX&EY0C_8ml|aUnqzT~od!7Xts(<*OMT6O2z4M8-(Kl_zV=8x`sSF>g4SZ2( zy&PJcko61GOGGdyQqx{?>21JN;{QN`;5ujuZ1ejZo10ec337`|-mU8mon}X0Kd)cj z4npr7jy-{Pahk<2cudT5JV8M%Z%71Sv*JnSn@4rM>zZ_jJ(~k18 z3c$q~;79=!C`rGH6c-JO`ACz_bxkzS?q8vEWV+F83cqSxz z`xQu}a#Gx5>8{}ytj&YExRt1%)llI{J%Z@G^t!owgQGi^BA;B{$5M#NH4AKiC4zh1 zG{R=dL9XnZ_5<&TJ0dR#^L$>opgw%W+N3W>NU@f83T>A=g1A}Z3Y-LRoFxZ}R1O`l5eArWXhQ4zxK>$rc8{9bc{lVZ3X*=TCG4m1GR?^J zdxQfZ96G4)60B1T>UwIkEe%~5w~V5-7@RP%Tt;@j@{F8FMp-U1T(d#b+(n ze$aK*(oOOvjq}-$E%Ru?KBT{=Oo?PX->b|EyU30M*oRrHgd~wX#l5VANlQx|XRET)Om)&ymCIkp{!Dj=)=XN5tHFs?et4P7C{0|BO~agtXOcJGQn@gnasdd5 z2u?*@?PEU7z-29Dv+EF6mCf*X=qq*)vE71PNM6@43g zSRlYZyodyp@VU=^>2>xgG2rqAkwfD2C>UF?EQcPo0+T5L20^ZxW-_oifHh zUzE3T`80i00B}21_{j=(7e`K7MYhrBzqbF9mP^q?j;~UcYl+vglKKo+50;hXw4;lC zrEW^qmPqAMz)x1^+D&cTAvrUnUOoOvyicdGU9V*a`PS`y5)j-LNPEnWC3RL%r;g1_ zNF)q_2`=AkE~8wmJY9Pk@BR9=YaV?U?U}U94Wo5A2z2YUn!7@f4td{|@2i#~vL5y1 zc`Hc~XM89dg}h>&QN-W2{F}FuAxZwZVxh+@T%bu$9#_QWk&q{5$>Gxy)rhD;bEwM=`~ zvdWK0-0~lybW(F97iG8hBnZ4}yJQ{UV_gQuJ|4HBh6RAEUCLw#W{4J<>(gC`HQCi$@DrF5sT}bt&&IjWpIaR zDR>U;E)5u$bxZ4#s3(x9kE_93&El4@Keb7(4BcQ2(|cg1hsw>>Y%cnSv7+1AM=KYW zf$1EyTHZ^O^^K7v9#Y^M+G7}O>j%qUTjw)bM@{Xmr+wPzukHE)2Slhtz}Tr*7Ax4m z=)91FkjNE`!D%H_##ZMlo)9{POP6&x7c1_OBBO6#7B|)oBTsFf0Po$pQ!akz7p>tw zR3^(CDbX?SA)bLQX3mQs(SR6PLx^1+bSU|s1zEH!oY;t19xDoz<@vuCK+ zjQcYsCK{T|_rtRa#@Uw#X(qCU+1zxak)DfkJxhCxe-45fDPPOQTml=6Gk!?O zAG%ggnm<`yWb*cI;kbS{j~Q?5qsmYXiS>iqo;$J2w+ZKYqK;$MhB6v_DtTjOE3jX} z3Juowqj`vnd!~c7Kw>hK+X0%!$fF31#2c|OgVr-k1%Q#O4yCk-x|u2-*V|r!qKA)D z{MssfMy%{Bm-FC{j@=okE0Xj9FZ6OQN22%Wqo6+&YzP*ju@8N9)3Pz*f2rgy8v?wtlc!DI5I9$G$F!7Pvj2Z^+mYK z%S_C%0oT%*hj7)nh}eU9Y2;{4I)#wfKol}6XU(d<8ghIyyEJrl_X8^h(rnuJDcwk} ztcv%SyghBWYvqxOd95VqMjAJ9v5~1;_hScF))qmQs!mCB-3=5* z?MFzfs%C$o^;i>Of773Mi54D}3o>)Wdkn>GSB>v*cD4oY^?gp;XOo+2MjRTIX&md> zFY7M)DO2EyRY7J8LE6Jn+AeF!{#f2;Zys9;nOWm_KPA9h^JvWs@&k9<`7=r)*Dw{& zmM8L#Ml_6Xd1i;bXVf0#-`I#V(vU_mE{2p&rHT6|%ND@Ws{S|n!%yFS{P^|9H_o9f z{>QiP-$X0He@X2D|EE@h$rspCa>X|yxBu}C%o=WqZ)N@M-#>l-@BVLp`~LvXe>hI@ G@c{tQCwgK4 literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..92673693064f598b06df5f88bf579f9b3dc6acee GIT binary patch literal 353 zcmV-n0iOOJiwFP!00002|E-ZtPs1<_hVT6qKC(m87RC_UX+q-6#}T2DHeQL=CdGC@ zsfwE|K%$=Y8Y=z*QVZ-}}%(8-q~n>#?^1;k7iCDdD6T4$g`#w)fkc+k|w; z8{6(NxWc?e+6y6aLD*Qv{9<{T57vj`7iy8g&zOn!9~s}AIRLJG@cfJpo&;H*V5GS7 zJ;Fl@aTGuJcXW>+mO7LZUVsm9er8suJa@xg3SrWlnz^uS$Ct^ZZdKfhZ&Y3mwL)8D zuJIHN@wYiU&RJACKL!*Io7Cw#{)v?%ww2pmE?4KP@~mmfMV8U0P&F%M(*iD$ZF;OZ z4TGgG#CkmQ{oM#lAvmzS6Svp!#nvZ#(PsY@?E z9Gbdn^D6tn{ZYQWefjb0k6&K?|H{>Oj) zg#YtjWm;8v^M5q?+yB1Le$D@te>wd6^~*oM{*rzB*Vli2$ zYCr~~Z9yZ9!vtRbuc9j4DbHU17cI5@UoW)8Ls8~0|8G~-m$o<;f9Bc$_tK`#F>n9F z{pU+jy?i~teErq&uV4OWS5=r^R~0P=a4)Xe|5O&c9~&2D=rZ?%YiV*n$Ur}3`K~+C z|GmUV`NQGMKX(7z{rvMUKY#say8HI^KmYc(M#um9_2(BJ|8MxeFX?{Yox3t^>n3rR zeE-{T{NZoEWtn@;+x;nN@-%bwf9X5LTb|$tO}hU`((-ajUw-`Y$Cvu`!C1o&lB~{M zQq^s8$(y!ocFCnK3mWr4Gt8SyQ&jD=kxw64lP0fanG`O0%|G%C|CsjMece@UaxSVw zS9(q_E@|sze|E_wZPK%wHV{rmhM-kpF0y5>GRU0C=tSsXT~g5huivZWwWzYB$nvT! z+K+`RdnehHoO0ric#Yw&b5@%-4gD!A9P$1KjVH^#G1J zwz!luK)1Wp7ks6tE1qIn7U{%!ooM4vsY~{B8fAU-Tb3N~P1%RMddtiDk|*FZw{CiY z^=3>m8tWWbmi{sA=Po_t$K8ckE>D_{{yWR}bw&qJ=11I0KFPez8)BYnpHCc?w&QAj znBk0->4Lgy=91_K71;d7F4MnQrX|dFZr+$T)^&eG2sb~{dqI57uSvRZlZNh9mb6WA z(aS@kct-^IidMAm{lSjMh7Lt|xvmby5#v=^{hrXxsk=5&B5$#2ZC*=~_QfQPD1zdN{V&-dAO_v0*8VDPO10@CuVQmJ*vM zS$dp0@%;ORwxehVGo^u(kMztBNpY;|CeNmh81G<#xzqJKLmOy|Jzh9wpnUkG-8nBh z+@x-bBTXr0IdH@NltOe75u#e?+(+Jq@x5-+_oQnN|4c|kG&H$A>811v7iXF!9U+8@ z{ZN@EtFg_!p=qf}y`%Gglgc?QNwlA7u7`$rIj?BCOqqB(Ey`qH)-IpIf%ak;^PLL^$>Fzxv@6mD zY5c;jByFRsyJpX_;XoVJx&?GxULNQlhrGc7(!xk+p3|m(koos~--(!kvN^mq*Z05= z4{=fEt^ASB!qGO)9NJ610<)!+yN_zmTR|HB7PT$Hv@L8*Z*X(EGaV&ZO1lq_QvGHYOEs8=VhvyDLeuC$v>n zcg}IYLD$g~Xs{h-lQzZlXTuB1{-fo(=B>cN?>pDlXW2>+V(CfjQn^drw2zk*y~%W# z$>8nQ?aDe#0)h!7dRzexxOvk@DJlsan)xa0q?6-kMhHJzx{NF@$>eKU_!E=i?g<;z z4GD(?ggVj~cEz!)J2!di@yR!NOE=$%G81bGVb6rw`om+AyAL|e9ogYfT3i^N{)aUC z152cV^7!ay*PqY&IhnnK&Av7XgTbX!Y;`AS;Ey6N$>s{5+%hF64Xe-Nwu8D$%2NJN zl;DfWgOBoqw{ASYxjl&*sKWlEqS}`o)JWMS&nHz!>ej9(hk|F)DpVM3vo&Hv;uTN% zjXCAtMX?@H2<{A2!BkN}ma};)cO&>2Qkv#|(ImY=1=Cq??-s+JJ{(K2zHlwr^Uy*) zqGztX@vz<-cd@QNuCuuOoL10d9Vx|W+u^Ne?HC)72|xV-ml@uiT`o@zEriudR^F%1 zoNU`%pCIEd{?mfZ9o#hkWE6jjoX#44z;2B^z7r*+ISpFoDSiwIPI|+MzGv16>A5&8 z+5uu`HIK}!QIR^j8^pI{D>T9CJT&yK%2v3Z1yI)YYj>G6hA<8|%m%_klwKSgYFgzvqM}Ju`6eJlPBs+Izu=O`D}dX zBXK4vlHpmyo;BqMzG;_q!Z2j<@U-YO$%t;%X*)i`EG9jWBEhaI(A1B@i45!qQGaI~ zhW8ALGDc5&*HpFyqDr&E^;K(q!hqhp=EWFX!|f@TheqkpMM#7XXvgUUPI=QC+hw8^ zw0-Z3H!|g4EX~S#*B3n}dOCu{8{G>Ye&8^O#G>6QGmi@Rv9EoPy-~yu5KD$6a>&^H z^Ggk;fO|>o)uPJNY^~@IFnxpT+2mkpeg}CkA1?S%z;0?kA|HmhSNN`bO^n^r zgZCyLM)X~^v}h{cAR=j0Gt-D0?eT=vb(Q#u6;9_i5bpgSkvN6m7KNZDNeK@F%{}Fy zq1Ohy=5R?r$~wI>;+0A-k9wzE;io4CKNLqtPtZBNIU>n~BZ=VyZ7I9Dm?w`NBA5`_ zYP`xxl5GYr!_BcGH(g@-qKj2oUiG*pJ4f93$GE9o%m1Se7M|rhbgpySywcL^%kA0v zQH0^cJJG^pZ>>C^yzl`d>uU9V2_8RwT73Z3$un_#I@pc=C4_4> z$Ks33-k9Nn)Zy&;ENIo{7XPD!Tsod}xboHQ$9t0}YJ~NdLvMXKa3@u8DR6YEZwpSF zNpuN3ulFS+v#@G3Y7Wzp{NCYN=XYn)e%nrmqxTOK^HB2MunSR~1Jj0#9?YQ?dh_kN zF_!lyk{~HX=RCM@#O!~KIc@C8xiw9^zZjAvb;AMNNPB=Rgg{OcNZD{T&BHZ`LXhjx zX|e;>D30IbTf?1L9*0TL^hDUl+AUc$LDd*+zq_NL$y=TW^*9neoS-4e2K>5rJMYNL zzpFM7mN;oKL&kRl#dOFxytGIz4{iOMn^vV4cdAFCqHHAv$Hu(Tg18?YSEH@4TwMV( z%tlw+Ab0dm{YHQIk&~e=sdIVu@0Twxh$YB>X#g1id8W6s)BhG3Lbr!*1a~m1Cq2$uRnkJ z;JwPWh78cT+_pN zQWKcDbm72F&aae1pfML~0wec^qha>1Wa+p(7J5*8pl-MMk=P*wDD{;J2?Q*EKFG+W z8!j*JqtSg55=#>HL7Ka-$?00{!Sf%SV~1;+yywO7)IJ)O^bj9f6(v>!Y)~{|v>v)) z9CBn7=r=ij-ZvdY`D$|Lh%`HSSZT62w@M^7YxMoK=VYh-_S=~(aXOmddpjck&9mUK zP7c-Ez~4W%zR|ToJ5RGSgqird;W8PY8`%rjAF=FIg_ z)89!2vaivl9cXC}2B1xUT`N9$w=UviR;9q|~WlY)|dXl;f&Qtbb5vS&PsXi^_ zsP#svFEoir>e75T{6cEiP>sL;)da4LW5w?b9JcT2z}2AF&^CYE`KhMA z&@(5AUHd^opU+7Ub%*#dKfuI6d|}AUH(noJ3SA|bOzu5Zp1aQCyzz6Hp)OIQa*)?} z7&vXE7{;2!Bg~%c+C}GOeo|N4{-|FkO;CRT{iioE zq()5{fODyMSx_~8{{EfSWpMb4Wqc!L6fQitj;bMx-x$is3zLcG09jp-3^QgldSZ8% zHkrL(jqsi5r}VS70<(rEJNb5KSbC57g}l`p{*rr=T>c>Gg!Gni;ulohI)rQk_hbIH zs<$G4chkv;FSxGg758>AZkM;=WtZ4{Cb39fTM7eK{@`Y=sY!B9@XqoQF$~lmV081i z5_=VOMlU!A1v&oBO4-~aP|tspA4$t|BBmYt9e4#(hdXqftJ2n&*GQ=$y&FUA?>NIJ zOz~7W4(hzo=}L@>1Z%RRCj^S)!UdBs&Nw9JwJ8K9BRgTAW>-mh4^GXB1#ouev1ei` z=y9=YJX#twnmRToC(!@T;qdLdl0u3?8pXO3Nj0wT=~JpRC9GG)!qd#(@NzGc7{RNR zhiHmqo52+#y4S3I@V6aC+sBpLtjyB`wAeHuv+8(Vq;O~CLXBge)h4>2IN?~(sa!$e ztz62$rkow{IMaYgL8nK1=n0z6#G1u+A?*m6;jC1oM`WqCTojkE>2gp-XVzx0S`xzU zQ3M6|M0`t<{jSL}la~+@G}#6UE_hA{zU#%Eq|U*u7?rSO#4Bw9y)$TclP{iDCfGHG zehC##19m3dgyKk6^qrwkAG?OEUzA-$S|o zRGM+HSZgTY-h(&E7cB9$7|#K?w@2c9XSTGNg1hmy)28o?Pgy$3S&)reUd?;NJ#wz! zc)ov>LFeFM=1$TF=9O@CUut9zC8#0i@UQqIi|IU49}X^W{i%VgU+F!hs+?cli4BUIho^4lDldro+rc--8|!NXtPdc2dMkrS!8Ozgk7-< zqKi+L`Gf46fh!tS-rj3qM5e0-w+7ddBI6(V*oStNh5PNc6Q}=1vkwUc0yWo8mj&GI zTDqh@GyZHcW1i1iLZ?}97ITmSFJi(DoD;mOrZK@COG|Jomi-Sg?8lcqL5sMdP12n{ zO5|MqTXWS+eFAJJyhQ+;*uM*x(=Y^gZ$!KFy;?^r%ftg$FzkXiJw#eW1&%Xx?}%}g zS0iO}sN?&5x5=-CWQM~oi0m?dnZw(mW_x6@Pgl6SIJ>E$ZMG-rNYmA;YMz^q_t3#Rh^}o}f1JgkBbvFJ`#z^xdT9s#1~D__-4AXa&X}<(sIemGJJf#Xpkg?0 zQe_*cvbtD7;TXZ$n4-|C%K1lE<9+m(o&%v;U|sBI9IeL5d?KkBuaS8r=RV85G0=wM ze9+PQmOJbL10>qXRhFs@*0(1kSFHt>%slkMW$O4may&6FDCs)xh=oqYW%dP+qam90 z7jp1z{BZw9%a0#h?~=yA8ZfQFd-+6fG=14p)vT_>vYPbnj3ijx+pO-qW1ZNsU96cK zaPemandL*(c$&i!KZiNV&lPE7Wp{}nqw2b31cve0(LKhoY$tl|;Db5fzs=lT;Oys| z+gnQ1g7Jbb*}m7_8^xt|sz)*aU153oqblT`@yU1+$rUNBoEp$`cxkG$ycVw29Q^4t z!q)m&gk+*O`I${lX0yrnCUT?|0o*C=T9lKusw3UWCJbZu9v;`SIYqB*a9|v`%9j+g zddGLBKn$m*Kj4}y95m+&VN6ms{J<8oHB!dG$huD?A~hi&(DAxU&s*DiAqJ8xoZ19Z&n+A+@$bQxoNe;cqv_%9|*mMlezsAuvaR zTBUYm8r=1nYJGNYT#Wd@2^rJKze{NXlDpo{tP;zUCf-bi5%j7O3H%Gi>}MzMx$!UTP4flfnZ0z2{)Gi%m@ku=AN zOgbE&hd8u!e;+tjay-KKorHRrlX%T5K{hGZocO4y>VD6>KZqNQS^Q1 z!V1dhoMT6vIoZ@UkB?L=3V>dm z2;yP+@c6$M*&xCGp3A|i-t`FjI9J{(9k_00duADtGRQ@CUu0Sjt7W|VnhbZhzx~*7 za0D>MAOx2yO33^5CynJ)S(-MdH4k4Vx?DHSqWeU+e4{@&{`5x&4GNpMcw|3BTE0X3 zyKos38DCcC!bLGd^x~ZSZGuafsuUhzB_^AQ+1=M@C`7O}aYSR}y6B8`@0{yTE(4lu(X4==nD zrw=rFgh(0u&R!BZx?%3{Ax;|gUuN?OF+MWayG78^Anp+@!Ym}=f4ZY&Ct+qW+(q`p z;pe-C#GNUNhbhMG_3m5Fc9ZlHOdXY53$}#CjaD~Qh(POR7gIPhG6LxN?|G)I`lU%3 z%`s%&B*=uhZ~fevFV}YNNZaD5vgZ^(=VZW4%b5k`b3T(UP#Ww}0&t4dN7Bx>gNjp0 z*^DL7um{*7QdX7Vf`tfUE8LPk;flaKD<_A#ZUsEfX9~d8={V_v#gSYBH^5F8v){_=e^37Lcqv27m>PJ?Ibv!O&UO!X5wCoM=Bm2)5P9EcPR9Jjq_8im5Wqrht z7jH)}g7Hl1(GrL4_=&haSqn)|CWsxb>i3adP9#KgX_8;mIPNZ!hn0Zw;EBoiaH%kr z-EEEY>i)db>nZ8|zO*fvMGzY|12(tMDV$9d*N^IKu=6_%32#Ex%y28)7{kl z%taVH;}LR@ca|e@Cw)8m$V63F`T7Az=g$WOiCGwv8J6m)qP4g|^Uq zH9pY(WB)*Lm`({#QDB%r;I_^=ee6i9*mV2VH#odZTF?5iV0^i%RX^S+7ILV7(=fmN z{I)OUA*HL%b0?zRuk4&ZbU3?x&58T$KM5(Z_yo!FGYdGiXis@$Sj@?JxL07%Gv^B- zz+o;DGnQ^O)dw_*fR)(m#xT$KZWx~O`2?=z_`TNpgehJ#@y=9j3v{ z442XCp{OKBj?ipfzy(${4K~xs&0b#VQp(7 zx9@rqQP&cTSZF4d6djD9vo9hNu4W*|-C6Da^U^k@W8OMRF_Pj#Tj7?3_%IlSQjcf- z;A7WVktO0F$x{eLJ;$bo%oY)}$2&lb6?{Z@;*rG5v+Jdbo;9ru^{-$#8c@tth zixjFM8N6H?)3`~9ol9Xt91Hf%J~x;H@1Q6wE~@^TKQJ_kMqF29q8|WSr*r4bxoVRU zb>)KES3E3Wo;0}6K*`4J;x!{%zHYJfDGfszI+X?F$;$I4mTz|WX<#U-YS0$J6*&SeM)VZHw^ob%w})(R6%Poe!7{;^k}s3l=*h$Ar)xhnBIM1 zLt4mm-gr?J70h%G?ZC+2Hx;29gNgj2DLO9|7f&qefg7f%?Lv~hW(u8WPEhMlkw6xp zcibLne7IXzJy@p>XooTH$MvPhXWAqVL4i9q|Lw8Dj_F8?nT!VW>}8D+EgKyl-|(fk}oI3%ELOHRVvk@M(Ooi9zM*R;45RP$Fy3ND7F!(onbwbSrh z*115(DBU`U~wJIBXO#VNKF!l6M6HtEGP`N3m2va@8rq?4_aFxg>n0dEAU= z49;;So(Xr*lvMDV;VWv>1XL=|J_BE@JKW{q{1O4Fc&hPt=I)X%D(aNY*aFhcLHS_n z0tU@J6MeLBs@h~j3V=BNdb;PkeGCf3XfIOJq7Q^fy)jTAZA=T%O(SOY9-uG90lDHO zhs&TMelx(vQ9ESP5dW|t@Py7i2>%Yl1?Zvp0&Lc(^yp9PstIRb1vfJ!q+EtHFP5;k zyZI`@f+o@?gH~nsXM5OoP5qizQD3akooNcPnGGAmJY1Ol!^#YY-rp5PsMvj6)f+wD z`jFO8*S-`k6tO9!$nD+nIFcf~$o6ZBBP`Nn&?ek2ghG6~GL_m=h&{5PFRTps6!Jl! zVf3s;A5jp{@0TV@D!DWhelGkC~_Hj7w0|-2)6JU8@(#8 zxF5hMsLcxA)3ke^SQz&g9>Elxv@n6%D+|Y2qEJH`oKWMH(p>9)Mq05?x}osehr77^ zn%-XkJ+$3kn_doAaC_$WNA9ZVjiv>qjg_mDkbh5`Oqv6v&90zlp2qXQ^VxiH(@#qe zWn$i0nYr;!sI#)(dCLR$rd7YJLsg^u2np9PKF)P{^6Qtc^#6Yj>Cz^Nw8(%{h;P(Q zF1ZsSw=~K`MzOG@9?Le~&-=Rrjcrzt3QSbQaevz?S9B>ZH_>9#$CqL!gCNAjE#}!@ zFaHKVd3Ek_$Y~ViE(TVVdTI>`c&$Zn@9)VX?mE!1d!1~zHn;v{A5m+9Ta9Cby%Doh zeXxA+krtujq!n#pSZi%WakSpzuwUsB1<-V{S|(8QXc8qdrSqs&rZmBeLDtgND9rG+ z^^%rUdc(G_MTPcQjWngY^ycz(1Tj-4LT6!*PpRkvf*r^ERyU-DRY6^Gv(F<+BjH0Z zmksGD47Tmh%LbT!2Bbgd(wCvz;dxhn)3XI?*KC+-+4YRnkX?qR~0wiP`AO3n}d*X#5pNyjD&VBSI}kgt`9>x z{lp1_oHig*KGIPv`SFu9Wpt>4N>Ks5uc)yi*+K_k408)JZe-HOCcT^@H%x%q1i+$u z^m7j_Z+C{9`La1kFRfQr&}Ga^<2l@Geo=hQw=@HLNw{|$lkx7DJY1w`erQ`4F$sK- zWKMag%ti486cXSj(4*-(p2wFurVRKbTo!5d2oV{>nm?@EoY6wZu&LE8*&ERO$LMVJZbGYJCMqi(2MgCEO!mzLZmjS-dbYjE%T3Ao&7z3SRA2OuZV zh2XB$aY;`(aaD_TIOA8`RyFi7pEQ5%s%PNG_F!6{_d;CB9eU=`9=1N0tr=!d+`F4? zdc5*^NR!FyPO!Ho8s?^_rv7yJGqb1-gg#v)W#8mD4+FmI?;)dJ3YyVi`xIQ1V>x#Q zur73hUQXAS74EhXyt=8Y<6~e@LC5VEn+!J#%^0l*UvmYJIW`Z!<@b1D#vS*KXRq6N z16;n}IKJ7BddSA+Id8{r?HX^gUO2sKmlQ~!=_TC{H40l|FQbCd^Q$_s_9Bf0P(tXg za3dM*kLA0`#$afQTROGM8E63$r|0LgBTdTnraE_o3*2*>PFpuuX%tgO@JbN|$+FQ< z`_!=nq}ZyWMv@k$4rGab&7FLm(U=H>vq!NgcHPB5*5=H-MuNDSThug{C0sG*2T{x1 z0FyrYcHSMPAWL1c#a0f~$K~+5%_n zL6JFJAA)~EvxGPv?Bf~yC3OJZpCuZylg#+QF#=}z)V3G)*mAxUmj#(rS59K^Ge$WN z7~yr9Pu&f?D!2DyhMpGnkYZm*^TvqtxrADz4THRB9y4N|r_1*_z|~KqN+4g5wdVLSfYkA22!{}N;@y4#1leWt9*qeUeXU2rZ z;q1>O+H~nEB($$v3*B(+GP)i3d>H2s&C(>3;8Ywq=HHklZFvToTd)I+=IBr}&dsj| z$o79)qCm3jOg&|};1nG_5l3Q#myN8Zxcg($ih&jbB zu7dmj>046V-P5~Rxls~*FGOvPqIzuv)E72$TSbx(wb$6=%hBa4Bz9@2aH?;&cHu(g ztZt**%=WB|uc(Z9D!@3m;alC*;X?(qwTj`dxX|j-Qk?F0B)K(jepA?4`Bway@R{sr z{_-G>)Mm^Jw)=l;NI{a-)Y)tev_iTI2jgKjyG3Ul2X>r-2rJ+ndKhRA26eDNIq#&C z=UtHDagsi&GbU2xw)mG5q0qqR%a&%!3RL8%Ih~*vmZjIlN8X|txsZD6qSOfQ#BlTb z^r+KBr6c1#%b~b#2Y@BXV-vHTp9Cm3fNADP&4R2>6Go(t=G++N&0ZX?{*atv?`ar$ zOy_B7_IGhp5B%TKt+SuKM^|pvdiqwpHENCWu2@TeBgWhF80b3HeL>um-vH?pAdMu5 z6D7msJ^d4G}UB^R`YR7y;4z@5qzeE$qzK?PVC_ zUZAYan&aK>%{uRfL`4a`Xp=qwsRlBm_JVcYPLC%0zOUEK8TA~4oMd1+HvvI?oPDBx zhjFVy4skU#3AqU4X^?7pn9+(mtSR&)w#60ITm725q~HG9IvLDbO{xcPifkyjkREjw zScQ3bv-V=7WwWj)BrT&upY~@{u96p^sDMDo-l`dmKX1p+?s_vHi&;rF8cH(%kZ`Ij zcWLq3cTghGO>J=}P{XqjT%n8va>T=)<8*vWs-@@n`_vA)(5v24k3=d^nRWE%>3R0( zwVHKh*ncT^h|E2U`3uNil-D%tID{>(55v7WIf>{9qRf7ei9XXx_V+7fpGgdlHDk_v zw|wX3O{zRBG(WR%y#|nFeSYmeyO>?Nc?20Bp}ww=!5Xo}dtzS+UKM>>Bt_mU>rB8% zYN(0<)PJKt{P^Bi@z#uUN|!2K8j?sRZJOHd%qqonl|hU zl?2U=byTgekGtB3^8$*KTeE;O&6U$+>bAV-IdJMfSAO59Y`sB4U6q^|;KS#=Sedva zBrjWT#g1ff?$(Yj5xXS%O3sbMd?8BRO0~@SIeX>nySLl0@CUI_`A#4+INfe!$8Jbc zedx`vU|0=M__mrhyHwqChCc7|EHgZ|r;zXkCQlii{Xe8dp#3$eBg3H8R%ELD9qz$3 zLdkcjL|HjINEvfMRINxP4Ph&8EuqLX;u$60o9S_A;x>&8*n*%@rh2KA`wB3Xra)dh zJFnwb3)SvhQ1ZLQK46uKvb24DtoW}ygKCH^Hk%YX z&wO1l!t6**Qp`CnedK_A)-|F!I4&)<|F->{Q&$r}dKa41BD2@#|wOkGV30N%Od zbiL4QnryWFR9<0HT^#AK@H;+LVZewlZQE$JFDke=(H+0O)RoKEuS)CVirO#CQX#4& zax{=#P;SGH{MQif082_!-(FB*{le5cZ8^(Lt-&er01IT7{Cu=F^fAq0IWcsbSILv- z20`-u;CU}Hx12e^1&LI;i`>dCK*xel1S2mWm~?tpQvI7RiU3WTBze}!^c$faX z`g~yqE7pj$@FI{kdu(zk|8r!GqE)6yVR;m2*9GMm7}0#e!Rp-^HDOr)B2LcZ`prelZKEESr0<>g_^a ztmpJip>xHp(ZhgbpMLXgNuH4`09C*^15cggdG#zR>{fo#*wO>M%`ZZ{*eFa0ca4{&zk)`wpf6_)-Eb7j+oBVD{vFe<+a*EFD zHCi!}>-v(!-wKwB{Bh+zxz0cj#2E3n?mcZPj&#<1Wf)lC8MFc+4W?R@q@{(IWDb^_ z(7xga=DU09(+XCr-Q2P7-o_BYi$Y4iMwm-E3>5(RE@NT#A!Yj#uDaIdvgTLp`l`3b zdGKcj07i=#O8_&Ba88@oK}PuGTgAnOHO7ll_^ii;Vbie|2^p^Le@?t!p@;R_@rr|| zyBwPo4V9p?;#UD=_wJ7J3E|&zblOQRYQGSWE)eC*_`pBn&(&ssPj9(|=TfqkP!emi zi)LBRjtdsJ>8P#Sjd>b6%W`+})l0)iqB?&ib>O;gS>W`G7ZRB_KmwnJd(0q{|2~PwFSE`TKZO)#97AJE` z5w5*YZa|OUQ(@&8h-@o7Uoh6zKlrGX>MkDTO3GTJUGQP_tGcK~5Q11Ok6?hT*|uO0U~vno!qQpV&SmJ#gOMUC*F*wLaW}4{oW1+H;^#q=Mi{De@&p zg^(;#5xp)3Q})7HCoHbsiqz)8QJ7tweQ@R2RmJ=^^aQrei_6WHJPigWUN(Ce_Cr4e| zlFJ!OA1N3P_Mp}eynXGNhLRe0KX>ge$@mG7hK};`sHMnxgM-_>fQlXR@Sd7KdzWUD zoIzX2fDM>T{9@Vw%nE+TV5C;u53(5aeoF^>e!9^JX9Uw``6?nY0{h)@Uej_p!frmy z^nj?k_kkz>`0{uE;Ew`tOuQc~xozipPzYGLZ^6G$x`^1~liYk0uDbSpcbA+kig4lA zvli3wj7kKLr@+a1^_G|Qu!~QqZAsd1uK5Ngn|e(rv1I%MO^!B@%ay>6A5zJ1eb)S53Be8TvVun-eC7- zI8-8D6h&mKL0@TyU*-BDLQnwmy1N z!7!!F7RqiKS2@?&cCQ<&Wu~}_W|Kq#|OL>>%Jip=gWOBO~J|Xj4q5q;KIGK6F1?0UaQ0l();AqFWvw~k64*pFoVg~ z*0Q5G*$bqx>XcggMJJw?u8y`)YZ**rUv>4Kqa`!+wm>SPx!^!MSFIdMmRUIMbxW&x zOB~Ezps@L=IdPq~cIK`R6Vh6z=p)=>V^<$LXTJ7Zdh|qZpA=HvHtmBCbAwl7;8m3V z&{=0C^lt^qe0RqQOlsJ<0fkGsb^sy|iu(?KIQ&e6FnDEow}+#p0RCE20ZQsI0MWP@Y^2rDCQCQ$MY$Gf6;jI1d2`HH zQEciz^U!cmYw{{Fq<+2`h)hxTa&GNgw36{58k;1H zU!hXXTb}P5q>y+${3}OV28(`XV$IDML7A9IGyY8br_xp(E7$AQ8A4$rn_?ErK!%p&N>Ks3;p?-#p&i=Ox@aB=59TB9+9cP|3p z)*>arH0*0Yp*11I4^mF|oR|XFweYXP?{8)6dhq@4g-^Xy2!e$n*-;OOY1I5+8%;G* zp0kl$WyHy3J!B3OeaE!frH`MlQL8Sx3EIWUH$-xwlY0cd#!Y)F-VM@u-amK2;|L@Z z?tYp2q(Mnbw>Huu{hqRl>u<*GI?3+LKZh)UG_I@aoD|;;!OS7oP{x&(E3ciG1RQBPYz#Qa&#EXU~Kl&$RvJ+h>y{UGUH}8Se9Dzz*xbhAihnD&@iZ@w{V@ zSe8e6i88F7$$h>NT~~qpE}*u zc7y+dg0LIP6#(q65+1hSk9)t_(kx#$l`ZU$bu(CPy>CE%V~c)^8NuOKu1{Uz8(0>H zoO`W%Z!CWC*R621O+SJBGxgR9+hXCogc31MSJwSGy1YdwA0R0x#O8 z9<(PD^d;wvZ1pF13O>8foWTaptw}y6ZHyRFjYV8Xe{r8g*!KJy7~xr_+Obw2I?k5` zH~&h^HB%z0?8&y$SS~j(ocJXtLI|h7In^fd>GtRGKvP_(u1;_- ze~JOdkpJw~=XPGv&t}X4B^tgt9jYY9sHfd`4ld|h&fmj}#yZWiIKqD;BY5r>C>*n_ z165%7O|>;asiYi!bAk9=K@E&Olc!fun=2OAl3J2>w1TG9l6@AndE$1n&0LsIHHkhq zROQTVE-#aQdT~qDPIG~6uiYnZw#c&A_p~{F64}|=UDIhj+i9`MOyaVIN&xy=K`D2C z)l^8OEGJ-3Jn>BCIrfZ z&Wq^z3wlYb(9EPm#*FlRnI}lIiTSaGTbq70fz+nKgN5{-bdXPydD&m?wO==d^$DA6 z>GyN$xI^scQGXJ%ZIq)qfs)6SFvO$hoZv7m(90xwFG(~a{~~)X(XiIgYVkAZ0xtzV zd!Uy|0&3D#>04Tq9I0S4W!o8SnfEw>oKJYtqKZEWnwQmyD&Go)jRF_EDDFpZ26ecY_gv%=3Xc9$nH+r!kuu;#ayQxw( znc(KC($b|qi-Df4m#v%wFhhgfUl?>>IfwPi;pz#fa4uw7rv4QF#P@ePuAwKxKpB z>9LR&SGD3~CMnF~;!ONB_ohss5#=U$I$Kci`eeZ0VaUhcEu5od!8I_Ai|C7Ux#-5& zfr~g!w2X~zj6KRmMrMQYNBJoIzKo9^)9CY&(Ak2*_IzY<@5l1)?pK$*!<*4wkm#`) z?YUaPW`t*R?N)^Glp|A#cKoj(KVRf*w(x~2(~Jo$)hUd^bOm6=2|iE-=8t@LU(D2S znf7gf9?9`~bYns_1XF}@wB=&FFh2ndkz2*&u^*1X;{_~wspy%fkMt&z70ons*ipLE z-}fhQ$rInO2g=2o#XNfQpU5S%w2ke!1Y&+ly7GB5HQ+DMdX_D6jlQvr;$kgh8M(!} z#WHfMAJ%1LmunJt=`DQ2Z;g4Ky3MUL=9zSl2M4yWT1G&I?ao1rZ9vCbspE46a$Xc; zxi!A{wlTPdk5S+5P*KO)YWh*uX|_EbpO(cKzuS?~-afih_!jCF>?N67vhaWkKK4M~ zKys`4=*U}xOK9OA zVq2bbbdkN(WkL5EAk{ul!%;nHD9azr*eC0eHNIEP83_@=GPBrWvda&3V@*zQ=O?r9 z*b*iF2GI2H{0W+FkjxE2Q5T(hnEEAT-qEE2tyB_4#V6$f;61r-6Y*$N?RT;3(8grg zS1#)=?BTSo8*2t5{;Y`SN&R0jibO}{tS70^la7YFxUylwe_Sy?t%`;j<&S2?9F<3k zxWlOQZfI-%PoLs9DWm$%#9{o(-|5?Q5B7XDR}6@(h-v7mcLs6N=Ybm#OdLkq^j@|3 zv~m>(M9OKn_u~U<7a{(u_1O_s3%{>zB>y*w?DMJoILez zm!-$V?N4-t;|?9BVMy{OHwG!V+l-CCK*-vIOOpRG-wI-aE8-?$ZAoG+B+vu@m^%aLuj!UbquKwmM~0;ZN8LFnN}6cgW?F+ixeXf+mIfM zex7Uz@Nz(_czkp>=wOauhpTfdYp)Si+EAiQFBg25mkw55N>e1dW{RiI(b^z_(M1$j zz7shUovk)t!l3iRgVpYo^B5flbs7;SmI?MLE zrz>j78_NV_4OtFTXh2Pd!Wr52`%RXU`JJZ;9g4=MX{{NY`I2ux_if~6Z7xU${rg%9 zjt92`o??H`8c{Fgren6ds}^V|jl>7$1}ne@1E;o*8lx;eIO$HT$73*&VYM|5?~wb73?a7+b7 z=qPqFT8zOm_PxjEk|LF(#OE2QSkUV5O4Iv-0v&1Vnl<%Bt&W*vo82yD4TZBz%XhpX zs@j${lmu>FioS^|o6I=7Q^PLAL3YOvcRpGl+iD~aWvL}c^wyUlG z9{Y0IW-d#`daUBZMqMdY4~uoMX}aaZl?1C!T|(&|E*rWd$NASA!xz?t0k=1DDhER7 z>&dPSxju`|1R2h z;FbDA+3^l#HB&jKuXGcO@4a5Ia(Lwg56FOuy15X0p`Y*aQeo?MbgQ9S_Q!G9vDFE;?W_~+3KrpK8fYA>)hro-_!LgoC}!h}RWrE^Zut^D z4@g_b!l43Xg3ssHfZ$h|6P#Q?D`H99b{zj#K$1nf0A#YwPK8+2?{8o?2 zsaa8EOZAx1HJDl{@A6t}PAqqMe1*r`CQmusEghYbkMvyj1{HU;j0bxS?H~7K7{8us z^ks@arcd|#yp+2ZEaI|Y&$ov6@)mAfgoCPH1N!}m`-RahIO8^uh4ZxS^O4V)wNCe( zq#D>k(56OM!$K>!Qz%w%ECXFfzW{BqIZ&qS8`psK3)1X-@0usiLi(r&^$b=YC%i^| zM$c@yc<{D?YjE|PfW4h^UL3m6({>=!A^#3Vxy;i88~tJqvKL7P%+llXS@e#;-^@%p1PHA|Bc&?^D&40@-sY@-oB2kI0e1>rKAYfTk`?AAH!d46Mjcula~9%5hmAlT*=#yaEm*3=R*0dKQi*3qCIVc&)qe(>AQ4 zI}Kk2J(r!j`7srL-9*7=5cTFp{?-#_B|E_OozFr=M-ND%D$RZ;)2b2=o)?+o2HN5* z$ELU^<5l({%#m%_bKa(G1qDun+s8?IknCm7xwKf*#{3WbqO{>~5Br@4mw;@UuB?gv z8!#p~e&s7anDw{i_jCQ$GkY3!F+W){-PnY)z{$hZ;b%R|_oZh0Fen-yjYK4TR?X*3 zOtq~u?NEUyD{zh-T)>ctFhPn>;vQ@q{f+h`npsFZ5=mw1)2GY`33uQuy+~DBaS+Ex zLS!!`dz?Ol5!E6iICYX)uSIJaF!#zOVR^&2IAt!=6`RSve7TF_dM(-BOJf`|(shso+!KxT3ehWL^?G zF(1GVMt9C^@|=0Fo#CyECP%75Bw@HF&?_Nhft{@;(p8$D4jn^84=FfwE-ZRQ#7GX* za`Mv5HnPuwK`Kc&{CL44ShVM#^X+T>t4E)C15hE3V zvzs~9e%_YPQje?}4X-&Ejuwd2QL+Kx7c!~2u}WrHj(E_UkXT`9lqo&2WKXVAeIBcV zyP{%?oqoqjouUxVWN1&$v73U#I=yyuQx6}eChV{x?u8UGA$f15Ak+NXm!FegHm9w} zrxkg?x>NI5LsM2Mft$}+MkBGR=gh+-P7JyVF;B+RbxP!xQz#5uA}$bvUKHgDVH_-`HDftv}vnxzX34z#ss^nM^a2ZGc{?!&!aBUaLueI-g-y6S`bptAjv*<0~PPp zPI8JL{o;^v@U=Z{P4%i;b2ExJxw)&Xz`-^ducqaOBZMcA8LT>l^pq#awi|a@lG8YV zSk+Q^(Cb7a!01=)XP-DDP7zew)S{@QX%}qa^9*}4$$*33@Nh)l^cFy-w(Vt*`($Ax z)|;$oQv`{K`EOsoEnFr7Rs|=0dY4ULJ^cg^Yte9*Epdo9=6w1-~ zvc0mM!Gwx8nN8-cw%G_7(;_*&PcDj+nswA%_M6ccbH_l3%{DaOEmsUn(*wx12>+R_ zC>BGoVX zL!L>z)KrbP5D+?)=`pDf2YOy|c8>TGktBS3JsH6rXJT>h6i(Wh5Z|7wKZFN^p9(uO zd)K{IE0xbKq-M`mu1Jh=DQuP=X{~q}jfYS9{*`@D3M8EsZxVUXf_sT)dr3c5lDmd{ zkbDJQ{MP#uu%i7qs7WD_mZRu=b5;q-H`jTzho2 zUhO&^c$rA$woQVDVe&^c4>^DF*Rf4cr?J(7yK0}4c5x_18J<+`3bFGjTI`b!%8#w) z5FRmq*A*|}DvA%Y67RMg2y)oFi`!QHk|a1@Z^U+7kZA14{O4^QWyGaw1KW;X{{kC~ zD_)Nm<{=hjhkLn~0^?y%2k;SwFa!(7Pu)kg_lb?s^|%7_IOpejB3u?8wKlJiBVrGCv4r@TSoql0#L?Qe?K$+VC!7{L3?!!n_eeF( zt2SL=u|ArF$b}@$GH7q?6uZLdb8C1I1NT1H-L2}1QEDOH0|OfWi9lzyaYSUSS$52V zfMIjYh>YMOG563wPZF~ZlANL99>MsHk3haH?-)q1K@yjH>(5q6P4N<7U(tGA5}Ggg zPZOtPmNfj-7Mb}EX0Fqu4dmt!FUaeF;t%nnRT?B2oX{3-kPxS4){+a2MQ7r85`~EF zQfGM{TP!n5_>+wB<5Ba}b6KIPKqw*O?>>X50IVG9QkPj&T4=^e&>X$nf~+Yg{Q76o z%!@-qIMaHQ?Ua?&2Mab=%SO z&x~ql!Y=F+f!(mw7tTttYYT23*vEO2&xWotu9R6A+Mcky2>2}8%gvlR>Rbrb8D2Y# zl3+aWK=rY}2RXh-pPyh)gLuhs*Kzq%>%E$7K@E^bfeyRq1jmYY<+>B_QX;t_=Xw!F zPkq-2zI_C{UXq$nZgLZR?lxnfrVeKn1z!)nzz7tSGb#Hpz_uU`#wl}MC%hH#ln7-0 zb_UzPO)<2KqK19mW3auKS1w?~j3eM=P-8R@MaoY8-KuN1*w;8YO)U>!MIj+r%YO9} zEbIDp_M3QHukvUYd`qRKMcayg=UNw4elmrvjc^K&>17RPcX2L}o~f_1VWpSLj_Cg0 z1R7nl@ZcWs0p7TdM500w#)EhX0>a2vD@T3-uO62R8f9T|Fn$TPB6G~Ej-iA)P8af3 zfub~c38}hQSCz$U{)E*cSt(9Z8q?QT4Bx|_6s}=3SmkACN5%29tDBIxwiyd63}b09 zq8dhLbIH_-m1g=)+EpE;W*84e&%tKQ8Z;DD^rE=!u#DyJsDd+x&f#b)7VJQYgxiFQ+bXFw6o^W!LiWWwkh(SCUzS(Ou( z*9hw{eCc^)yP#}ELKddQ*(Au~yc(85N|Q$m^XYjH8m z3zt7p9P&Zd% zv%c3?!rck@qBuFl?d0F`P3eNv6;7OT4SNn%B0^_`fE3RU5-fL{&m&s0#K}tY{DS)L z=vTtU!5bXrJkXgL3(#Zlci+LggqGG!rTkx84~|N+s8k~z{$&$- za<`>ljun=2>8Z5hY3s!2gKkWZgI-!^V!9yb3$z;&LM}0^2k8aqu^_fSDk8F_z8y-(PUyWG_$t#5o42sEfYle^b7X| z?vj&BcMjU!q@!Dsaw1BaJEfPL%Q*D`ftdhzZhc60xirw&qYNN>=Js-LsJ*?UP+6aI zr2epkZKZ}bYQ83)yRFGu9=ei`QZk>;#P1Dk^hS1@ZmDEb!5H{JG`h;SS*-OmWvS-1 zv&|#{BhqO<+hCsaFiCw;REqw#B9CBu+I`W-3v}C4vE9@aK8uWHEBGpt?dU{uY^1g` zb>pRGo2-goA!*&x)B+8`Jq6mXc?Pu*GjH;9%{}%cMWG(~Yyuv}vT0a)3j$%F^~RYC zX;r)4v#>p36KP^{xo{g}6rM=TZOvUAl|s%pn2P_w58tWYKYy*4f=hmY_Kpf0HpDlQ z;F5tb5NQIkCc?)zuaF0PLQ3!oE22@W=*jzQ&!j9VW}c^*ecZP`mPuP#N~+E9n;Z1S zg#WnpBG~p%<$t%Cyc$3T1+}De`yy)3YcR9OvCJ$QL^_ny%W07%M2IK}aTn zm=h2o_7H9?u13@`zzB5>yKe>G8Zmb0;Y7Qs9;e~VTW_sB$ZXwxu@a1j?I;R1qwS}( zX60?x|9LOAJkP)65gMiZ#3zC2z5SAWO*BrY4}r!(iGE0HlU8#2h)9Y#Wa^}O#KqQ{ z?-G>^5vxewyo^0BdZ5@_ZaOPL(an4^yk^Nw1PrTWia};KMds1QyoD^g13&HRg+`y* z+-$DtvB|I7XRkZeUo|yQkJK*=eo5TWhGzE`fiTUnZ|*JgT0<3Xf-jq2;F<|x~RFF zkft+lcr&6k+(Yv68B|Q=(yL`ekr=ui5e;Gc3?EGO;T;=cicw5~ygS!Ph z8~^>VsL!&iNCjzhL;CyXlqEWTbIFgw?n`LJ$7yho3=HkK{Psz!{$i)<4Qpm_-6{a)BE_nX-P7EoDZe_@il)m7~)uNBqt8N$OC2UpJsa`m2c~ zMvG*fOZ;iw=pknT;IZ}=1it!<~B4c6Ow!S23c$%l62~ zx~?Qa5xXaq+yG{-a$>5@Ru&(WrGfmJCtn9JT=OVId>U~Umq3HJm}6HkSe9K6o|czW zI$qbdZ0z2~$mm~^FJG6BoL6soSzqE>B~z0Pl?)2u81(-vI>%b88KgnD6?wFEhaE#+ z;Z|e6lj%?e;el{&+N{}plo;7)W^EnH^!K{V+qL%$WB1N9H1YWQZN;!3S(6H?XpQFV z5-VXX7=K(j>DQ4BdF5=iq)gGewb~dcra_n&k#XJGo9CT*EBVIIcnhGMqy9pg z6|EW*e)htx9z{@T4(S>7_TcYuJ*D)k5%fMl)~tACtsA2BMZ?wKr*wMFRrlUmquLC) z4UM^kdMiQ`n;Fu?0Bi>>-{4A)#Ji?^hu$0tbgZ2__yNvM4`Vf&rt^xlw6Z%_n@+}B z@<{265Rl%;ES=yqt)$VDa0&}929n!!8gM~{b{M^Ra22C%1a}|@i)GTSXWL0wQ?ULd zxXOSh*O+{IPd~W9flp4vL$T>xB+L>)-e?mOt`0Q*ke3M)IS{eyMEbRp=qf4mgI9St zAOA3FB_L_i3KTWdBA7N2xfyaK@x`%%?UX&usXLp8A`SOh`dn`-KT)jc0qo#nkY?B(;5^hXz+tvnnDuTI z+}bYf1a*faE0(S!kJuB53UUh4zG`bSh%sE5y3;cktN9vMmb59M1DY60MBkw!MHbf> zZAz)hSqatpIaGg*C+pgA!g;mv1dtl75~e^8);5C55?8dPhw@*aynKVBjClFU6xl*y zG#IYD<)Y^isNQLWx$A;NPzQtstr)SBGG!pZQ+G~9%>yHEN=uO&Re!~;fuR$%a^miq zMO)Hr^;23NfaO?ic~eF@ zY}mnazei^RQBkg*Misb|$v?h-NB3Bc{B;-UV=$6;zv=$!6NW<-##ZkA_!N>6T-4LH z_DqaZH>m&R{8Zau=@0wt?KOoVZpZC0e|tK`2wv!xyC4lqq#~CcM*ac%oQ)9XGO#$` zF$vzzwjRMk+LU+3hqq!%yE zbT@>!%JlL)0UeAutqB>!wOG4Qm*-ioA!iQ~!^PQ8TUZ>dXDv$ci$D7z^2Zvgf{W@& zyd-dy*+(T!(%WL0j(BCTirZ^-D#8_oJ0WX{5x@{uI1K^G|WXD;$>Z5R?Oef#$+L-&Rp|)us2Cov$L7Y$BJFo z=37uRPf?q`%GlhI+Nwi%U7>})%wZYwf|oy|AWc7^%bkWM$;D&g{1+hlrsyt81F>!oF2kFl14g z4_rUj@C>skqHSKD-35;{ua;nwpvSlaCObPqme z#uC|oS*5jHQ8LWbODZqSN(I8-y?t(Xj>S6q2+8XrXvUn)&4;u&q;GmF#~=$1qMt7{ zUwZ=%O-q;&AL03IyvScmu%(Sc>moGFQhl7>Dxd8y*Qh-`}d`zh@lr}Qq3 zDtkma9g+)R%~3$=WW#Dxw9pkaiv4TQRQT=9I`+M+*2AL*ws$D{Hm+La z>UH7pViHjYzx;bE%4Fv2v2*671(bYbfTJKcLhFWQsR*ng`DHtT!>k%~(_!CY2)-@$ z`AKLy2)a&KLapa>!M3s*waY{8BWET`W|LcuSCwAescyGhLjRcP&ZE}=XOa6WwarRg z*SrGY4(8wqlIs`qR9Ce zOQN=3qe9)F#o&K>&C;E#o1II_{Lo5$%E{tytiVZ+;APIU8{~Zy(iHcUy;H2xZ%0L) zfX^kNEw~1%!ZLg90g?EvDmGk%d7DIkXXU-M1TbeR-bfkqJ)%hE>en7viH7!TFM9Q} zsq+nOCX+BU4%cs51n@h~(iWU~w~0!2>iX(jb|gM{)=cL*`l#Dy56o<+u1G){{r)U`Vd`kl8da zXBWei`L@_|)fA8Z`L! z#nMm)Y#=xV@c&osv#C~ftuQo#a=9T*K|hVm+>ORPcalXYACujjH{x)(h9}yJh`F)f z1Tv6}_4Azd+;}%&T_~)yy3M8PEa`vW-j36ZG{zegfwjS|YG__QfqnQ$Nw+sxIZ}M2 z*sxpOfRJRK6%Lzh9M6J)@vk>K#-80%2{EPWqHK5Ey|sHALQ&XF5#rIisjX1bJx~eQ zNRd9D8!go3K(dM>dEsWqn_yf1G_q*U&FyaJ6Ry;ul}gc8G>t8(+snZh34d|^{^;e! zW14MD7774=YR#f^fJ@BuM$RB(5(v*I7cw+ymX)`<=WRpOnSm{@VcEIU^NzC6$OM6_bs&0rH%%jbLW;9cmR3e7t7h5kvQ(RC4pr>n?An1#( z=%c91f)_zbA4!;^DQ&yiv;`-x@VM?izf(%OCC`2{Y66s>y9@E7fxL52UQ8|Ry}ESQ zF2r@;F+7`Q$?n6W)ww9lDOe(kXf1KPg#^i9Lgb%==CR|50NiO!84V4N2lb$DI6t7<3Vp@F}+@d4x} ze-pq3oSid!b&tJ1e89J|&|n)vcX&vid9VnB(HKfZUl?_KYlRGiq@WPlj9utp%H{5( z{*2WZj|Y&}h=7o*_Qj&B4a2T(iqv4!_Vi3g%}}If-Mn54cTb)0w{*nx#u|BAHl73B-tDyDZ)uVFDMmh35cK z%I&M^t7?uRcw|eFXfM(}x1;)+e?TkLjuBUtav(`xSt@P^X`YdpJKmXU&d7WtTkb{0 ziUH|^Vno0x(ev~yBmpQZc&*T!RKZF~FFaNncelK0P5u*mx~r^J-(uZIcV3u)@xr|h zZ>J}>Q(*G?0s_rS`VZoWIk8L`+UYj#Uy)kCtZ%DcYnjuQb&R1@YUDLITCf1ca6 zT5VQeKpfGQc%{`VN}FmNrlsFYi4qs~To8rZjxzrtpwMQy+mqhLO_A%%g81x+Z+qZL z|NGCsM&9jReoEhp8F^6yIxtVpmyX*OeGGgmQ7w`TxR8i;O!cH#IKlJc%SnOM!l)dQ z2Py{cOmvQkGzI*ouU2o?6j}g54M9#5XD=xP3cFcJ(O(>K=_Sp|d<^WAK?d0z<;Pe3 zZ!>c&Zj;HsD{eay+M~>0S=j~zs+yOAS5q`k6&@u+Zn5CR!Zx7fIgZupeY-G=Vq|2p z+#2r}j$HWIBUyrW;?#w1K$0^hEO=lwYh&@t=$0D`1KW^$m+ZNq2jVz5RO^$&_ z>&|+6F#g1eVonp`kyYYkbo>Sm0hg;XNmqLjD%~w3TyrP$i0`-xwm@>n%>3t@*+71a zc3mcgX`1I~-}L&EDdb4eSYiHNX`^aieE4=r>XI_;cw8M`Bty6Ww)hjkt_W4At?V7n zQqQPLDRHLU!Q-gGv53BXD8M(u`^OG>ZH^TnV4^#*NP&E`aEHa)E;gqaPJrilL&UMw zR`C+3%$#5IoQ&-OHCZ=?g&rjWVu0!(1&%HUQim!23(5IDghgicvnyRY@reDhOjPU! z=N@1c>cyT>S-$QLA41wl)-@wcMsj2_0Y5c$@5a|U+%k{j@y?LfN3i=`>~hx&5eCO98WsnE z%E(Bb1?RyMJs_ici^cy8nu05^HYB7u0h*_vrzHJtEJ=7P#s`G8+Ti{}JVsr$Kf$!g zY#M2^-;g5OheiZhCIv!?(B9}8==XyB<%o{mbS`rZH)1R}JYufx>zY+{Ngwyl_1F=` zCJBDY037tsqZNO8-}!zuoj8jfSdDQoCM{(Y%c)h9vHUCeWu*Kp8 z^%u!A>Cu;G!=&z9>_eIG~OqW?$y&bzIlXn^qX)rT5ydpWtzMY65Pgs(q z%iEO^0$)QNd7pi)*a3-r!TBD{Z9`gh%%H{vnn$cz3wY<1v)jpndx>R0%OL&8Fn zvr}UqIWyE9Uw)WjIhnXhMTvPr*52$3YVA@jCLevI&P(5?PdcS-GKX=B96lFlLHtHx zu7O&Qd&qK1ep41Z_WNuxmn@UVj-FV9wZ?l8LDR@QHrafDCq&eIFCqpgmWag@$UJ#b zub)co@=n891%vB$mJ^9?+ZywcpYfFDJ+TYOf*^A7O3ti;;-a6-!+@Sjxji+xubY3` zLcu@VQbLA9$vVX??`q1Krgb^1(c`Vr^|j)6@uSGm)J<^!^_#0H^z#H*J_g6)0aycM zq|}OAa-3Duev4!nU&0kxE_iL&$^8bSGc=6%I2$d+0(0?F&>-$|QJlM3b-A~ww_lSZ z!_;W^u=eNPxvd}ny$);Q___oBg#fJy*cpE%pV?h;K#v>RK>h_~Bg+cjQGVvSO8rTh z-^#-IuseUn02$|KSPy3rrl*$#>n5rgU9?{srKo*iYERNOW;xK^3H>-^iXILt3fGvbmE`4T=+Z zn|{HkBFFZ=vElGlD!vVTGG02r;rJpx_#ZP{b<8A$q%Uno;MEpw4fY8n_4WzDXVGJy z3%zfD=Fg(w3hNf<$|4K!AWx%L0vSz{63{>nWGn;!UxPg|DkCOkc$ctNRS91#dIx_VIPB7?G zlCTlL{*W;n< z%E|?$Y8i?Sqk~%ag=rC$hlDg^m-A~73HwF>bfmK1JS3 zmxTNoEn1hz2Pev{q!&;h@^;^g8_F6cc5*$J*z%i z^f?8$dltQzi1)e5s&ttf5rckhXOd@@zUG6lQoeAwKqr!KA^MyO@MOEy94zC52*810 zu|{rXyb%rV;5gc;BRbkmkMTCe8QeL9w2D%8{HYmg-A8&ZdwtkAM%osSd?Is5`4*NZ zGtn|6ZJ$`*A!BqqN@9QU1w89C{_wl78B{DAVLSCkZ*ru|W8V6pq5*;YJMHB;-GYXoZU(O{+1@BJKTuoYD(eMq4Gbls0&)>g){qp6D&�a9ypmO@;!C0P2o-z z{EC7S^S+9bWC^%aQ&g`}%TP{XQV@nqds;<;tnC6i{v9IkMYfViNJ*MNj=$&l0Yl{o z|1+!6N3G4>M$>5POTrB~ci^c6AF2)9E3{rtEl$Y##{3d7%!$;r*Hmy2m`mIrNDy2{ zO^H4IJOkQkc{$RdvX zuiyO1AIkLiy3E_;Yx3n=bo9%&>vh^vURM!#{W;>^SFVXB^PXWGcX>mv!q3Ui|N3Py zi6$@2EQ6*1EP;bZI#8;inN3l(-L+jlq$k~*&wrEwHG24!eJ!ycy%bXF9kOQ`pIB2! z7m~uCbJFv95~Zs1+hQ4zb1eRGbB>wQetH3uKpLS&4arbjnvJ;Z@8a) zJj9By{&u$ll@OwBlU!laIdPYi7p3aBmJ(C0mSD?oqq+h0;Um^2 zxEwykTJ9;dT}A}SXQ7|=h;kQ)DC`KYuw{wzM+5L{1x10A0FJZGi6WIlyLR}VAsm{} z`aZ66)tQZPGdu6P7R5mVT~xq+3QS~1rrRSNfXAVM`YypbwV;L_ux)7Q!r(5H&c)~@jS=m(!Q*yV$cs}kJGt2E9>Keo)H3G)!JQt9o<`#e^e z=Vy@|JBSanSZR~w<;ljf5+)5Tb);$bCZJ?F8ptQj5TA9@P-ULJ1$?H_4oj29;c9ad zl^;%eTndQGv94J@aDVsOSt=KLeJ&s&5n=5q)UY#h%!r;i7$y-Ka2V8<8x0`O8B%8! zHE+Y8`EkgO3rji?`N5ftbrQeUk*G&Xjlj{-8XK{?&9*0l;;K<_9C#SsXcO71>7je{KII z2v5;Pj*n86Yl+vglKN<`ZVW5KDWZ#grK$_nmPqAM!VgC0LQMtkker#ISBF1|_vsXX zfh{}8UGlx#gal*tC`^5*5iNA*Yn>UhelKgYUK)1JO zMOE*GTQIV!HiQQrQ$!+LMtf?58RMx(e;#dEH+i7w8*<^exPh#OE1b3|L+V_#4mtW< znm%wQzp*3^jW=Ffn65@$Ktxo5wYAlbp%4Usjq4b|0z!gZ#qc#!(9aJI&H)wILi_(bvnY@Hozg0T*lU&*&en%%xGURw{j?L*%i5h|I4 zy5~1*nDnSCK0SB@q_uS#dp?V71q4goU$ZVIJdm6vIwDEt0&E$EYfCkkU76x*PTPg8~dmdR6}BX@3v=bbon;z z`3g{&n-dySt`!uBc|o=aa}D7?k)|6@#M%WKaFt3&;~8x(Y0Zukeb|Q6p51@=2J;#c zXw(f^QtQlWW7mzuJpBnr2RJfKRBLp^Tu7!0=j1 ztvRPHGdiGx!b|IzBpA5-lDdt>m3aE~nG7ttr=8(uFAY0LJ~lpwAWpRR8c*=X*~MSdOVPv=Ci)vAfLFb6m6Nub0f1|1r$V?l^5}|rhk)^cL50c*vT-TrSFC8> zeBJ)k^$r1>Hc?KuGgT?c^V=NS*WY^;&${zkNzep&vC1gDuOTVi9*wG2K~JkN zfLxc%X?;HJ;>G%(si79igwl3s>Z%Q(sjQzuD5{PqP{?a;#g!Hwl?$>|f^7E1Z3Eyt z%!IuXW4O=TqM9<4X(Sp^nZ~i6{W9RoyF!6`a(Q(@7}73|@^(>4*2!|5y?J8EXJ&=Q zQpyh3=FyraUI#|o`7=r)*Dw{gw(QWh(Pr$*9l(X=Die;l+X<`6j znF2&w+5Sd<`0?fI*I&NAa1Lem@0Twxq6^`_r1pUSQ!Byb3v7(#*$c7Tzh5A%;a7Pn Xs+V7X{QEcex4-@W_>Z_M8m|HX4_n-< literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..71b437d69c0b7995b59b0ccd9f9190248dbff64a GIT binary patch literal 589 zcmV-T0yn#_xKHy|tOa{~%2pbvUG@onhbrZ89= z6I(J`IUzhfE2q#54^SUu*_L*H-u*rS0Nix0-WuD$UYQW@8|PtTI)wNEoP}yMod#_@ z*1`FM;7zx#A50flYZX*Ebjl*^^0Y{EIM4b+e4DKvS6{x4=nqMy>MdHGrF-YM!6Q=U zu~LM|<$Rg+%Gm5B&MYQ?dt(i?s9_zD!WsteCaCYDbwe<{`HlJq1mznH=QIOjq39vc zY1$1BqqPxlv?jy=1mx?iGga#8g^)BQ6c{c!2Ff&6W5eS>CqT)3@$TZPDi&|6%SE-& z#Y_nQ9KV1VUBm&1tD_n$(Kf~+Nl9A_bzAC6N@F$tl5#?k(&D~s(4tqtI9q!BIdXNW z-0;wrOcb!>Hk6~STV)&6rN#{xB(8=uP2qm^@$=mWAvVf%BX5#IBz(OV0tM~H05PQy zq4lUVoV`x&GdRCmF6a4t27|JOb)`Xt6B%Z)(f_ez|4_1tB zqNn(C$DGiQ+)~xC2^a?yOmHx?2QszlfY2KxVO>B8tOnSd(8BuR+v8Jtv%3BC^su_U zdta;p?ap{-d$a*|${SU&ii2Z65%V56a&$-Vsz!0z(5T|qDXBQhmsgHmkk8smA?E@U zp}uC3N31x@!8F$S{|Q*Qso-frUU>0RQVp% literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..4365115a7d3e3547a52a76df5770cc24debb688a GIT binary patch literal 626 zcmV-&0*(D2iwFP!00002|E*L_Z`&{oz3W#HzO(^qJMKr?xY!}>T42}?ThpC}!I3Q{ z(PB%UL?_6G{r6E$){k2U#0Ocn$j8S=QUCxqwXIg#EMOxwh4q+>q-Z6Isl;?(!v@X#3(=mlq=N-x`=fk{y@aY?#l^0X}&v_kI-JjKNMXXak{ z*b|eEcL7p`Ww+q9eG^uUSY>KHJ|A6-hG%6t90=j|gtDHJ2n(pr$+E`6cTFSR6GWc3 z=)CrX>gmX_y>#S@ji#mTsHpWC`LDwT8#<=J{g5OH+)qAyx_d9gT0zXILHwkUyU!bv+)30X*Bnc0!c4gF8!bK=BHPq{2~|1alb*(2UQNg zqu^5`VU*8h?eQc+=d6qJXu3kGmQx-r1cyI($T)rmcFxN`Ck4#C?Z`(bkv6<3s@i%~ zQBJO7EKZU-&tN;yKf+kT9Jm`GBo4EOuhYlkW^()S@nLd%_b!_OnzeS;G-wFc(&?Zm zkmWh|ob(nXgmGetDikm49NPZC1rWPFWz(*veBWCM47nVhle}gjb~;}3KfN%lRKIOd z`mWz+m*L7HM(vfl=CA!+hk;E-=U#|jn!KNjJ={sim#MHyArj2u_T@zr$GzoCD~T&1EK>#0TlW4-Xj3Oqp>D*sG%ebRKn<8V|s*dlXWAR!Tdt~9Z2~W zX%^;SEG(1tDhP;YH(6_J3zIbz-vR%n+xI3{WFN&MGO$gdDUWpwlx-mlwprlcZIH`G*;tJ zDUVR3wD?@KXwfS&&K4ejOvYm<8V+5-hzN_u`C_tlr)-P5(AY2^u^N&jfyeE~&krAj zXq4$Eek6rh@VOL(K-x_NaikE`c~ly%E;IWIX1AO5^=dPRQCU;(*&&MHpIT`Y`adk` zUnuDjNjf;qMvymImdlM43#Bpmx0$))7izz2VtV2j%YTm#53}Ls$DD~;0`DIoUxCHHgXYhI^5$hpmIoowd*|hGCC&}s eL?S#!`r<4o7vn3f*g?FKrT7iro@e&(0{{RIap*|^ literal 0 HcmV?d00001 From b013c915698fb0646edf1ce74549bb9f2da800a6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:19:25 +0000 Subject: [PATCH 05/25] style: satisfy workspace formatting after the compiler rename --- control_plane/src/physical/compiler.rs | 4 +++- control_plane/src/physical/realization.rs | 4 ++-- .../asap_query_engine/post_asap_readout.rs | 8 ++++++-- .../tests/support/issue_701_702_process.rs | 17 +++++++++-------- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 29ea07b08..5076bc06b 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -7822,7 +7822,9 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = DeploymentPlanCompiler.compile_promql(workload, env).unwrap(); + let bundle = DeploymentPlanCompiler + .compile_promql(workload, env) + .unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 2); } diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs index 71426ad69..0f16ebf30 100644 --- a/control_plane/src/physical/realization.rs +++ b/control_plane/src/physical/realization.rs @@ -3,8 +3,8 @@ //! Providers may validate and price physical implementations, never rewrite //! selected logical roots or infer a pane width from a query's slide. use super::compiler::{ - CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, - DeploymentPlanCompiler, QueryCompilationInput, + CompileError, CompiledPhysicalPlan, DeploymentPlanCompiler, PhysicalCompilationRequest, + PhysicalDeploymentContext, QueryCompilationInput, }; use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; use asap_aware_mapping::cost_model::Cost; diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index a0327fb8e..0254b0254 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1077,7 +1077,9 @@ mod tests { #[test] fn compiled_window_schedules_execute_exact_ranges() { use crate::precompute_engine::window_manager::WindowManager; - use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; + use control_plane::physical::compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, + }; for evaluation_secs in [20, 45, 60, 120, 90] { for phase_ms in [0, 5_000] { for full in [false, true] { @@ -1176,7 +1178,9 @@ mod tests { // Compile the two readouts, store one pane series, and execute the actual ratio. #[test] fn compiled_shared_sum_panes_preserve_each_lookback() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; + use control_plane::physical::compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, + }; let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) diff --git a/data_plane/tests/support/issue_701_702_process.rs b/data_plane/tests/support/issue_701_702_process.rs index d39c3f16a..f2dbe6b6c 100644 --- a/data_plane/tests/support/issue_701_702_process.rs +++ b/data_plane/tests/support/issue_701_702_process.rs @@ -152,14 +152,15 @@ async fn run_warm_workload(queries: Vec<(String, u64, u64)>) { let quotes = candidates .into_iter() .filter_map(|candidate| { - let plan = - match DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) { - Ok(plan) => plan, - Err(error) => { - errors.push(error.to_string()); - return None; - } - }; + let plan = match DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + { + Ok(plan) => plan, + Err(error) => { + errors.push(error.to_string()); + return None; + } + }; let warm = fully_warm(&plan); found |= warm; From d8bf03e23971d5bf757d7125d8b5075a015b9936 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 17:14:48 +0000 Subject: [PATCH 06/25] feat(sds): separate bound output routing from persisted semantic identity --- Cargo.toml | 8 +- .../examples/audit_clickhouse_corpus.rs | 2 +- control_plane/src/clickhouse.rs | 31 +- control_plane/src/physical/compiler.rs | 72 +++-- control_plane/src/physical/erp.rs | 18 +- .../src/physical/executable_binding.rs | 6 +- control_plane/src/query_plan.rs | 21 +- control_plane/src/query_plan/residual.rs | 79 +++--- crates/asap_types/src/aggregation_config.rs | 8 + crates/asap_types/src/derived_input.rs | 22 +- crates/asap_types/src/erp_observation.rs | 10 +- crates/asap_types/src/executable_plan.rs | 4 +- crates/asap_types/src/lib.rs | 2 + crates/asap_types/src/plan_publication.rs | 2 +- crates/asap_types/src/policy_fingerprint.rs | 63 +--- crates/asap_types/src/precompute_plan.rs | 37 ++- .../asap_types/src/precompute_plan/catalog.rs | 71 ++++- crates/asap_types/src/producer_plan.rs | 12 +- crates/asap_types/src/query_plan.rs | 43 ++- crates/asap_types/src/sds.rs | 130 ++++----- crates/asap_types/src/semantic_fragment.rs | 2 + crates/asap_types/src/summary_catalog.rs | 176 ++++++++++-- crates/asap_types/src/summary_semantics.rs | 129 +++++++++ .../examples/audit_clickhouse_fallback.rs | 2 +- data_plane/src/drivers/ingest/otel.rs | 71 +++-- .../drivers/ingest/prometheus_remote_write.rs | 25 +- data_plane/src/drivers/query/servers/http.rs | 6 +- data_plane/src/main.rs | 2 +- .../coordination_checkpoint.rs | 4 +- .../src/precompute_engine/erp_observer.rs | 16 +- .../src/precompute_engine/frame_lineage.rs | 2 +- .../precompute_engine/maintenance_runtime.rs | 122 ++++---- .../multisource_coordinator.rs | 30 +- .../src/precompute_engine/output_sink.rs | 4 +- .../src/precompute_engine/subdag_scheduler.rs | 13 +- data_plane/src/precompute_engine/worker.rs | 4 +- .../accelerator.rs | 6 +- .../asap_query_engine/catalog_resolver.rs | 18 +- .../asap_query_engine/exact_subqueries.rs | 8 +- .../asap_query_engine/live_serve.rs | 8 +- .../asap_query_engine/post_asap_readout.rs | 44 ++- .../asap_query_engine/summary_executor.rs | 27 +- .../asap_query_engine/test_plan.rs | 41 ++- .../sketch_db/index/admission.rs | 68 ++--- .../sketch_db/index/maintenance.rs | 32 +-- .../storage_engines/sketch_db/index/mod.rs | 268 +++++++++++++----- .../sketch_db/lifecycle/eviction.rs | 2 + .../sketch_db/persistence/immutable_output.rs | 1 + .../sketch_db/persistence/metadata.rs | 102 ++++++- .../src/storage_engines/sketch_db/sds.rs | 72 ++++- .../types/hot_reload_config.rs | 2 +- .../tests/test_utilities/engine_factories.rs | 16 ++ .../tests/promql_differential_process_e2e.rs | 222 +++++++++++++-- .../tests/support/durable_summary_process.rs | 5 +- data_plane/tests/support/physical_fixture.rs | 7 +- .../catalog-physical-plan-runtime.md | 25 +- 56 files changed, 1555 insertions(+), 668 deletions(-) create mode 100644 crates/asap_types/src/semantic_fragment.rs create mode 100644 crates/asap_types/src/summary_semantics.rs diff --git a/Cargo.toml b/Cargo.toml index 5bce59217..b5592550e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index ceaa441f2..d7488a927 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -69,7 +69,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { ) .unwrap(); let reference = sds.reference().unwrap(); - precompute_plan.summary_catalog = Some(reference.clone()); + precompute_plan.bind_catalog(&sds).unwrap(); transmission_plan.summary_catalog = Some(reference); ClickHouseSqlWorkload { summary_catalog: sds, diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f22bd867b..140fb5b3e 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -268,7 +268,7 @@ pub async fn compile_automatic_clickhouse_workload( ) .then_some(config.slide_interval.saturating_mul(1_000)), materialization: config.policy_fingerprint().into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( config.policy_fingerprint().into(), ), output_grouping: PhysicalGrouping::Reduce(config.grouping_labels.names()), @@ -307,11 +307,10 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut precompute = PrecomputePlan::build_backend_local(request.envelope.clone(), configs) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - precompute.summary_catalog = Some( - sds.reference() - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, - ); precompute.executable_dags = installed_dags; + precompute + .bind_catalog(&sds) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut transmission = crate::physical::compiler::build_transmission_plan( request.envelope.clone(), &precompute, @@ -319,7 +318,7 @@ pub async fn compile_automatic_clickhouse_workload( ) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; transmission.summary_catalog = precompute.summary_catalog.clone(); - let publication = crate::physical::publication::PhysicalPlanPublication { + let mut publication = crate::physical::publication::PhysicalPlanPublication { summary_catalog: sds, precompute_plan: precompute, collector_plans: Vec::new(), @@ -336,6 +335,10 @@ pub async fn compile_automatic_clickhouse_workload( entries, }, }; + publication + .query_plan + .bind_catalog(&publication.summary_catalog) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; publication .validate() .map_err(ClickHousePlanningError::Lower)?; @@ -459,7 +462,7 @@ pub async fn compile_clickhouse_workload( } let mut precompute_plan = request.precompute_plan.clone(); precompute_plan.executable_dags = installed_dags; - let publication = crate::physical::publication::PhysicalPlanPublication { + let mut publication = crate::physical::publication::PhysicalPlanPublication { summary_catalog: request.summary_catalog.clone(), precompute_plan, collector_plans: Vec::new(), @@ -476,6 +479,10 @@ pub async fn compile_clickhouse_workload( entries, }, }; + publication + .query_plan + .bind_catalog(&publication.summary_catalog) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; publication .validate() .map_err(ClickHousePlanningError::Lower)?; @@ -631,7 +638,7 @@ fn bind_selected_node( ) .then_some(selected.slide_interval.saturating_mul(1_000)), materialization: selected.policy_fingerprint().into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( selected.policy_fingerprint().into(), ), output_grouping: PhysicalGrouping::Reduce(selected.grouping_labels.names()), @@ -1427,7 +1434,7 @@ mod tests { }; let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - precompute.summary_catalog = Some(sds.reference().unwrap()); + precompute.bind_catalog(&sds).unwrap(); let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -1745,8 +1752,10 @@ mod tests { let envelope = request.precompute_plan.envelope.clone(); request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - request.precompute_plan.summary_catalog = - Some(request.summary_catalog.reference().unwrap()); + request + .precompute_plan + .bind_catalog(&request.summary_catalog) + .unwrap(); request.transmission_plan = crate::physical::compiler::build_transmission_plan( envelope, &request.precompute_plan, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 5076bc06b..f5af7a460 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -472,7 +472,7 @@ pub struct CompiledPhysicalPlan { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationLifecycleEstimate { - pub materialization: asap_types::sds::SummaryDefinitionId, + pub materialization: asap_types::sds::StoredOutputId, pub consumer_query_ids: Vec, #[serde(rename = "window_implementation_id")] pub window_realization_id: String, @@ -1470,6 +1470,25 @@ impl DeploymentPlanCompiler { })?, ); } + let compiled_dag = executable_dags[query_index].as_ref().expect("selected DAG"); + let semantic_root = + compiled_dag + .node_ids + .node_id(&selected.node) + .ok_or_else(|| CompileError::Query { + query_id: query.query_id.clone(), + reason: "persisted semantic root is absent".into(), + })?; + runtime_materialization.semantic_fragment = Some( + asap_types::semantic_fragment::SemanticFragment::from_stored_output( + &compiled_dag.dag, + semantic_root, + ) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, + })?, + ); let materialization = runtime_materialization.policy_fingerprint(); let consumer_query_ids = state_consumers .iter() @@ -1623,6 +1642,17 @@ impl DeploymentPlanCompiler { // summary. PrecomputePlan is keyed by physical identity, not query ID. let mut materializations_by_fingerprint = BTreeMap::new(); for materialization in compiled_materializations { + if materializations_by_fingerprint + .get(&materialization.policy_fingerprint()) + .is_some_and(|old: &asap_types::PrecomputeMaterialization| { + old.semantic_fragment != materialization.semantic_fragment + }) + { + return Err(CompileError::Query { + query_id: "shared-output".into(), + reason: "one deployed output cannot have different semantic definitions".into(), + }); + } materializations_by_fingerprint .entry(materialization.policy_fingerprint()) .or_insert(materialization); @@ -1713,7 +1743,7 @@ impl DeploymentPlanCompiler { .then_some(materialization.slide_interval.saturating_mul(1_000)), readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint.into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output(fingerprint.into()), output_grouping: PhysicalGrouping::Reduce( materialization.grouping_labels.names(), ), @@ -2029,7 +2059,7 @@ impl DeploymentPlanCompiler { reason: error.to_string(), })?; } - query_plan.validate_against_catalog(&summary_catalog)?; + query_plan.bind_catalog(&summary_catalog)?; let storage_routing = crate::emit::backend_wire::storage_routing_document( crate::emit::backend_wire::DEFAULT_TENANT, &routed_algorithms.into_iter().collect::>(), @@ -4367,7 +4397,7 @@ pub(crate) mod tests { raw.ingest.endpoint_path = "/api/v1/write".into(); raw.ingest.timestamp_unit = TimestampUnit::UnixMilliseconds; raw.ingest.require_plan_identity = false; - raw.ingest.require_summary_definition_identity = false; + raw.ingest.require_stored_output_identity = false; raw.ingest.require_registered_producer = false; raw.producers.clear(); raw.bind_catalog(&catalog).unwrap(); @@ -5364,7 +5394,7 @@ pub(crate) mod tests { .compile_promql(with_evidence, backend) .unwrap(); assert!( - !plan.summary_catalog.definitions.is_empty(), + !plan.summary_catalog.outputs.is_empty(), "measured exact-composition evidence must expose the rate child as a SummaryStore binding" ); } @@ -5399,7 +5429,7 @@ pub(crate) mod tests { // ExactComposition candidate. The absence of evidence must therefore // leave that direct legal path intact rather than inventing a composed // cost or forcing an exact fallback. - assert!(!plan.summary_catalog.definitions.is_empty()); + assert!(!plan.summary_catalog.outputs.is_empty()); let entry = plan .query_plan .entries @@ -5609,7 +5639,7 @@ pub(crate) mod tests { .compile_promql(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.summary_catalog.definitions.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); let bindings = bundle @@ -5652,7 +5682,7 @@ pub(crate) mod tests { let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); - assert_eq!(bundle.summary_catalog.definitions.len(), 2); + assert_eq!(bundle.summary_catalog.outputs.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { assert_eq!(collector.materializations.len(), 2); @@ -5924,7 +5954,7 @@ pub(crate) mod tests { let actual = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.definitions[&binding.materialization]; + let identity = &plan.summary_catalog.outputs[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -6823,7 +6853,7 @@ pub(crate) mod tests { let bound = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.definitions[&binding.materialization]; + let identity = &plan.summary_catalog.outputs[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -7003,7 +7033,7 @@ pub(crate) mod tests { .entries .values() .flat_map(|entry| entry.materialization_bindings()) - .map(|binding| binding.stored_output_reference) + .map(|binding| binding.stored_output_reference.clone()) .collect::>(); assert_eq!(stored_outputs.len(), 2); assert_eq!(stored_outputs[0], stored_outputs[1]); @@ -7127,7 +7157,7 @@ pub(crate) mod tests { assert_eq!( bundle .summary_catalog - .definitions + .outputs .keys() .cloned() .collect::>(), @@ -7178,7 +7208,7 @@ pub(crate) mod tests { bundle.transmission_plan.validate_frame(&wrong_version), Err(TransmissionPlanError::InvalidFrame(_)) )); - assert_eq!(bundle.summary_catalog.definitions.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!( bundle .query_plan @@ -7270,7 +7300,7 @@ pub(crate) mod tests { endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }; envelope_plan.producers.clear(); @@ -7429,8 +7459,8 @@ pub(crate) mod tests { bundle.precompute_plan.schemas[0].window.pane_origin_ms, Some(7_000) ); - assert!(bundle.summary_catalog.definitions.contains_key( - &asap_types::sds::SummaryDefinitionId::from(config.policy_fingerprint()) + assert!(bundle.summary_catalog.outputs.contains_key( + &asap_types::sds::StoredOutputId::from(config.policy_fingerprint()) )); assert_eq!( bundle @@ -7470,7 +7500,7 @@ pub(crate) mod tests { let compiled = DeploymentPlanCompiler.compile_promql(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); - assert_eq!(plan.summary_catalog.definitions.len(), 1, "{promql}"); + assert_eq!(plan.summary_catalog.outputs.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); assert!(plan.collector_plans.is_empty(), "{promql}"); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -7597,7 +7627,7 @@ pub(crate) mod tests { .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 4); - assert_eq!(bundle.summary_catalog.definitions.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); assert_eq!(bundle.precompute_plan.producers.len(), 2); @@ -7639,7 +7669,7 @@ pub(crate) mod tests { let bundle = DeploymentPlanCompiler .compile_promql(compilation_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); - assert_eq!(bundle.summary_catalog.definitions.len(), 2); + assert_eq!(bundle.summary_catalog.outputs.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!( bundle @@ -7671,7 +7701,7 @@ pub(crate) mod tests { .values() .filter_map(|node| match node { crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => Some( - bundle.summary_catalog.data_descriptors[&bundle.summary_catalog.definitions + bundle.summary_catalog.data_descriptors[&bundle.summary_catalog.outputs [&binding.materialization] .data_descriptor_id] .time_series_metric() @@ -7974,7 +8004,7 @@ pub(crate) mod tests { let bundle = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect("certified TopK compiles"); - assert_eq!(bundle.summary_catalog.definitions.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!( bundle.collector_plans[0].materializations[0] .evidence_source diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 26bf26af2..72dfb3e2a 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -363,7 +363,7 @@ pub struct ErpObservedShapeSource { #[serde(deny_unknown_fields)] pub struct ErpPopulationObservationScope { pub catalog_generation: asap_types::sds::CatalogGeneration, - pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub stored_output_id: asap_types::sds::StoredOutputId, pub input_semantics: asap_types::erp_observation::ErpObservationInputSemantics, pub freshness: asap_types::erp_observation::ErpObservationFreshness, } @@ -389,8 +389,8 @@ impl ErpPlanningInput { return Err("ERP evidence catalog differs from the active catalog".into()); } let materialization = catalog - .definitions - .get(&populations.summary_definition_id) + .outputs + .get(&populations.stored_output_id) .ok_or("ERP evidence summary is absent from the active catalog")?; let summary = catalog .summary_descriptors @@ -477,7 +477,7 @@ impl ErpPlanningInput { .map_err(|error| format!("invalid ERP population observations: {error}"))?; observed.validate_identity_and_freshness( &scope.catalog_generation, - scope.summary_definition_id, + scope.stored_output_id, now_ms, scope.freshness, )?; @@ -495,7 +495,7 @@ impl ErpPlanningInput { asap_types::erp_observation::ErpPopulationObservations { schema_version: 1, catalog_generation: scope.catalog_generation.clone(), - summary_definition_id: scope.summary_definition_id, + stored_output_id: scope.stored_output_id, observed_at_unix_ms: now_ms, window_start_ms: 0, window_end_ms: 0, @@ -1352,7 +1352,7 @@ mod tests { let observed = ErpPopulationObservations { schema_version: 1, catalog_generation: generation.clone(), - summary_definition_id: definition, + stored_output_id: definition, observed_at_unix_ms: 1_000, window_start_ms: 0, window_end_ms: 1_000, @@ -1370,7 +1370,7 @@ mod tests { implementation: "asap_sketchlib".into(), population_scope: Some(ErpPopulationObservationScope { catalog_generation: generation, - summary_definition_id: definition, + stored_output_id: definition, input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, freshness: ErpObservationFreshness { max_age_ms: 100, @@ -1480,12 +1480,12 @@ mod tests { .unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = catalog.reference().unwrap(); - observed.summary_definition_id = *catalog.definitions.keys().next().unwrap(); + observed.stored_output_id = *catalog.outputs.keys().next().unwrap(); observed.input_semantics = asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; policy.observed_populations = Some(observed.clone()); policy.resolve_population_data_descriptor(Some(&catalog)); - let expected = &catalog.definitions[&observed.summary_definition_id].data_descriptor_id; + let expected = &catalog.outputs[&observed.stored_output_id].data_descriptor_id; assert_eq!( &policy.resolved_data_descriptor.as_ref().unwrap().id, expected diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 187235955..639dfb344 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -9,15 +9,15 @@ pub fn install_selected_dag( query_plan_sink: QueryNodeId, materialization: impl Fn( planner_types::post_asap::PostAsapNodeId, - ) -> Option, + ) -> Option, query_node: impl Fn(planner_types::post_asap::PostAsapNodeId) -> Option, ) -> Result { let mut nodes = std::collections::BTreeMap::new(); let mut precompute_sinks = Vec::new(); for node in &dag.nodes { - let binding = if let Some(summary_definition) = materialization(node.id) { + let binding = if let Some(stored_output) = materialization(node.id) { precompute_sinks.push(node.id); - BackendNodeBinding::Materialization { summary_definition } + BackendNodeBinding::Materialization { stored_output } } else if node.output_state.timing == planner_types::post_asap::ExecutionTiming::MaintenanceTime { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 19c12c689..db7557c26 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1029,10 +1029,9 @@ mod catalog_binding_tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: config.policy_fingerprint().into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - config.policy_fingerprint().into(), - ), + stored_output_reference: catalog + .output_reference(config.policy_fingerprint().into()) + .unwrap(), output_grouping: PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1162,10 +1161,9 @@ mod catalog_binding_tests { SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); let (mut counter_plan, _) = fixture(); binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); - binding(&mut counter_plan).stored_output_reference = - asap_types::sds::StoredOutputReference::for_definition( - counter.policy_fingerprint().into(), - ); + binding(&mut counter_plan).stored_output_reference = counter_catalog + .output_reference(counter.policy_fingerprint().into()) + .unwrap(); as_rate_plan(counter_plan) .validate_against_catalog(&counter_catalog) .unwrap(); @@ -1208,10 +1206,9 @@ mod tests { Ok(MaterializationBinding { full_window_slide_ms: None, materialization: PolicyFingerprint(7).into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - PolicyFingerprint(7).into(), - ), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + PolicyFingerprint(7).into(), + ), output_grouping: PhysicalGrouping::PerEntity, window_ms: 300_000, pane_origin_ms: Some(0), diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index a8ffa3132..b5d0af51f 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -585,46 +585,45 @@ mod hybrid_tests { ) .unwrap(); let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let entry = crate::query_plan::compile_bound_composable_mapped( - "hybrid".into(), - query.into(), - &selected, - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |node, _| { - let (_, _, spatial_filter) = - crate::physical::compiler::raw_materialization_input_contract(node) - .map_err(QueryPlanError::Invalid)?; - Ok(MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint(if spatial_filter.is_empty() { - 7 - } else { - 8 - }) - .into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( - asap_types::PolicyFingerprint(if spatial_filter.is_empty() { - 7 - } else { - 8 - }) + let entry = + crate::query_plan::compile_bound_composable_mapped( + "hybrid".into(), + query.into(), + &selected, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::Reject, + |node, _| { + let (_, _, spatial_filter) = + crate::physical::compiler::raw_materialization_input_contract(node) + .map_err(QueryPlanError::Invalid)?; + Ok(MaterializationBinding { + full_window_slide_ms: None, + item_labels: Vec::new(), + materialization: asap_types::PolicyFingerprint( + if spatial_filter.is_empty() { 7 } else { 8 }, + ) .into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - }) - }, - |_, _| {}, - ) - .unwrap(); + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + asap_types::PolicyFingerprint(if spatial_filter.is_empty() { + 7 + } else { + 8 + }) + .into(), + ), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + }) + }, + |_, _| {}, + ) + .unwrap(); assert_eq!(entry.materialization_bindings().len(), 2); assert!(!entry.nodes.values().any(|node| matches!( node, @@ -1309,7 +1308,7 @@ mod remote_boundary_regressions { use super::*; #[test] - fn summary_definition_identity_is_independent_of_matcher_order() { + fn stored_output_identity_is_independent_of_matcher_order() { let first = LabelMatcher { name: "job".into(), value: "orders".into(), diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 92dfd41dc..d16b6df7e 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -91,6 +91,12 @@ impl WindowMaterializationLayout { /// An `aggregationId` field in input YAML is ignored for compatibility. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputeMaterialization { + /// Explicit deployment output allocation; independent of semantic identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stored_output_id: Option, + /// Planner-selected dependency closure ending at the persisted output. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub semantic_fragment: Option, pub aggregation_type: AggregationType, pub aggregation_sub_type: String, pub parameters: HashMap, @@ -296,6 +302,8 @@ impl PrecomputeMaterialization { let spatial_filter_normalized = normalize_spatial_filter(&spatial_filter); Self { + stored_output_id: None, + semantic_fragment: None, aggregation_type, aggregation_sub_type, parameters, diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index d53dd0807..eb1976b1f 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -6,12 +6,12 @@ use planner_types::post_asap::PostAsapNodeId; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::{executable_plan::OwnedPostAsapDag, sds::SummaryDefinitionId}; +use crate::{executable_plan::OwnedPostAsapDag, sds::StoredOutputId}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct DerivedInputIdentity { - pub inputs: BTreeSet, + pub inputs: BTreeSet, pub program_sha256: String, } @@ -35,7 +35,7 @@ impl DerivedInputIdentity { pub fn from_dag( document: &OwnedPostAsapDag, root: PostAsapNodeId, - frontiers: &BTreeMap, + frontiers: &BTreeMap, ) -> Result { if ![ crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, @@ -163,7 +163,7 @@ mod tests { #[test] fn derived_source_is_distinct_and_generation_independent() { let raw = config(); - let raw_id = SummaryDefinitionId::from(raw.policy_fingerprint()); + let raw_id = StoredOutputId::from(raw.policy_fingerprint()); let mut derived = raw.clone(); derived.derived_input = Some(DerivedInputIdentity { inputs: BTreeSet::from([raw_id]), @@ -173,7 +173,7 @@ mod tests { let a = SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); - assert_eq!(a.definitions, b.definitions); + assert_eq!(a.outputs, b.outputs); assert_eq!(a.data_descriptors, b.data_descriptors); let mut renamed = derived.clone(); renamed.metric = "output_alias".into(); @@ -187,7 +187,7 @@ mod tests { fn raw_utf8_metric_cannot_impersonate_derived_policy_domain() { let mut derived = config(); derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "d".repeat(64), }); let mut raw = derived.clone(); @@ -203,7 +203,7 @@ mod tests { fn catalog_rejects_missing_derived_dependencies_and_raw_source_conflicts() { let mut derived = config(); derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "b".repeat(64), }); assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); @@ -244,7 +244,7 @@ mod tests { #[test] fn semantic_signature_ignores_node_and_query_numbering_but_not_inputs() { - let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let source = StoredOutputId::from(config().policy_fingerprint()); let a = program(1, 2); let first = DerivedInputIdentity::from_dag( &a, @@ -307,7 +307,7 @@ mod tests { }; let config = config(); let input = DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), program_sha256: "f".repeat(64), }; let data = DataDescriptor::new_typed( @@ -333,7 +333,7 @@ mod tests { use crate::precompute_plan::{PlanEnvelope, PrecomputePlan}; let mut config = config(); config.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), program_sha256: "c".repeat(64), }); let envelope = PlanEnvelope { @@ -369,7 +369,7 @@ mod tests { let mut edge = dag.edges[0].clone(); edge.producer = PostAsapNodeId(3); dag.edges.push(edge); - let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let source = StoredOutputId::from(config().policy_fingerprint()); let frontiers = BTreeMap::from([(PostAsapNodeId(1), source)]); let first = DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap(); assert_eq!(first.inputs, BTreeSet::from([source])); diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index 5e7cb103b..df23f6f1b 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -1,6 +1,6 @@ //! Versioned runtime evidence about the inputs of one installed summary. //! Shape is generic so the transport contract does not depend on a planner. -use crate::sds::{CatalogGeneration, SummaryDefinitionId, SummaryInstanceId}; +use crate::sds::{CatalogGeneration, StoredOutputId, SummaryInstanceId}; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -22,7 +22,7 @@ pub struct ErpPopulationObservation { pub struct ErpPopulationObservations { pub schema_version: u32, pub catalog_generation: CatalogGeneration, - pub summary_definition_id: SummaryDefinitionId, + pub stored_output_id: StoredOutputId, pub observed_at_unix_ms: u64, pub window_start_ms: i64, pub window_end_ms: i64, @@ -44,13 +44,13 @@ impl ErpPopulationObservations { pub fn validate_identity_and_freshness( &self, expected_generation: &CatalogGeneration, - expected_definition: SummaryDefinitionId, + expected_definition: StoredOutputId, now_ms: u64, freshness: ErpObservationFreshness, ) -> Result<(), &'static str> { if self.schema_version != 1 || &self.catalog_generation != expected_generation - || self.summary_definition_id != expected_definition + || self.stored_output_id != expected_definition { return Err("ERP observation belongs to a different catalog or summary"); } @@ -189,7 +189,7 @@ impl ErpPopulationObservations { Some(ErpPopulationObservations { schema_version: self.schema_version, catalog_generation: self.catalog_generation, - summary_definition_id: self.summary_definition_id, + stored_output_id: self.stored_output_id, observed_at_unix_ms: self.observed_at_unix_ms, window_start_ms: self.window_start_ms, window_end_ms: self.window_end_ms, diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index d84764dce..c7462edd1 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -8,7 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::sds::SummaryDefinitionId; +use crate::sds::StoredOutputId; use planner_types::post_asap::{ EdgeRole, ExecutableDag, ExecutableDagEdge, ExecutableDagNode, ExecutionDataState, ExecutionTiming, GroupingEdgeCompatibility, PostAsapNodeId, WindowEdgeCompatibility, @@ -259,7 +259,7 @@ pub enum BackendNodeBinding { QueryInput, MaintenanceInput, Materialization { - summary_definition: SummaryDefinitionId, + stored_output: StoredOutputId, }, } diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 0cccd182e..1d4af2fef 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -16,8 +16,10 @@ pub mod producer_plan; pub mod query_requirements; pub mod routing_index; pub mod sds; +pub mod semantic_fragment; pub mod storage_backend; pub mod summary_catalog; +pub mod summary_semantics; pub mod table_population; pub mod traits; pub mod utils; diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs index 1122c4200..d678a23d5 100644 --- a/crates/asap_types/src/plan_publication.rs +++ b/crates/asap_types/src/plan_publication.rs @@ -97,7 +97,7 @@ pub fn validate_stored_output_references( .materializations .iter() .map(|config| (config.policy_fingerprint().into(), config)) - .collect::>(); + .collect::>(); for entry in query.entries.values() { for binding in entry.materialization_bindings() { let writer = writers.get(&binding.materialization).ok_or_else(|| { diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index ca7353aa1..18847d73b 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -1,50 +1,12 @@ -//! Content-addressed policy identity. +//! Legacy routing wrapper for a deployed stored output. //! -//! `PolicyFingerprint` is the merged-sid-identity-chain replacement for -//! the controller-allocated `aggregation_id: u64`. Where `aggregation_id` -//! is a counter the control plane mints and ships in the streaming-config -//! YAML, `PolicyFingerprint` is derived deterministically from the -//! `AggregationConfig`'s content — so two control planes producing the -//! same policy independently produce the same fingerprint, and the data -//! plane can index without a separate id allocation. -//! -//! ## Identity contract -//! -//! `PolicyFingerprint = h(metric, agg_type, sub_type, parameters, -//! grouping_labels, aggregated_labels, rollup_labels, window_size, -//! slide_interval, window_type, pane_origin_ms, spatial_filter_normalized)` -//! -//! The hash includes **every** field of `AggregationConfig` that -//! determines what the policy does — sketch / exact-agg shape, -//! group-by + rollup layout, window cadence, spatial filter. Two -//! configs that compare equal on these dimensions produce the same -//! fingerprint; two that differ produce different fingerprints. -//! -//! Fields *excluded* from the fingerprint: -//! - `aggregation_id` itself (the thing we're replacing — it's a -//! downstream label, not part of identity). -//! - `original_yaml` (incidental serialization artifact). -//! - `num_aggregates_to_retain` (retention policy, not aggregation -//! semantics — two policies with the same shape but different -//! retention are *the same policy* for ingest/query routing -//! purposes; retention is a separate concern). -//! -//! SQL source table, value projection, timestamp projection, and typed -//! population are included explicitly; the output metric is not a substitute -//! for these source semantics. -//! -//! ## Hash function -//! -//! `xxh64` keyed at 0, matching the existing `compute_agg_config_id` -//! helper this replaces. 64-bit gives ~4B-policy birthday bound -//! (collision probability ~10⁻¹¹ at 100K live policies); ample for -//! foreseeable workloads. Bump to sha256 if the control plane ever -//! manages >10⁶ live policies and we want deterministic uniqueness. -//! -//! The fingerprint is **stable across hosts and versions**: the byte -//! layout this module produces is the contract. Don't reorder fields, -//! don't change separator bytes — any such change invalidates every -//! deployed fingerprint and forces a cold-start rebuild. +//! An explicit `AggregationConfig::stored_output_id` takes precedence. +//! Otherwise the compiler allocates a deterministic default from the existing +//! policy fields (including pane layout and cadence). This identifier is not +//! semantic identity: `SummaryDefinitionId` hashes the versioned semantic +//! definition, and several deployed outputs may share that definition. +//! Catalog installation checks that an output is never assigned conflicting +//! computation or format contracts. use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -52,11 +14,7 @@ use xxhash_rust::xxh64::xxh64; use crate::aggregation_config::AggregationConfig; -/// Stable, content-addressed handle for an `AggregationConfig`. -/// -/// Wrap a `u64` so callers can't accidentally swap a `PolicyFingerprint` -/// with an `aggregation_id` — they're both u64-shaped but they index -/// different things (content-addressed vs. controller-allocated). +/// Routing handle for one deployed stored output. See the module contract. #[derive( Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, )] @@ -83,6 +41,9 @@ impl PolicyFingerprint { /// for nested-shape determinism (matches the existing /// `parameters_canonical` form used in `AggKind::ExactAgg`). pub fn from_config(cfg: &AggregationConfig) -> Self { + if let Some(output) = cfg.stored_output_id { + return output.fingerprint(); + } let mut buf: Vec = Vec::with_capacity(512); if !cfg.population_key_encoding.is_legacy() { diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index a6375da38..67ebe26f0 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -144,7 +144,7 @@ pub struct IngestContract { pub timestamp_unit: TimestampUnit, pub require_plan_identity: bool, #[serde(alias = "require_materialization_identity")] - pub require_summary_definition_identity: bool, + pub require_stored_output_identity: bool, pub require_registered_producer: bool, } @@ -222,7 +222,7 @@ pub struct StateSchemaContract { pub stored_output_reference: crate::sds::StoredOutputReference, pub schema_id: String, pub schema_version: u32, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub family: StateFamilyContract, pub source: Source, #[serde( @@ -256,7 +256,7 @@ pub struct StateWindowContract { pub struct ProducerContract { pub producer_id: String, pub collector_id: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub schema_id: String, /// Authoritative partitions for completion barriers. Empty legacy contracts /// authorize state ingestion only, never completion claims. @@ -334,7 +334,7 @@ impl PrecomputePlan { ); let value_projection = materialization.effective_value_projection().clone(); Ok(StateSchemaContract { - stored_output_reference: crate::sds::StoredOutputReference::for_definition( + stored_output_reference: crate::sds::StoredOutputReference::for_output( fingerprint.into(), ), schema_id: state_schema_id(fingerprint), @@ -381,7 +381,7 @@ impl PrecomputePlan { endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, } } else { @@ -390,7 +390,7 @@ impl PrecomputePlan { endpoint_path: "/v1/metrics".into(), timestamp_unit: TimestampUnit::UnixNanoseconds, require_plan_identity: true, - require_summary_definition_identity: true, + require_stored_output_identity: true, require_registered_producer: true, } }, @@ -438,7 +438,7 @@ impl PrecomputePlan { /// monotonic progress. The runtime must enforce those before accepting it. pub fn validate_watermark_scope( &self, - materialization: crate::sds::SummaryDefinitionId, + materialization: crate::sds::StoredOutputId, barrier: &crate::sds::SummaryWatermarkBarrier, ) -> Result<(), PrecomputePlanError> { self.validate()?; @@ -473,14 +473,14 @@ impl PrecomputePlan { self.ingest.endpoint_path == "/v1/metrics" && self.ingest.timestamp_unit == TimestampUnit::UnixNanoseconds && self.ingest.require_plan_identity - && self.ingest.require_summary_definition_identity + && self.ingest.require_stored_output_identity && self.ingest.require_registered_producer } IngestProtocol::PrometheusRemoteWriteV1 => { self.ingest.endpoint_path == "/api/v1/write" && self.ingest.timestamp_unit == TimestampUnit::UnixMilliseconds && !self.ingest.require_plan_identity - && !self.ingest.require_summary_definition_identity + && !self.ingest.require_stored_output_identity && !self.ingest.require_registered_producer } }; @@ -563,8 +563,8 @@ impl PrecomputePlan { .map_err(PrecomputePlanError::CatalogContract)?; for sink in &installed.binding.precompute_sinks { if !matches!(installed.binding.node(*sink), - Some(crate::executable_plan::BackendNodeBinding::Materialization { summary_definition }) - if summary_definition.fingerprint() == config.policy_fingerprint()) + Some(crate::executable_plan::BackendNodeBinding::Materialization { stored_output }) + if stored_output.fingerprint() == config.policy_fingerprint()) { continue; } @@ -589,9 +589,9 @@ impl PrecomputePlan { .iter() .filter_map(|(node, binding)| match binding { crate::executable_plan::BackendNodeBinding::Materialization { - summary_definition, - } if derived.inputs.contains(summary_definition) => { - Some((*node, *summary_definition)) + stored_output, + } if derived.inputs.contains(stored_output) => { + Some((*node, *stored_output)) } _ => None, }) @@ -686,7 +686,7 @@ impl PrecomputePlan { .map_err(PrecomputePlanError::CatalogContract)?; for node in &dag.nodes { let Some(crate::executable_plan::BackendNodeBinding::Materialization { - summary_definition, + stored_output, }) = installed.binding.node(node.id) else { continue; @@ -694,7 +694,7 @@ impl PrecomputePlan { let Some(config) = self .materializations .iter() - .find(|config| config.policy_fingerprint() == summary_definition.fingerprint()) + .find(|config| config.policy_fingerprint() == stored_output.fingerprint()) else { return Err(PrecomputePlanError::CatalogContract( "DAG materialization has no runtime configuration".into(), @@ -798,8 +798,7 @@ impl PrecomputePlan { || !stored_outputs.insert(schema.stored_output_reference.stored_output_id) || schema.schema_version == 0 || schema.encodings.is_empty() - || schema.stored_output_reference.validate().is_err() - || schema.stored_output_reference.definition_id != schema.materialization + || schema.stored_output_reference.stored_output_id != schema.materialization { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), @@ -1016,7 +1015,7 @@ mod source_window_cohort_tests { .validate_watermark_scope(materialization, &wrong) .is_err()); } - let other_materialization = crate::sds::SummaryDefinitionId(crate::PolicyFingerprint( + let other_materialization = crate::sds::StoredOutputId::from(crate::PolicyFingerprint( materialization.as_u64().wrapping_add(1), )); assert!(restored diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 1d474e0a6..f2322cd93 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -1,6 +1,6 @@ //! Catalog consistency checks for the precompute execution plan. use super::*; -use crate::sds::{SummaryDefinitionId, SummaryDescriptor}; +use crate::sds::{StoredOutputId, SummaryDescriptor}; use crate::summary_catalog::SummaryCatalog; use planner_types::pre_asap::Source; use std::collections::BTreeSet; @@ -13,11 +13,16 @@ impl PrecomputePlan { /// only the immutable snapshot reference; descriptors are installed once. pub fn bind_catalog(&mut self, catalog: &SummaryCatalog) -> Result<(), PrecomputePlanError> { for config in &self.materializations { - let id = SummaryDefinitionId::from(config.policy_fingerprint()); - catalog.definitions.get(&id).ok_or_else(|| { + let id = StoredOutputId::from(config.policy_fingerprint()); + catalog.outputs.get(&id).ok_or_else(|| { invalid(format!("missing catalog materialization {}", id.as_u64())) })?; } + for schema in &mut self.schemas { + schema.stored_output_reference = catalog + .output_reference(schema.materialization) + .map_err(|e| invalid(e.to_string()))?; + } self.summary_catalog = Some( catalog .reference() @@ -39,6 +44,52 @@ impl PrecomputePlan { catalog: &SummaryCatalog, ) -> Result<(), PrecomputePlanError> { catalog.validate().map_err(|e| invalid(e.to_string()))?; + let expected = SummaryCatalog::from_materializations( + catalog.plan_id, + catalog.plan_version, + &self.materializations, + ) + .map_err(|e| invalid(e.to_string()))?; + if catalog.outputs != expected.outputs || catalog.definitions != expected.definitions { + return Err(invalid( + "stored output semantics differ from installed writer computation", + )); + } + for config in &self.materializations { + if config.derived_input.is_some() && config.semantic_fragment.is_none() { + return Err(invalid( + "derived stored output requires its complete Planner semantic closure", + )); + } + if let Some(expected) = &config.semantic_fragment { + let mut found = false; + for installed in self.executable_dags.values() { + let dag = installed.document.decode().map_err(invalid)?; + for (id, binding) in &installed.binding.nodes { + if matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } + if stored_output.fingerprint() == config.policy_fingerprint()) + { + found = true; + let actual = + crate::semantic_fragment::SemanticFragment::from_stored_output( + &dag, *id, + ) + .map_err(invalid)?; + if &actual != expected { + return Err(invalid( + "semantic definition differs from Planner-selected producer", + )); + } + } + } + } + if !found { + return Err(invalid( + "semantic definition has no Planner-selected producer", + )); + } + } + } let expected_reference = catalog .reference() .map_err(|error| invalid(error.to_string()))?; @@ -51,14 +102,14 @@ impl PrecomputePlan { let ids: BTreeSet<_> = self .materializations .iter() - .map(|m| SummaryDefinitionId::from(m.policy_fingerprint())) + .map(|m| StoredOutputId::from(m.policy_fingerprint())) .collect(); - if ids != catalog.definitions.keys().copied().collect() { + if ids != catalog.outputs.keys().copied().collect() { return Err(invalid("catalog/reference/materialization sets differ")); } for config in &self.materializations { - let id = SummaryDefinitionId::from(config.policy_fingerprint()); - let binding = &catalog.definitions[&id]; + let id = StoredOutputId::from(config.policy_fingerprint()); + let binding = &catalog.outputs[&id]; let expected = SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { @@ -138,7 +189,11 @@ impl PrecomputePlan { return Err(invalid("session lifecycle is not supported")) } }; - if schema.schema_id != state_schema_id(id.fingerprint()) + if schema.stored_output_reference + != catalog + .output_reference(id) + .map_err(|e| invalid(e.to_string()))? + || schema.schema_id != state_schema_id(id.fingerprint()) || schema.family != expected_family || schema.source != source || &schema.value_projection != projection diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 2690d1655..1046028fd 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -10,7 +10,7 @@ use thiserror::Error; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct CollectorMaterialization { pub query_id: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub metric: String, pub algorithm: String, pub parameters: Value, @@ -76,7 +76,7 @@ pub struct FrameIdentityContract { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct TransmissionRule { - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub producer_id: String, pub schema_id: String, pub mode: TransmissionMode, @@ -194,7 +194,7 @@ pub struct RuntimeRulePolicy { pub struct RuntimeAdaptationEvidence { pub plan_id: u64, pub plan_version: u64, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub producer_id: String, pub schema_id: String, pub producer_version: String, @@ -230,7 +230,7 @@ pub struct SummaryFrameIdentity { pub plan_id: u64, pub plan_version: u64, pub backend_compat: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, /// Canonical producer-side identity for one concrete retained-label group. pub series_identity: String, pub schema_id: String, @@ -275,7 +275,7 @@ pub enum TransmissionPlanError { fn validate_catalog_projection( reference: Option<&crate::sds::CatalogGeneration>, envelope: &PlanEnvelope, - materializations: impl IntoIterator, + materializations: impl IntoIterator, catalog: &crate::summary_catalog::SummaryCatalog, ) -> Result<(), TransmissionPlanError> { let expected = catalog @@ -290,7 +290,7 @@ fn validate_catalog_projection( )); } for id in materializations { - if !catalog.definitions.contains_key(&id) { + if !catalog.outputs.contains_key(&id) { return Err(TransmissionPlanError::Catalog(format!( "unknown materialization {}", id.as_u64() diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 9e3e04bfe..2135d6540 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; pub use crate::QueryLanguage; -use crate::{sds::SummaryDefinitionId, PolicyFingerprint}; +use crate::{sds::StoredOutputId, PolicyFingerprint}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -93,6 +93,25 @@ impl QueryPlan { self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) } + pub fn bind_catalog( + &mut self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + for entry in self.entries.values_mut() { + for node in entry.nodes.values_mut() { + if let QueryPlanNode::ReadMaterialization { binding } = node { + binding.stored_output_reference = catalog + .output_reference(binding.materialization) + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + } + } + } + self.validate_against_catalog(catalog) + } + /// Validate semantic bindings against the authoritative snapshot before use. pub fn validate_against_catalog( &self, @@ -106,23 +125,23 @@ impl QueryPlan { "QueryPlan and SummaryCatalog have different plan identity/version".into(), )); } - let available = catalog - .definitions - .keys() - .copied() - .map(Into::into) - .collect(); + let available = catalog.outputs.keys().copied().map(Into::into).collect(); self.validate(&available)?; for entry in self.entries.values() { for binding in entry.materialization_bindings() { let identity = catalog - .definitions + .outputs .get(&binding.materialization) .ok_or_else(|| { QueryPlanError::Invalid( "query binding references absent catalog materialization".into(), ) })?; + if binding.stored_output_reference.definition_id != identity.definition_id { + return Err(QueryPlanError::Invalid( + "read definition differs from installed output".into(), + )); + } let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; if binding.window_ms == 0 { return Err(QueryPlanError::Invalid( @@ -148,7 +167,7 @@ impl QueryPlan { "counter readout must directly consume one catalog materialization".into(), )); }; - let identity = &catalog.definitions[&binding.materialization]; + let identity = &catalog.outputs[&binding.materialization]; let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; if !matches!( descriptor.fidelity, @@ -422,9 +441,7 @@ impl QueryPlanEntry { } } if let QueryPlanNode::ReadMaterialization { binding } = node { - if binding.stored_output_reference.validate().is_err() - || binding.stored_output_reference.definition_id != binding.materialization - { + if binding.stored_output_reference.stored_output_id != binding.materialization { return Err(QueryPlanError::Invalid( "read binding has invalid stored output or definition".into(), )); @@ -471,7 +488,7 @@ pub struct MaterializationBinding { /// None denotes disjoint pane storage. #[serde(default, skip_serializing_if = "Option::is_none")] pub full_window_slide_ms: Option, - pub materialization: SummaryDefinitionId, + pub materialization: StoredOutputId, /// Query operator grouping applied while folding those SIDs. pub output_grouping: PhysicalGrouping, /// Labels whose values form an item identity inside a keyed sketch. diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 88fadf379..9f4a50cfc 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -30,63 +30,68 @@ macro_rules! descriptor_id { } }; } -/// Semantic materialization reference. Wire-compatible with PolicyFingerprint, -/// but distinct from descriptor IDs and concrete [`SummaryInstanceId`] identity. +/// Identity of one deployed producer output. Runtime routing uses this identity, +/// never the semantic definition hash. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(transparent)] -pub struct SummaryDefinitionId(pub crate::PolicyFingerprint); -impl SummaryDefinitionId { +pub struct StoredOutputId(pub u64); +impl StoredOutputId { pub fn fingerprint(self) -> crate::PolicyFingerprint { - self.0 + crate::PolicyFingerprint(self.0) } pub fn as_u64(self) -> u64 { - self.0 .0 + self.0 } } -impl From for SummaryDefinitionId { +impl From for StoredOutputId { fn from(value: crate::PolicyFingerprint) -> Self { - Self(value) + Self(value.0) } } -impl From for crate::PolicyFingerprint { - fn from(value: SummaryDefinitionId) -> Self { - value.0 +impl From for crate::PolicyFingerprint { + fn from(value: StoredOutputId) -> Self { + Self(value.0) } } -/// Identity of one persisted producer output within an installed plan version. -/// V1 derives it from the definition ID because the runtime index is keyed by -/// definition; a future schema may allocate independent output IDs. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(transparent)] -pub struct StoredOutputId(pub u64); +descriptor_id!(SummaryDefinitionId); +impl SummaryDefinitionId { + pub(crate) fn from_semantics(bytes: &[u8]) -> Self { + use sha2::{Digest, Sha256}; + Self(format!("sds-v1:{:x}", Sha256::digest(bytes))) + } + pub fn validate(&self) -> Result<(), SdsError> { + let hash = self.0.strip_prefix("sds-v1:").unwrap_or(""); + if hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + Ok(()) + } else { + Err(SdsError("invalid semantic definition ID".into())) + } + } +} -/// Typed join key carried by both the writer and every bound reader. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +/// The installed writer/reader binding joins deployment identity and semantics. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct StoredOutputReference { - #[serde(alias = "state_slot_id")] pub stored_output_id: StoredOutputId, pub definition_id: SummaryDefinitionId, } - impl StoredOutputReference { - /// V1 binding for the single stored output of a definition. - pub fn for_definition(definition_id: SummaryDefinitionId) -> Self { + /// Internal compilation placeholder. Catalog binding must replace its empty + /// semantic identity before installation; it cannot authorize a read/write. + pub fn for_output(stored_output_id: StoredOutputId) -> Self { Self { - stored_output_id: StoredOutputId(definition_id.as_u64()), - definition_id, + stored_output_id, + definition_id: SummaryDefinitionId(String::new()), } } - pub fn validate(&self) -> Result<(), SdsError> { - if *self == Self::for_definition(self.definition_id) { - Ok(()) - } else { - Err(SdsError( - "stored output differs from its V1 definition binding".into(), - )) - } + self.definition_id.validate() } } @@ -149,7 +154,7 @@ pub struct SummarySourcePartition { #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryInstanceCoordinates { - pub summary_definition_id: SummaryDefinitionId, + pub stored_output_id: StoredOutputId, pub time_range: HalfOpenTimeRange, pub group_values: BTreeMap, } @@ -160,7 +165,7 @@ impl SummaryInstanceCoordinates { serde_json::to_vec(&self.group_values).map_err(|error| SdsError(error.to_string()))?; SummaryInstanceId::new(format!( "summary-instance:v1:{}:{}:{}:{}", - self.summary_definition_id.as_u64(), + self.stored_output_id.as_u64(), self.time_range.start_ms, self.time_range.end_ms, xxhash_rust::xxh64::xxh64(&bytes, 0) @@ -314,7 +319,6 @@ pub struct SummaryInstance { pub instance_id: SummaryInstanceId, #[serde(alias = "state_slot_id")] pub stored_output_id: StoredOutputId, - #[serde(alias = "materialization_id")] pub summary_definition_id: SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, pub data_descriptor_id: DataDescriptorId, @@ -334,13 +338,7 @@ pub struct SummaryInstance { impl SummaryInstance { pub fn validate(&self) -> Result<(), SdsError> { - if self.stored_output_id - != StoredOutputReference::for_definition(self.summary_definition_id).stored_output_id - { - return Err(SdsError( - "summary instance has an invalid stored output".into(), - )); - } + self.summary_definition_id.validate()?; if self.time_range.start_ms >= self.time_range.end_ms { return Err(SdsError( "summary instance time range must be non-empty".into(), @@ -435,10 +433,11 @@ impl ObservedSummaryInventory { )); } let definition = catalog - .definitions - .get(&instance.summary_definition_id) + .outputs + .get(&instance.stored_output_id) .ok_or_else(|| SdsError("summary instance has no catalog definition".into()))?; - if instance.summary_descriptor_id != definition.summary_descriptor_id + if instance.summary_definition_id != definition.definition_id + || instance.summary_descriptor_id != definition.summary_descriptor_id || instance.data_descriptor_id != definition.data_descriptor_id || instance.state_reference.state_schema_version != catalog.summary_descriptors[&definition.summary_descriptor_id] @@ -1223,7 +1222,7 @@ mod tests { }, instance_id: SummaryInstanceId::new("instance-1").unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(crate::PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(crate::PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 1_000, end_ms: 2_000, @@ -1289,7 +1288,7 @@ mod tests { SummaryInstance { instance_id: SummaryInstanceId::new("instance-1").unwrap(), stored_output_id: StoredOutputId(7), - summary_definition_id: SummaryDefinitionId(crate::PolicyFingerprint(7)), + summary_definition_id: SummaryDefinitionId::from_semantics(b"fixture"), summary_descriptor_id: descriptor( 200, FidelityGuarantee::KllRankError { @@ -1353,27 +1352,33 @@ mod tests { fn stored_output_and_payload_version_must_match_instance_definition() { let mut instance = observed_instance(InstanceLifecycle::Persistent); instance.stored_output_id = StoredOutputId(8); - assert!(instance.validate().is_err()); + assert!(instance.validate().is_ok()); instance.stored_output_id = StoredOutputId(7); instance.state_reference.generation = 3; assert!(instance.validate().is_err()); - let mut reference = StoredOutputReference::for_definition(instance.summary_definition_id); + let mut reference = StoredOutputReference::for_output(instance.stored_output_id); reference.stored_output_id = StoredOutputId(8); assert!(reference.validate().is_err()); } + // Old aliases and numeric semantic IDs must not authorize the new format. #[test] - fn stored_output_reference_accepts_legacy_state_slot_field() { - let reference: StoredOutputReference = serde_json::from_value(json!({ - "state_slot_id": 7, - "definition_id": 7 + fn stored_output_reference_rejects_legacy_identity() { + assert!(serde_json::from_value::(json!({ + "state_slot_id": 7, "definition_id": 7 })) - .unwrap(); - assert_eq!(reference.stored_output_id, StoredOutputId(7)); + .is_err()); + let reference = StoredOutputReference { + stored_output_id: StoredOutputId(8), + definition_id: SummaryDefinitionId::from_semantics(b"fixture"), + }; reference.validate().unwrap(); assert_eq!( - serde_json::to_value(reference).unwrap(), - json!({"stored_output_id": 7, "definition_id": 7}) + serde_json::from_value::( + serde_json::to_value(&reference).unwrap() + ) + .unwrap(), + reference ); } @@ -1700,9 +1705,9 @@ mod tests { assert_ne!(first.id, second.id); } #[test] - fn summary_definition_id_preserves_legacy_wire_identity() { + fn stored_output_id_preserves_legacy_wire_identity() { let fingerprint = crate::PolicyFingerprint(42); - let id = SummaryDefinitionId::from(fingerprint); + let id = StoredOutputId::from(fingerprint); assert_eq!(id.fingerprint(), fingerprint); assert_eq!(id.as_u64(), 42); assert_eq!(crate::PolicyFingerprint::from(id), fingerprint); @@ -1710,10 +1715,7 @@ mod tests { serde_json::to_value(id).unwrap(), serde_json::to_value(fingerprint).unwrap() ); - assert_eq!( - serde_json::from_str::("42").unwrap(), - id - ); + assert_eq!(serde_json::from_str::("42").unwrap(), id); } /// Every supplied alias must agree with the declared fidelity, including runtime w/d keys. #[test] diff --git a/crates/asap_types/src/semantic_fragment.rs b/crates/asap_types/src/semantic_fragment.rs new file mode 100644 index 000000000..390354bd9 --- /dev/null +++ b/crates/asap_types/src/semantic_fragment.rs @@ -0,0 +1,2 @@ +//! Planner owns the versioned semantic description and its normalization. +pub use planner_types::post_asap::SummarySemanticFragment as SemanticFragment; diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 8212e0ad7..4cc0e985a 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -6,19 +6,20 @@ use std::collections::BTreeMap; use crate::sds::{ - CatalogGeneration, DataDescriptor, DataDescriptorId, DataSourceIdentity, SummaryDefinitionId, + CatalogGeneration, DataDescriptor, DataDescriptorId, DataSourceIdentity, StoredOutputId, SummaryDescriptor, SummaryDescriptorId, }; use crate::PolicyFingerprint; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 3; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 4; /// Canonical definition binds operator and population descriptors. Writer /// layout and concrete state belong to installed plans and runtime instances. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct SummaryDefinition { +pub struct StoredOutputDefinition { + pub definition_id: crate::sds::SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, pub data_descriptor_id: DataDescriptorId, } @@ -31,7 +32,9 @@ pub struct SummaryCatalog { pub plan_version: u64, pub summary_descriptors: BTreeMap, pub data_descriptors: BTreeMap, - pub definitions: BTreeMap, + pub outputs: BTreeMap, + pub definitions: + BTreeMap, } #[derive(Debug, thiserror::Error)] @@ -67,6 +70,20 @@ impl CatalogGeneration { } impl SummaryCatalog { + pub fn output_reference( + &self, + id: StoredOutputId, + ) -> Result { + let output = self + .outputs + .get(&id) + .ok_or(SummaryCatalogError::MissingDescriptor(id.as_u64()))?; + Ok(crate::sds::StoredOutputReference { + stored_output_id: id, + definition_id: output.definition_id.clone(), + }) + } + pub fn reference(&self) -> Result { use sha2::{Digest, Sha256}; self.validate()?; @@ -113,7 +130,38 @@ impl SummaryCatalog { Ok((config.policy_fingerprint(), summary, data)) }) .collect::, SummaryCatalogError>>()?; - Self::build(plan_id, plan_version, entries) + let mut catalog = Self::build(plan_id, plan_version, entries)?; + let mut semantic_bindings = BTreeMap::new(); + for config in materializations { + let output = catalog + .outputs + .get_mut(&StoredOutputId::from(config.policy_fingerprint())) + .unwrap(); + let definition = crate::summary_semantics::SummaryDefinition::from_descriptors( + &catalog.summary_descriptors[&output.summary_descriptor_id], + &catalog.data_descriptors[&output.data_descriptor_id], + )? + .with_config(config); + let id = definition.id()?; + if semantic_bindings + .insert(config.policy_fingerprint(), id.clone()) + .is_some_and(|old| old != id) + { + return Err(SummaryCatalogError::ConflictingDefinition( + config.policy_fingerprint().0, + )); + } + output.definition_id = id.clone(); + catalog.definitions.insert(id, definition); + } + let used: std::collections::BTreeSet<_> = catalog + .outputs + .values() + .map(|o| o.definition_id.clone()) + .collect(); + catalog.definitions.retain(|id, _| used.contains(id)); + catalog.validate()?; + Ok(catalog) } pub fn build( @@ -127,21 +175,27 @@ impl SummaryCatalog { plan_version, summary_descriptors: BTreeMap::new(), data_descriptors: BTreeMap::new(), + outputs: BTreeMap::new(), definitions: BTreeMap::new(), }; for (fingerprint, summary, data) in entries { - let definition = SummaryDefinitionId::from(fingerprint); + let definition = StoredOutputId::from(fingerprint); summary .validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; data.validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let binding = SummaryDefinition { + let semantics = + crate::summary_semantics::SummaryDefinition::from_descriptors(&summary, &data)?; + let semantic_id = semantics.id()?; + catalog.definitions.insert(semantic_id.clone(), semantics); + let binding = StoredOutputDefinition { + definition_id: semantic_id, summary_descriptor_id: summary.id().clone(), data_descriptor_id: data.id().clone(), }; if catalog - .definitions + .outputs .get(&definition) .is_some_and(|old| old != &binding) { @@ -155,7 +209,7 @@ impl SummaryCatalog { catalog .data_descriptors .insert(binding.data_descriptor_id.clone(), data); - catalog.definitions.insert(definition, binding); + catalog.outputs.insert(definition, binding); } catalog.validate()?; Ok(catalog) @@ -165,6 +219,13 @@ impl SummaryCatalog { if self.schema_version != SUMMARY_CATALOG_SCHEMA_VERSION { return Err(SummaryCatalogError::SchemaVersion(self.schema_version)); } + for (id, definition) in &self.definitions { + if &definition.id()? != id { + return Err(SummaryCatalogError::Descriptor( + "semantic definition content hash mismatch".into(), + )); + } + } for (key, descriptor) in &self.summary_descriptors { descriptor .validate() @@ -185,7 +246,12 @@ impl SummaryCatalog { )); } } - for (id, binding) in &self.definitions { + for (id, binding) in &self.outputs { + if !self.definitions.contains_key(&binding.definition_id) { + return Err(SummaryCatalogError::Descriptor( + "output references absent semantic definition".into(), + )); + } if !self .summary_descriptors .contains_key(&binding.summary_descriptor_id) @@ -196,6 +262,31 @@ impl SummaryCatalog { return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); } } + for output in self.outputs.values() { + let expected = crate::summary_semantics::SummaryDefinition::from_descriptors( + &self.summary_descriptors[&output.summary_descriptor_id], + &self.data_descriptors[&output.data_descriptor_id], + )?; + if let ( + crate::summary_semantics::SummarySemantics::Configured { + computation: actual, + .. + }, + crate::summary_semantics::SummarySemantics::Configured { + computation: expected, + .. + }, + ) = ( + &self.definitions[&output.definition_id].semantics, + &expected.semantics, + ) { + if actual != expected { + return Err(SummaryCatalogError::Descriptor( + "output descriptors disagree with semantic definition".into(), + )); + } + } + } if !self .data_descriptors .values() @@ -207,13 +298,13 @@ impl SummaryCatalog { let mut pending = std::collections::BTreeMap::new(); let mut consumers: std::collections::BTreeMap<_, Vec<_>> = std::collections::BTreeMap::new(); - for (id, binding) in &self.definitions { + for (id, binding) in &self.outputs { let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { DataSourceIdentity::Derived { input } => input.inputs.clone(), _ => Default::default(), }; for source in &dependencies { - if !self.definitions.contains_key(source) { + if !self.outputs.contains_key(source) { return Err(SummaryCatalogError::Descriptor( "derived input references missing summary".into(), )); @@ -325,7 +416,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); assert_eq!(catalog.data_descriptors.len(), 3); - assert_eq!(catalog.definitions.len(), 3); + assert_eq!(catalog.outputs.len(), 3); } #[test] @@ -357,10 +448,54 @@ mod tests { SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.definitions.len(), 2); + assert_eq!(catalog.outputs.len(), 2); assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); } + // Stored pane duration identifies a deployment output, not the summary meaning. + #[test] + fn semantic_definition_is_shared_across_deployed_pane_outputs() { + let catalog = SummaryCatalog::from_materializations( + 1, + 1, + &[config("requests", "", 60), config("requests", "", 120)], + ) + .unwrap(); + assert_eq!(catalog.definitions.len(), 1); + assert_eq!(catalog.outputs.len(), 2); + } + + // A hot and rebuild output share meaning but remain independently bound. + #[test] + fn hot_and_rebuild_have_one_definition_and_two_bound_outputs() { + let mut hot = config("latency", "", 60); + hot.stored_output_id = Some(StoredOutputId(41)); + let mut rebuild = hot.clone(); + rebuild.stored_output_id = Some(StoredOutputId(42)); + let catalog = SummaryCatalog::from_materializations(7, 42, &[hot, rebuild]).unwrap(); + assert_eq!(catalog.definitions.len(), 1); + let hot = catalog.output_reference(StoredOutputId(41)).unwrap(); + let rebuild = catalog.output_reference(StoredOutputId(42)).unwrap(); + assert_eq!(hot.definition_id, rebuild.definition_id); + assert_ne!(hot, rebuild); + let mut forged = catalog.clone(); + let crate::summary_semantics::SummarySemantics::Configured { computation, .. } = + &mut forged.definitions.values_mut().next().unwrap().semantics + else { + panic!("fixture") + }; + computation.predicate = "service=other".into(); + assert!(forged.validate().is_err()); + let mut unknown = catalog.clone(); + unknown + .definitions + .values_mut() + .next() + .unwrap() + .semantic_format_version += 1; + assert!(unknown.validate().is_err()); + } + // Source/population changes never alias, while the operator can be reused. #[test] fn separates_population_and_operator_identity() { @@ -389,7 +524,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 2); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.definitions.len(), 2); + assert_eq!(catalog.outputs.len(), 2); } // Construction order cannot affect the published snapshot bytes. @@ -410,19 +545,18 @@ mod tests { fn catalog_definitions_do_not_store_writer_layout() { let mut materialization = config("requests", "", 60); materialization.pane_origin_ms = Some(7_000); - let id = SummaryDefinitionId::from(materialization.policy_fingerprint()); + let id = StoredOutputId::from(materialization.policy_fingerprint()); let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); - assert!(catalog.definitions.contains_key(&id)); + assert!(catalog.outputs.contains_key(&id)); assert!( - !serde_json::to_value(&catalog).unwrap()["definitions"][id.as_u64().to_string()] + !serde_json::to_value(&catalog).unwrap()["outputs"][id.as_u64().to_string()] .as_object() .unwrap() .contains_key("pane_origin_ms") ); let mut invalid = serde_json::to_value(&catalog).unwrap(); - invalid["definitions"][id.as_u64().to_string()]["pane_origin_ms"] = - serde_json::json!(7_000); + invalid["outputs"][id.as_u64().to_string()]["pane_origin_ms"] = serde_json::json!(7_000); assert!(serde_json::from_value::(invalid).is_err()); } @@ -498,7 +632,7 @@ mod tests { #[test] fn empty_catalog_is_valid() { let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - assert!(catalog.definitions.is_empty()); + assert!(catalog.outputs.is_empty()); catalog.validate().unwrap(); } } diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs new file mode 100644 index 000000000..387967dd2 --- /dev/null +++ b/crates/asap_types/src/summary_semantics.rs @@ -0,0 +1,129 @@ +//! Versioned, content-addressed meaning of a stored result. Runtime routing, +//! pane placement, codecs, retention and plan generations are deliberately absent. +use crate::sds::{ + DataDescriptor, DataSourceIdentity, SummaryDescriptor, SummaryOperator, ValueProjectionIdentity, +}; +use crate::summary_catalog::SummaryCatalogError; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryDefinition { + pub semantic_format_version: u32, + pub semantics: SummarySemantics, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SummarySemantics { + Planner { + fragment: crate::semantic_fragment::SemanticFragment, + }, + /// Restricted raw-input adapter for explicit native summary configurations. + /// General expressions must use the Planner fragment variant. + Configured { + computation: SummaryComputation, + value_source_column: Option, + aggregated_labels: crate::KeyByLabelNames, + rollup_labels: crate::KeyByLabelNames, + }, +} + +/// Typed semantic contract of the supported summary input. This is not a +/// deployment plan: it contains neither physical nodes nor storage references. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryComputation { + pub operator: SummaryOperator, + pub source: DataSourceIdentity, + pub value: ValueProjectionIdentity, + pub predicate: String, + pub grouping: crate::GroupingProjection, + pub timestamp_column: Option, + pub observation_semantics: String, +} + +impl SummaryDefinition { + pub fn from_descriptors( + summary: &SummaryDescriptor, + data: &DataDescriptor, + ) -> Result { + summary + .validate() + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + data.validate() + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + Ok(Self { + semantic_format_version: 1, + semantics: SummarySemantics::Configured { + value_source_column: None, + aggregated_labels: crate::KeyByLabelNames::empty(), + rollup_labels: crate::KeyByLabelNames::empty(), + computation: SummaryComputation { + operator: summary.operator.clone(), + source: data.source.clone(), + value: data.value_projection.clone(), + predicate: data.population_filter_canonical.clone(), + grouping: data.group_by_keys.clone(), + timestamp_column: data.timestamp_column.clone(), + observation_semantics: data.observation_semantics.clone(), + }, + }, + }) + } + + pub fn with_config(mut self, config: &crate::PrecomputeMaterialization) -> Self { + let fragment = config.semantic_fragment.as_ref(); + if let Some(fragment) = fragment { + self.semantics = SummarySemantics::Planner { + fragment: fragment.clone(), + }; + } else if let SummarySemantics::Configured { + value_source_column, + aggregated_labels, + rollup_labels, + .. + } = &mut self.semantics + { + *value_source_column = config.value_source_column.clone(); + *aggregated_labels = config.aggregated_labels.clone(); + *rollup_labels = config.rollup_labels.clone(); + } + self + } + + pub fn id(&self) -> Result { + if self.semantic_format_version != 1 { + return Err(SummaryCatalogError::Descriptor( + "unsupported semantic format version".into(), + )); + } + if let SummarySemantics::Planner { fragment } = &self.semantics { + fragment + .validate() + .map_err(SummaryCatalogError::Descriptor)?; + } + // Object keys are recursively sorted, independent of serde_json features. + fn canonical(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(values) => serde_json::Value::Object( + values + .into_iter() + .map(|(k, v)| (k, canonical(v))) + .collect::>() + .into_iter() + .collect(), + ), + serde_json::Value::Array(values) => { + serde_json::Value::Array(values.into_iter().map(canonical).collect()) + } + value => value, + } + } + let value = serde_json::to_value(self) + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + let bytes = serde_json::to_vec(&canonical(value)) + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + Ok(crate::sds::SummaryDefinitionId::from_semantics(&bytes)) + } +} diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index eb057edc9..22eefcd16 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -73,7 +73,7 @@ async fn main() { let reference = catalog.reference().unwrap(); let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); - precompute_plan.summary_catalog = Some(reference.clone()); + precompute_plan.bind_catalog(&catalog).unwrap(); let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 230fde049..070ea25b6 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1146,7 +1146,11 @@ async fn route_modified_otlp_sketches_to_precompute( } else { None }; - let series_key = format_series_key(&canonical_name, &dp.attrs); + let series_key = bound_sketch_series_key( + &canonical_name, + &dp.attrs, + frame_identity.as_ref(), + ); let ts_ms = (dp.time_unix_nano / 1_000_000) as i64; // Sid resolution — registry-allocated, NOT content- @@ -1260,7 +1264,16 @@ async fn route_modified_otlp_sketches_to_precompute( // policy". spatial_filter_canonical: String::new(), }; - let agg_kind_canonical = agg_kind.canonical_string(); + let agg_kind_canonical = match frame_identity.as_ref() { + Some(frame) => format!( + "{}|output:{}:{}:{}", + agg_kind.canonical_string(), + frame.plan_id, + frame.plan_version, + frame.materialization.as_u64() + ), + None => agg_kind.canonical_string(), + }; let definition = frame_identity .as_ref() .map(|frame| frame.materialization) @@ -1340,6 +1353,7 @@ async fn route_modified_otlp_sketches_to_precompute( sketch_algorithm_for(&dp), &dp.container_config, &dp.attrs.keys().cloned().collect(), + Some(frame.materialization), ) }); if observed_policy != frame.materialization.fingerprint() { @@ -1418,6 +1432,7 @@ async fn route_modified_otlp_sketches_to_precompute( algorithm.clone(), &cfg, &group_by_keys, + frame_identity.as_ref().map(|frame| frame.materialization), ); // Per-item dimension (item_label) the controller threaded // into the matched policy's parameters — recorded on the sid @@ -1991,38 +2006,47 @@ fn sketch_config_to_params( params } -/// Look up the policy fingerprint for a freshly-ingested OTLP sketch -/// by content-matching against the streaming-config registry. -/// -/// Sketches arrive with `(metric, attrs, sketch_kind, sketch_config)` -/// embedded in the DP but no policy reference. The matching pass: -/// snapshots the current streaming config, derives a -/// `PolicyRegistry`, and asks `find_policy_by_content` for the -/// fingerprint of a policy whose contents match. Returns -/// `PolicyFingerprint::UNSET` when: -/// 1. An unsupported planner algorithm reached this path -/// (defensive — shouldn't happen). -/// 2. No policy in the registry matches. -/// 3. Multiple policies match (would-have-been-a-bug case; -/// `find_policy_by_content` returns `None` on ambiguity). -/// -/// Callers register the sid with the returned fp regardless of -/// success — UNSET sids are simply absent from the policy_fp → -/// {sids} reverse index, and remain reachable via the legacy -/// `instances_matching(metric, gbk)` walk. +// Snapshot and delta bases are scoped to the producer, never only its semantics. +fn bound_sketch_series_key( + name: &str, + labels: &HashMap, + frame: Option<&asap_types::producer_plan::SummaryFrameIdentity>, +) -> String { + let key = format_series_key(name, labels); + match frame { + Some(frame) => format!( + "{}:{}:{}:{key}", + frame.plan_id, + frame.plan_version, + frame.materialization.as_u64() + ), + None => key, + } +} + +/// Resolve the framed deployed output, then verify its content contract. +/// Unframed input uses legacy content matching and must have exactly one match. +/// An absent or ambiguous match returns UNSET and cannot authorize bound writes. fn derive_sketch_policy_fp( ingest_state: &IngestState, metric: &str, kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, cfg: &crate::storage_engines::sketch_db::data::SketchConfig, group_by_keys: &std::collections::BTreeSet, + bound_output: Option, ) -> asap_types::PolicyFingerprint { let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { return asap_types::PolicyFingerprint::UNSET; }; let params = sketch_config_to_params(cfg); let snap = ingest_state.config_snapshot(); - let index = asap_types::RoutingIndex::build(snap.policy_registry()); + let registry = snap.policy_registry(); + let registry = if let Some(output) = bound_output { + asap_types::PolicyRegistry::from_configs(registry.get(output.fingerprint()).cloned()) + } else { + registry + }; + let index = asap_types::RoutingIndex::build(registry); index .find_policy_by_content(metric, group_by_keys, agg_type, ¶ms) .unwrap_or(asap_types::PolicyFingerprint::UNSET) @@ -2303,7 +2327,7 @@ fn preflight_summary_frames( decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; } else { - let series_key = format_series_key(canonical_name, &dp.attrs); + let series_key = bound_sketch_series_key(canonical_name, &dp.attrs, Some(&frame)); let (mut base, base_window_start) = ingest_state .sketch_snapshots .get(&series_key) @@ -2352,6 +2376,7 @@ fn preflight_summary_frames( sketch_algorithm_for(&dp), &dp.container_config, &dp.attrs.keys().cloned().collect(), + Some(frame.materialization), ) }; if observed != frame.materialization.fingerprint() { diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 7459c6e24..5411f0a10 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -433,9 +433,9 @@ impl PrometheusRemoteWriteReceiver { } for start in starts { let (start_ms, end_ms) = manager.stored_bucket_bounds(start); - for summary_definition_id in &affected { + for stored_output_id in &affected { coordinates.insert(asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: *summary_definition_id, + stored_output_id: *stored_output_id, time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values: labels.clone(), }); @@ -992,7 +992,7 @@ mod tests { endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), @@ -1040,8 +1040,10 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; + use asap_types::{AggregationType, KeyByLabelNames, AggregationConfig}; let aggregation = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -1167,8 +1169,10 @@ mod tests { use asap_types::enums::WindowKind; use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; - let config = - |aggregation_type, grouping: Vec, aggregated: Vec| AggregationConfig { + let config = |aggregation_type, grouping: Vec, aggregated: Vec| { + AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -1628,9 +1632,12 @@ mod tests { let binding = asap_types::query_plan::MaterializationBinding { full_window_slide_ms: None, materialization: asap_types::PolicyFingerprint(policy).into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( - asap_types::PolicyFingerprint(policy).into(), - ), + stored_output_reference: ingest + .summary_store + .summary_catalog_snapshot() + .unwrap() + .output_reference(asap_types::PolicyFingerprint(policy).into()) + .unwrap(), output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 1608d3e09..9ba8e6b2e 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2528,7 +2528,7 @@ mod tests { endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), @@ -3536,6 +3536,8 @@ aggregations: for marker in active_agg_ids { let metric = format!("metric_{marker}"); let cfg = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -6018,7 +6020,7 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon .producers .iter() .map(|producer| { - let definition = asap_types::sds::SummaryDefinitionId::from(producer.materialization); + let definition = asap_types::sds::StoredOutputId::from(producer.materialization); active .precompute_plan .schemas diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index ecf778a12..9218db907 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -750,7 +750,7 @@ async fn main() -> Result<()> { endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), diff --git a/data_plane/src/precompute_engine/coordination_checkpoint.rs b/data_plane/src/precompute_engine/coordination_checkpoint.rs index 249fad538..873725505 100644 --- a/data_plane/src/precompute_engine/coordination_checkpoint.rs +++ b/data_plane/src/precompute_engine/coordination_checkpoint.rs @@ -377,7 +377,7 @@ fn invalid(message: impl Into) -> io::Error { #[cfg(test)] mod tests { use super::*; - use asap_types::{sds::HalfOpenTimeRange, sds::SummaryDefinitionId, PolicyFingerprint}; + use asap_types::{sds::HalfOpenTimeRange, sds::StoredOutputId, PolicyFingerprint}; use std::collections::BTreeMap; fn generation() -> CatalogGeneration { @@ -399,7 +399,7 @@ mod tests { fn coordinates() -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 10, diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 47593276b..38c43b07d 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -21,7 +21,7 @@ struct Population { struct Observations { generation: Option, populations: BTreeMap, - extent: BTreeMap, + extent: BTreeMap, total_keys: usize, metadata_bytes: usize, invalid: Option, @@ -97,7 +97,7 @@ impl RuntimeErpObserver { } let extent = state .extent - .entry(coordinates.summary_definition_id) + .entry(coordinates.stored_output_id) .or_insert((timestamp_ms, timestamp_ms)); extent.0 = extent.0.min(timestamp_ms); extent.1 = extent.1.max(timestamp_ms); @@ -133,7 +133,7 @@ impl RuntimeErpObserver { let population = state.populations.entry(id).or_insert_with(|| Population { source: format!( "summary-definition:{}", - coordinates.summary_definition_id.as_u64() + coordinates.stored_output_id.as_u64() ), coordinates, semantics, @@ -178,7 +178,7 @@ impl RuntimeErpObserver { return Ok(()); } let mut groups: BTreeMap< - (SummaryDefinitionId, i64, i64), + (StoredOutputId, i64, i64), ( String, String, @@ -188,14 +188,14 @@ impl RuntimeErpObserver { > = BTreeMap::new(); for (id, population) in &state.populations { let c = &population.coordinates; - let Some((first, last)) = state.extent.get(&c.summary_definition_id) else { + let Some((first, last)) = state.extent.get(&c.stored_output_id) else { continue; }; if c.time_range.start_ms < *first || c.time_range.end_ms > *last { continue; } let key = ( - c.summary_definition_id, + c.stored_output_id, c.time_range.start_ms, c.time_range.end_ms, ); @@ -207,7 +207,7 @@ impl RuntimeErpObserver { ErpPopulationObservations { schema_version: 1, catalog_generation: generation.clone(), - summary_definition_id: c.summary_definition_id, + stored_output_id: c.stored_output_id, observed_at_unix_ms: now_ms, window_start_ms: c.time_range.start_ms, window_end_ms: c.time_range.end_ms, @@ -294,7 +294,7 @@ mod tests { } fn coordinate(group: usize) -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: asap_types::PolicyFingerprint(1).into(), + stored_output_id: asap_types::PolicyFingerprint(1).into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index 7f5e0a847..da79aad47 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -45,7 +45,7 @@ pub enum FrameLineageError { struct FrameLineageKey { plan_id: u64, plan_version: u64, - materialization: asap_types::sds::SummaryDefinitionId, + materialization: asap_types::sds::StoredOutputId, series_identity: String, producer_id: String, producer_epoch: String, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 44f7d67db..393aab5b8 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -70,7 +70,7 @@ type PendingOutput = ( enum MaintenanceInputs<'a> { Live { - definition: asap_types::sds::SummaryDefinitionId, + definition: asap_types::sds::StoredOutputId, state: SummaryState, }, Frozen(&'a [crate::storage_engines::sketch_db::index::FrozenExactWindows]), @@ -91,7 +91,7 @@ impl MaintenanceInputs<'_> { fn frozen_population_value( inputs: &[crate::storage_engines::sketch_db::index::FrozenExactWindows], - definition: asap_types::sds::SummaryDefinitionId, + definition: asap_types::sds::StoredOutputId, family: Option, complete: bool, ) -> Result, String> { @@ -133,7 +133,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { node: &ExecutableDagNode, ) -> Result, String> { let definition = match self.binding.node(node.id) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Ok(None), }; let family = node.output_schema.fields.iter().find_map(|field| { @@ -206,9 +206,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { ); } let target = match self.binding.node(node.id) { - Some(BackendNodeBinding::Materialization { summary_definition }) => { - summary_definition - } + Some(BackendNodeBinding::Materialization { stored_output }) => stored_output, _ => { return Err( "maintenance SummaryAgg lacks installed materialization binding".into(), @@ -705,7 +703,7 @@ fn prepare_frozen_maintenance_sink( > { installed.validate()?; let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("immutable sink lacks a materialization binding".into()), }; let config = configs @@ -744,10 +742,10 @@ fn prepare_frozen_maintenance_sink( .nodes .iter() .filter_map(|(node, binding)| match binding { - BackendNodeBinding::Materialization { summary_definition } - if expected_input.inputs.contains(summary_definition) => + BackendNodeBinding::Materialization { stored_output } + if expected_input.inputs.contains(stored_output) => { - Some((*node, *summary_definition)) + Some((*node, *stored_output)) } _ => None, }) @@ -768,7 +766,7 @@ fn prepare_frozen_maintenance_sink( let key = MaterializationCommitKey { plan_id: generation.plan_id, plan_version: generation.plan_version, - summary_definition: target, + stored_output: target, window_start_ms: i64::try_from(output_window.0) .map_err(|_| "output window exceeds timestamp range")?, window_end_ms: i64::try_from(output_window.1) @@ -859,7 +857,7 @@ pub fn execute_completed_maintenance( group: &BTreeMap, ) -> Result { let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("maintenance sink lacks an installed output identity".into()), }; let derived = configs @@ -895,14 +893,14 @@ pub(crate) fn execute_completed_maintenance_cohort( installed: &asap_types::executable_plan::InstalledPostAsapDag, configs: &[asap_types::PrecomputeMaterialization], sink: PostAsapNodeId, - source_sids: &BTreeMap, + source_sids: &BTreeMap, target_sid: u64, window: (u64, u64), group: &BTreeMap, ) -> Result { use asap_types::executable_plan::BackendNodeBinding; let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("maintenance sink lacks an installed output identity".into()), }; let target_config = configs @@ -1078,7 +1076,7 @@ fn execute_finite_source_cohort( .as_ref() .ok_or("finite maintenance requires a catalog generation")?; let Some(BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }) = installed.binding.node(sink) else { return Err("finite maintenance sink has no installed definition".into()); @@ -1238,7 +1236,7 @@ fn execute_finite_complete_populations( generation: &Arc, ) -> Result<(), String> { let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("complete maintenance target is not bound".into()), }; let config = plan @@ -1425,7 +1423,7 @@ pub(crate) fn execute_finite_maintenance( for installed in plan.executable_dags.values() { for sink in &installed.binding.precompute_sinks { let Some(BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }) = installed.binding.node(*sink) else { continue; @@ -1573,7 +1571,7 @@ struct CommittedState { struct CommitRegistryState { generation: Option<(u64, u64)>, entries: BTreeMap, - frontiers: BTreeMap, + frontiers: BTreeMap, pending_batch: Option<[u8; 32]>, batch_has_published: bool, admitted_keys: BTreeSet, @@ -1590,7 +1588,7 @@ impl CommitRegistryState { if !self.admitted_keys.contains(key) && self .frontiers - .get(&key.summary_definition) + .get(&key.stored_output) .is_some_and(|(latest, horizon)| { key.window_end_ms <= latest.saturating_sub(i64::try_from(*horizon).unwrap_or(i64::MAX)) @@ -1680,7 +1678,7 @@ impl CommitRegistry { } let frontier = state .frontiers - .entry(key.summary_definition) + .entry(key.stored_output) .or_insert((key.window_end_ms, *horizon)); frontier.0 = frontier.0.max(key.window_end_ms); frontier.1 = frontier.1.max(*horizon); @@ -1688,7 +1686,7 @@ impl CommitRegistry { let frontiers = state.frontiers.clone(); state.entries.retain(|key, _| { frontiers - .get(&key.summary_definition) + .get(&key.stored_output) .is_none_or(|(latest, horizon)| { key.window_end_ms > latest.saturating_sub(i64::try_from(*horizon).unwrap_or(i64::MAX)) @@ -1799,13 +1797,13 @@ impl MaintenanceDagSink { output: PrecomputedOutput, state: Box, ) -> Result, String> { - let source_definition: asap_types::sds::SummaryDefinitionId = output.policy_fp.into(); + let source_definition: asap_types::sds::StoredOutputId = output.policy_fp.into(); let source: SummaryState = Arc::from(state); let mut derived = Vec::new(); let mut matched = false; let mut lineage = Sha256::new(); lineage.update(b"asap-maintenance-lineage-v1"); - let definition_bytes = source_definition.0 .0.to_be_bytes(); + let definition_bytes = source_definition.0.to_be_bytes(); lineage.update(definition_bytes); if let Some(input) = &output.input_revision { if input.generation.plan_id != plan.plan_id() @@ -1834,7 +1832,7 @@ impl MaintenanceDagSink { .binding .nodes .iter() - .filter_map(|(id, binding)| matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source_definition).then_some(*id)) + .filter_map(|(id, binding)| matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == source_definition).then_some(*id)) .collect::>(); if source_nodes.is_empty() { continue; @@ -1851,9 +1849,9 @@ impl MaintenanceDagSink { // Derived summaries consume complete immutable windows at the // completion barrier, never additive worker fragments. if matches!(installed.binding.node(*sink_node), - Some(BackendNodeBinding::Materialization { summary_definition }) + Some(BackendNodeBinding::Materialization { stored_output }) if plan.precompute_plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() + config.policy_fingerprint() == stored_output.fingerprint() && config.derived_input.is_some())) { continue; @@ -1867,8 +1865,8 @@ impl MaintenanceDagSink { !has_input && matches!( installed.binding.node(*node), - Some(BackendNodeBinding::Materialization { summary_definition }) - if *summary_definition != source_definition + Some(BackendNodeBinding::Materialization { stored_output }) + if *stored_output != source_definition ) }); if foreign_source { @@ -1893,15 +1891,13 @@ impl MaintenanceDagSink { } matched = true; let target = match installed.binding.node(*sink_node) { - Some(BackendNodeBinding::Materialization { summary_definition }) => { - *summary_definition - } + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("precompute sink lacks materialization binding".into()), }; let key = MaterializationCommitKey { plan_id: plan.plan_id(), plan_version: plan.plan_version(), - summary_definition: target, + stored_output: target, window_start_ms: output.start_timestamp as i64, window_end_ms: output.end_timestamp as i64, input_lineage: lineage.clone(), @@ -2109,21 +2105,21 @@ impl OutputSink for MaintenanceDagSink { /// semantic dependencies rather than assuming source and output identities match. pub(crate) fn affected_materializations( plan: &asap_types::precompute_plan::PrecomputePlan, - source: asap_types::sds::SummaryDefinitionId, -) -> BTreeSet { + source: asap_types::sds::StoredOutputId, +) -> BTreeSet { use asap_types::executable_plan::BackendNodeBinding; let mut affected = BTreeSet::from([source]); for installed in plan.executable_dags.values() { let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { - matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) + matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == source).then_some(*node) }).collect::>(); let mut frontier = reachable.iter().copied().collect::>(); while let Some(producer) = frontier.pop() { for edge in &installed.document.edges { let immutable = matches!(installed.binding.node(edge.consumer), - Some(BackendNodeBinding::Materialization { summary_definition }) + Some(BackendNodeBinding::Materialization { stored_output }) if plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() + config.policy_fingerprint() == stored_output.fingerprint() && config.derived_input.is_some())); if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { frontier.push(edge.consumer); @@ -2132,10 +2128,10 @@ pub(crate) fn affected_materializations( } for sink in &installed.binding.precompute_sinks { if reachable.contains(sink) { - if let Some(BackendNodeBinding::Materialization { summary_definition }) = + if let Some(BackendNodeBinding::Materialization { stored_output }) = installed.binding.nodes.get(sink) { - affected.insert(*summary_definition); + affected.insert(*stored_output); } } } @@ -2152,7 +2148,7 @@ mod tests { WindowEdgeCompatibility, }; - fn definition(value: u64) -> asap_types::sds::SummaryDefinitionId { + fn definition(value: u64) -> asap_types::sds::StoredOutputId { asap_types::PolicyFingerprint(value).into() } @@ -2293,10 +2289,10 @@ mod tests { let binding = BackendExecutableBinding { nodes: [(1, definition(1)), (2, definition(2)), (3, definition(3))] .into_iter() - .map(|(id, summary_definition)| { + .map(|(id, stored_output)| { ( PostAsapNodeId(id), - BackendNodeBinding::Materialization { summary_definition }, + BackendNodeBinding::Materialization { stored_output }, ) }) .collect(), @@ -2399,14 +2395,14 @@ mod tests { ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: source_definition, + stored_output: source_definition, }, ), (PostAsapNodeId(2), BackendNodeBinding::MaintenanceInput), ( PostAsapNodeId(3), BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }, ), ]), @@ -2506,7 +2502,7 @@ mod tests { scheduled_binding.nodes.insert( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: source_definition, + stored_output: source_definition, }, ); scheduled_binding @@ -2528,7 +2524,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 1, plan_version: 1, - summary_definition: target, + stored_output: target, window_start_ms: 0, window_end_ms: 1000, input_lineage: vec![1], @@ -2572,7 +2568,7 @@ mod tests { durable_binding.nodes.insert( PostAsapNodeId(3), BackendNodeBinding::Materialization { - summary_definition: durable_configs[1].policy_fingerprint().into(), + stored_output: durable_configs[1].policy_fingerprint().into(), }, ); let installed = InstalledPostAsapDag { @@ -2605,7 +2601,7 @@ mod tests { let generation = store.active_catalog_generation().unwrap(); for ((start, end), value) in [((0, 1000), 2.0), ((1000, 2000), 7.0)] { let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source_definition, + stored_output_id: source_definition, time_range: asap_types::sds::HalfOpenTimeRange { start_ms: start, end_ms: end, @@ -2947,14 +2943,14 @@ mod tests { ); document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; let mut binding = binding.clone(); - for (node, summary_definition) in [ + for (node, stored_output) in [ (1, first_id), (5, second_id), (3, target.policy_fingerprint().into()), ] { binding.nodes.insert( PostAsapNodeId(node), - BackendNodeBinding::Materialization { summary_definition }, + BackendNodeBinding::Materialization { stored_output }, ); } binding @@ -3028,7 +3024,7 @@ mod tests { BTreeMap::new() }; let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: config.policy_fingerprint().into(), + stored_output_id: config.policy_fingerprint().into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: start, end_ms: start + 2000, @@ -3613,7 +3609,7 @@ mod tests { nodes: BTreeMap::from([( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: config.policy_fingerprint().into(), + stored_output: config.policy_fingerprint().into(), }, )]), query_sink: PostAsapNodeId(1), @@ -3678,7 +3674,7 @@ mod tests { nodes: BTreeMap::from([( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: config.policy_fingerprint().into(), + stored_output: config.policy_fingerprint().into(), }, )]), query_sink: PostAsapNodeId(1), @@ -3724,7 +3720,7 @@ mod tests { let key = |end| MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: end - 10, window_end_ms: end, input_lineage: vec![0; 32], @@ -3759,7 +3755,7 @@ mod tests { let key = |end| MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: end - 10, window_end_ms: end, input_lineage: vec![0; 32], @@ -3798,7 +3794,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(target), + stored_output: definition(target), window_start_ms: 0, window_end_ms: 10, input_lineage: vec![0; 32], @@ -3874,13 +3870,13 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Materialization { - summary_definition: definition(1), + stored_output: definition(1), }, ), ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: target_definition, + stored_output: target_definition, }, ), ( @@ -4015,7 +4011,7 @@ mod tests { ( PostAsapNodeId(id), BackendNodeBinding::Materialization { - summary_definition: definition(if id == 0 { 1 } else { id as u64 + 1 }), + stored_output: definition(if id == 0 { 1 } else { id as u64 + 1 }), }, ) }) @@ -4044,7 +4040,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 2, - summary_definition: definition(4), + stored_output: definition(4), window_start_ms: 0, window_end_ms: 10, input_lineage: b"batch:1".to_vec(), @@ -4086,13 +4082,13 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Materialization { - summary_definition: definition(1), + stored_output: definition(1), }, ), ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: definition(2), + stored_output: definition(2), }, ), ( @@ -4119,7 +4115,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 2, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: 0, window_end_ms: 10, input_lineage: b"batch:1".to_vec(), diff --git a/data_plane/src/precompute_engine/multisource_coordinator.rs b/data_plane/src/precompute_engine/multisource_coordinator.rs index d429d774a..1afe3a407 100644 --- a/data_plane/src/precompute_engine/multisource_coordinator.rs +++ b/data_plane/src/precompute_engine/multisource_coordinator.rs @@ -24,7 +24,7 @@ pub struct LogicalSourcePartition { #[serde(deny_unknown_fields)] pub struct CoordinatedInput { pub input_node_id: String, - pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub stored_output_id: asap_types::sds::StoredOutputId, pub partitions: BTreeSet, } @@ -36,7 +36,7 @@ pub struct MultiSourceNodeSpec { pub consumer_node_id: String, /// The content-addressed installed output binds its source/window contract. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_definition: Option, + pub output_definition: Option, pub inputs: Vec, /// Named output grouping. An empty projection represents one global group. pub output_grouping: Vec, @@ -122,7 +122,7 @@ impl MultiSourceCoordinator { let supplied: BTreeSet<_> = spec .inputs .iter() - .map(|input| input.summary_definition_id) + .map(|input| input.stored_output_id) .collect(); if &supplied != expected || supplied.len() != spec.inputs.len() @@ -143,15 +143,13 @@ impl MultiSourceCoordinator { let source = plan .materializations .iter() - .find(|config| { - config.policy_fingerprint() == input.summary_definition_id.fingerprint() - }) + .find(|config| config.policy_fingerprint() == input.stored_output_id.fingerprint()) .ok_or_else(|| invalid("coordinator input is not installed"))?; sources.push(source); let producers: Vec<_> = plan .producers .iter() - .filter(|producer| producer.materialization == input.summary_definition_id) + .filter(|producer| producer.materialization == input.stored_output_id) .collect(); if producers.is_empty() || producers @@ -201,7 +199,7 @@ impl MultiSourceCoordinator { .installed_plan .as_ref() .unwrap() - .validate_watermark_scope(input.summary_definition_id, &barrier) + .validate_watermark_scope(input.stored_output_id, &barrier) .map_err(|error| invalid(error.to_string()))?; } } @@ -265,7 +263,7 @@ impl MultiSourceCoordinator { .iter() .filter(|input| input.partitions.contains(&logical(&barrier.source))) { - plan.validate_watermark_scope(input.summary_definition_id, &barrier) + plan.validate_watermark_scope(input.stored_output_id, &barrier) .map_err(|error| invalid(error.to_string()))?; } } @@ -347,7 +345,7 @@ impl MultiSourceCoordinator { .iter() .find(|requirement| requirement.input_node_id == input.input_node_id) .ok_or_else(|| invalid("staged input node is not required"))?; - if input.coordinates.summary_definition_id != requirement.summary_definition_id + if input.coordinates.stored_output_id != requirement.stored_output_id || !requirement.partitions.contains(&logical(&input.source)) { return Err(invalid( @@ -359,7 +357,7 @@ impl MultiSourceCoordinator { .materializations .iter() .find(|config| { - config.policy_fingerprint() == requirement.summary_definition_id.fingerprint() + config.policy_fingerprint() == requirement.stored_output_id.fingerprint() }) .ok_or_else(|| invalid("staged input definition is not installed"))?; let window = input.coordinates.time_range; @@ -544,12 +542,12 @@ mod tests { inputs: vec![ CoordinatedInput { input_node_id: "left".into(), - summary_definition_id: PolicyFingerprint(1).into(), + stored_output_id: PolicyFingerprint(1).into(), partitions: BTreeSet::from([partition("0")]), }, CoordinatedInput { input_node_id: "right".into(), - summary_definition_id: PolicyFingerprint(2).into(), + stored_output_id: PolicyFingerprint(2).into(), partitions: BTreeSet::from([partition("1")]), }, ], @@ -569,7 +567,7 @@ mod tests { }, instance_id: SummaryInstanceId::new(format!("{node}-{epoch}")).unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: PolicyFingerprint(definition).into(), + stored_output_id: PolicyFingerprint(definition).into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 10, @@ -656,7 +654,7 @@ mod tests { }); spec.inputs.push(CoordinatedInput { input_node_id: format!("input-{ordinal}"), - summary_definition_id: definition, + stored_output_id: definition, partitions: partition_ids .into_iter() .map(|partition_id| LogicalSourcePartition { @@ -695,7 +693,7 @@ mod tests { }, instance_id: SummaryInstanceId::new(key.clone()).unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: requirement.summary_definition_id, + stored_output_id: requirement.stored_output_id, time_range: HalfOpenTimeRange { start_ms: start, end_ms: start + 60000, diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index a710e57a9..820bb7a21 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -208,7 +208,7 @@ impl SketchStoreSink { return false; }; let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: output.policy_fp.into(), + stored_output_id: output.policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values, }; @@ -389,6 +389,8 @@ mod tests { // via `PolicyFingerprint::from_config`. Callers obtain the id // via `config.policy_fp_u64()`. AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 8d20c07f5..3d75003e8 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -12,7 +12,7 @@ use std::{ pub struct MaterializationCommitKey { pub plan_id: u64, pub plan_version: u64, - pub summary_definition: asap_types::sds::SummaryDefinitionId, + pub stored_output: asap_types::sds::StoredOutputId, pub window_start_ms: i64, pub window_end_ms: i64, /// Producer lineage identity, including source and immutable input payload. @@ -67,11 +67,11 @@ where R: PrecomputeOperatorRegistry, S: IdempotentCommitSink, { - if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { summary_definition }) if *summary_definition == key.summary_definition) + if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { stored_output }) if *stored_output == key.stored_output) { return Err(ScheduleError::Invalid(format!( "commit key materialization {:?} does not match sink {}", - key.summary_definition, sink_node.0 + key.stored_output, sink_node.0 ))); } binding @@ -210,8 +210,7 @@ mod tests { ( PostAsapNodeId(id), BackendNodeBinding::Materialization { - summary_definition: asap_types::PolicyFingerprint(u64::from(id) + 1) - .into(), + stored_output: asap_types::PolicyFingerprint(u64::from(id) + 1).into(), }, ) }) @@ -319,7 +318,7 @@ mod tests { MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: asap_types::PolicyFingerprint(u64::from(node_id) + 1).into(), + stored_output: asap_types::PolicyFingerprint(u64::from(node_id) + 1).into(), window_start_ms: 10, window_end_ms: 20, input_lineage: b"checkpoint:3".to_vec(), @@ -491,7 +490,7 @@ mod tests { ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: asap_types::PolicyFingerprint(2).into(), + stored_output: asap_types::PolicyFingerprint(2).into(), }, ), ] diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 2c48006f4..438dcdb2c 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -608,7 +608,7 @@ impl Worker { observer.observe( &revision.generation, asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: policy_fp.into(), + stored_output_id: policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: bucket_start, end_ms: bucket_end, @@ -657,7 +657,7 @@ impl Worker { observer.observe( &revision.generation, asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: policy_fp.into(), + stored_output_id: policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: bucket_start, end_ms: bucket_end, diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 87c751e06..e402f6343 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -558,7 +558,7 @@ mod tests { config.pane_origin_ms = Some(0); config.table_timestamp_column = Some("timestamp_ms".into()); let sds = SummaryCatalog::from_materializations(41, 1, &[config.clone()]).unwrap(); - let materialization = *sds.definitions.keys().next().unwrap(); + let materialization = *sds.outputs.keys().next().unwrap(); let read = QueryNodeId(0); let readout = QueryNodeId(1); let input_schema = relation_schema(&[ @@ -590,7 +590,7 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization, - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition(materialization), + stored_output_reference: sds.output_reference(materialization).unwrap(), output_grouping: PhysicalGrouping::Reduce(Vec::new()), item_labels: Vec::new(), window_ms: 1_000, @@ -784,7 +784,7 @@ mod tests { &["fixture".into()], ) .unwrap(); - precompute.summary_catalog = Some(sds.reference().unwrap()); + precompute.bind_catalog(&sds).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope.clone(), &precompute, diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 19486299c..4e3bb7f31 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; -use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; +use asap_types::sds::{StoredOutputId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; use asap_types::AggregationType; @@ -21,10 +21,10 @@ fn miss(reason: impl Into) -> EngineError { /// Borrow descriptors; never reconstruct them from bindings or store payloads. pub(crate) fn resolve( catalog: &SummaryCatalog, - id: SummaryDefinitionId, + id: StoredOutputId, ) -> Result, EngineError> { let identity = catalog - .definitions + .outputs .get(&id) .ok_or_else(|| miss(format!("unknown materialization {}", id.fingerprint().0)))?; let summary = catalog @@ -155,7 +155,7 @@ pub(crate) fn validate_payload( QueryPlanNode::SummaryMerge { inputs } => { let mut ids = BTreeSet::new(); for input in inputs { - let children: &BTreeSet = states + let children: &BTreeSet = states .get(input) .ok_or_else(|| miss("summary merge has no state input"))?; if children.is_empty() { @@ -167,7 +167,7 @@ pub(crate) fn validate_payload( } QueryPlanNode::SummaryEstimate { input, .. } | QueryPlanNode::ExactReadout { input, .. } => { - let ids: &BTreeSet = states + let ids: &BTreeSet = states .get(input) .ok_or_else(|| miss("readout has no state input"))?; if ids.is_empty() || ids.iter().any(|id| !resolved[id].supports(node)) { @@ -256,11 +256,11 @@ mod tests { fn resolves_without_descriptor_copies() { let bundle = fixture(); let catalog = &bundle.summary_catalog; - let id = *catalog.definitions.keys().next().unwrap(); + let id = *catalog.outputs.keys().next().unwrap(); let result = resolve(catalog, id).unwrap(); assert!(std::ptr::eq( result.summary, - &catalog.summary_descriptors[&catalog.definitions[&id].summary_descriptor_id] + &catalog.summary_descriptors[&catalog.outputs[&id].summary_descriptor_id] )); let mut broken = catalog.clone(); broken.data_descriptors.clear(); @@ -270,8 +270,8 @@ mod tests { #[test] fn rejects_operator_fidelity_mismatch_during_resolution() { let mut catalog = catalog_fixture(); - let id = *catalog.definitions.keys().next().unwrap(); - let descriptor_id = catalog.definitions[&id].summary_descriptor_id.clone(); + let id = *catalog.outputs.keys().next().unwrap(); + let descriptor_id = catalog.outputs[&id].summary_descriptor_id.clone(); catalog .summary_descriptors .get_mut(&descriptor_id) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 0ba9c966c..b15da276b 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -960,10 +960,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: MATERIALIZATION.into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - MATERIALIZATION.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference( + &store, + MATERIALIZATION.into(), + ), output_grouping: PhysicalGrouping::Reduce(vec!["job".into()]), window_ms: AT, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 59bffd71d..103247fdf 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -194,10 +194,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - policy.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 0254b0254..366517767 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -154,7 +154,7 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { .catalog .as_ref() .and_then(|catalog| { - let definition = catalog.definitions.get(&binding.materialization)?; + let definition = catalog.outputs.get(&binding.materialization)?; catalog .data_descriptors .get(&definition.data_descriptor_id)? @@ -882,10 +882,10 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - config.policy_fingerprint().into(), - ), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, @@ -1126,16 +1126,16 @@ mod tests { } } let idx = SketchStore::new(); + idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) + .unwrap(); idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config( + config, + ), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, @@ -1196,16 +1196,14 @@ mod tests { let config = &plan.precompute_plan.materializations[0]; let policy = config.policy_fingerprint(); let idx = SketchStore::new(); + idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) + .unwrap(); idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config(config), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, @@ -1321,10 +1319,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - policy.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1422,10 +1420,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), - stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - policy.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 13a19d368..f3015afb8 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -453,13 +453,28 @@ impl QueryExecutionContext<'_> { use asap_types::query_plan::PhysicalGrouping; if binding.stored_output_reference.validate().is_err() - || binding.stored_output_reference.definition_id != binding.materialization + || binding.stored_output_reference.stored_output_id != binding.materialization { return Err(SummaryExecutorError::Unsupported( "read binding has invalid stored output", )); } + let catalog = + self.index + .summary_catalog_snapshot() + .ok_or(SummaryExecutorError::Unsupported( + "bound read requires installed SDS definitions", + ))?; + let expected = catalog + .output_reference(binding.materialization) + .map_err(|_| SummaryExecutorError::Unsupported("stored output is not installed"))?; + if binding.stored_output_reference != expected { + return Err(SummaryExecutorError::Unsupported( + "bound read semantic identity differs from installed output", + )); + } + let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { start_ms: i64::try_from(self.t0_ms).map_err(|_| { @@ -1461,7 +1476,7 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( asap_types::PolicyFingerprint(7).into(), ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, @@ -1899,9 +1914,7 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: Some(20_000), materialization: fp.into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( - fp.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference(&index, fp.into()), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 60_000, @@ -1968,9 +1981,7 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: None, materialization: fp.into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( - fp.into(), - ), + stored_output_reference: super::super::test_plan::bound_reference(&index, fp.into()), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 8027ccdc6..563d85a1b 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -61,9 +61,14 @@ pub(super) fn entry( .then_some(config.slide_interval * 1000), materialization: config.policy_fingerprint().into(), stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( - config.policy_fingerprint().into(), - ), + asap_types::summary_catalog::SummaryCatalog::from_materializations( + 1, + 1, + &[config.clone()], + ) + .unwrap() + .output_reference(config.policy_fingerprint().into()) + .unwrap(), output_grouping: grouping, item_labels: config.aggregated_labels.labels.clone(), window_ms: config.stored_window_ms(), @@ -104,7 +109,7 @@ pub(super) fn install( .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); - precompute.summary_catalog = Some(catalog.reference().unwrap()); + precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -158,3 +163,31 @@ pub(super) fn engine( .with_sketch_index(index) .with_active_physical_plan(active) } + +/// Low-level operator fixtures install the descriptors of their explicit states. +/// Process tests use compiled publications instead of this fixture adapter. +pub(super) fn bound_reference( + index: &SketchStore, + output: asap_types::sds::StoredOutputId, +) -> asap_types::sds::StoredOutputReference { + if let Some(catalog) = index.summary_catalog_snapshot() { + return catalog.output_reference(output).unwrap(); + } + let metadata = index.snapshot_instances(); + let entries = metadata + .iter() + .filter(|m| !m.policy_fp.is_unset()) + .map(|m| { + let (summary, data) = index.descriptors_for_series_id(m.sid).unwrap(); + (m.policy_fp, (*summary).clone(), (*data).clone()) + }) + .collect::>(); + let catalog = asap_types::summary_catalog::SummaryCatalog::build(1, 1, entries).unwrap(); + index + .install_summary_catalog(Arc::new(catalog.clone())) + .unwrap(); + for metadata in metadata { + index.register((*metadata).clone()); + } + catalog.output_reference(output).unwrap() +} diff --git a/data_plane/src/storage_engines/sketch_db/index/admission.rs b/data_plane/src/storage_engines/sketch_db/index/admission.rs index dc30248ee..4ad4a5452 100644 --- a/data_plane/src/storage_engines/sketch_db/index/admission.rs +++ b/data_plane/src/storage_engines/sketch_db/index/admission.rs @@ -1,7 +1,7 @@ //! Store-owned tracking of accepted, not necessarily published summary updates. //! This records known work; it does not infer an event-time watermark. -use asap_types::sds::SummaryDefinitionId; +use asap_types::sds::StoredOutputId; use asap_types::sds::{CatalogGeneration, HalfOpenTimeRange, SummaryInstanceCoordinates}; use std::collections::{BTreeMap, BTreeSet}; @@ -27,7 +27,7 @@ pub(super) struct AdmissionInventory { revision: u64, windows: BTreeMap, metadata_bytes: usize, - replay_floors: BTreeMap, + replay_floors: BTreeMap, observed_extent: Option, finite_input: FiniteInputState, published_series: BTreeMap, @@ -70,7 +70,7 @@ impl AdmissionInventory { for coordinate in &coordinates { if self .replay_floors - .get(&coordinate.summary_definition_id) + .get(&coordinate.stored_output_id) .is_some_and(|floor| coordinate.time_range.end_ms <= *floor) { return Err("summary input precedes retained replay horizon".into()); @@ -252,7 +252,7 @@ impl AdmissionInventory { pub(super) fn known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -261,7 +261,7 @@ impl AdmissionInventory { pub(super) fn known_empty_with_layout( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, full_window: bool, @@ -276,7 +276,7 @@ impl AdmissionInventory { .get(&definition) .is_none_or(|floor| range.start_ms >= *floor) && !self.windows.iter().any(|(coordinate, state)| { - coordinate.summary_definition_id == definition + coordinate.stored_output_id == definition && (if full_window { coordinate.time_range == range } else { @@ -297,12 +297,12 @@ impl AdmissionInventory { pub(super) fn has_pending( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, range: HalfOpenTimeRange, full_window: bool, ) -> bool { self.windows.iter().any(|(coordinate, state)| { - coordinate.summary_definition_id == definition + coordinate.stored_output_id == definition && (if full_window { coordinate.time_range == range } else { @@ -315,11 +315,7 @@ impl AdmissionInventory { /// Advancing this configured replay floor also rejects future old admission. /// Pending work is never forgotten because a different series ran ahead. - pub(super) fn retire_completed_before( - &mut self, - definition: SummaryDefinitionId, - frontier_ms: i64, - ) { + pub(super) fn retire_completed_before(&mut self, definition: StoredOutputId, frontier_ms: i64) { let floor = self.replay_floors.entry(definition).or_insert(i64::MIN); *floor = (*floor).max(frontier_ms); let frontier_ms = *floor; @@ -329,7 +325,7 @@ impl AdmissionInventory { .flat_map(|state| state.pending.iter().copied()) .collect(); self.windows.retain(|coordinate, state| { - let remove = coordinate.summary_definition_id == definition + let remove = coordinate.stored_output_id == definition && coordinate.time_range.end_ms <= frontier_ms && state.published >= state.admitted && !pending.contains(&state.admitted); @@ -365,7 +361,7 @@ mod tests { } fn window(series: &str) -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(asap_types::PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(asap_types::PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -385,9 +381,9 @@ mod tests { .admit(&generation, BTreeSet::from([a.clone(), b.clone()])) .unwrap(); inventory.acknowledge(&generation, &a, revision).unwrap(); - assert!(inventory.has_pending(a.summary_definition_id, a.time_range, false)); + assert!(inventory.has_pending(a.stored_output_id, a.time_range, false)); inventory.acknowledge(&generation, &b, revision).unwrap(); - assert!(!inventory.has_pending(a.summary_definition_id, a.time_range, false)); + assert!(!inventory.has_pending(a.stored_output_id, a.time_range, false)); } #[test] @@ -407,12 +403,8 @@ mod tests { .acknowledge(&generation, &coordinate, first) .unwrap(); assert_ne!(before, inventory.revision()); - assert!(inventory.has_pending( - coordinate.summary_definition_id, - coordinate.time_range, - false - )); - inventory.retire_completed_before(coordinate.summary_definition_id, 1000); + assert!(inventory.has_pending(coordinate.stored_output_id, coordinate.time_range, false)); + inventory.retire_completed_before(coordinate.stored_output_id, 1000); assert_eq!(inventory.windows.len(), 1); inventory .record_series(&generation, &coordinate, 42) @@ -420,7 +412,7 @@ mod tests { inventory .acknowledge(&generation, &coordinate, second) .unwrap(); - inventory.retire_completed_before(coordinate.summary_definition_id, 1000); + inventory.retire_completed_before(coordinate.stored_output_id, 1000); assert!(inventory.windows.is_empty()); assert_eq!( inventory.published_frontiers().get(&42), @@ -485,12 +477,12 @@ mod tests { let revision = inventory .admit(&generation, BTreeSet::from([current.clone(), future])) .unwrap(); - assert!(inventory.has_pending(current.summary_definition_id, current.time_range, true)); + assert!(inventory.has_pending(current.stored_output_id, current.time_range, true)); inventory .acknowledge(&generation, ¤t, revision) .unwrap(); - assert!(inventory.has_pending(current.summary_definition_id, current.time_range, false)); - assert!(!inventory.has_pending(current.summary_definition_id, current.time_range, true)); + assert!(inventory.has_pending(current.stored_output_id, current.time_range, false)); + assert!(!inventory.has_pending(current.stored_output_id, current.time_range, true)); } // A neighboring full snapshot may overlap an empty query population. @@ -525,28 +517,28 @@ mod tests { .unwrap(); } assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true )); inventory.seal_finite(&generation).unwrap(); - assert!(!inventory.known_empty(first.summary_definition_id, 1, other.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 1, other.time_range)); assert!(inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true )); assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 2, other.time_range, true )); - inventory.retire_completed_before(first.summary_definition_id, 2000); + inventory.retire_completed_before(first.stored_output_id, 2000); assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true @@ -576,13 +568,13 @@ mod tests { inventory .acknowledge(&generation, &second, revision) .unwrap(); - assert!(!inventory.known_empty(first.summary_definition_id, 1, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 1, second.time_range)); inventory.seal_finite(&generation).unwrap(); - assert!(inventory.known_empty(first.summary_definition_id, 1, second.time_range)); - assert!(!inventory.known_empty(first.summary_definition_id, 2, second.time_range)); - assert!(!inventory.known_empty(first.summary_definition_id, 999, second.time_range)); + assert!(inventory.known_empty(first.stored_output_id, 1, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 2, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 999, second.time_range)); assert!(!inventory.known_empty( - first.summary_definition_id, + first.stored_output_id, 1, HalfOpenTimeRange { start_ms: 2000, diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index c1bc5f220..199b643d5 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -8,7 +8,7 @@ use crate::storage_engines::types::AggregateCore; pub(crate) struct FrozenExactWindows { pub(crate) sid: u64, - pub(crate) definition: SummaryDefinitionId, + pub(crate) definition: StoredOutputId, pub(crate) generation: Arc, pub(crate) group: BTreeMap, pub(crate) windows: BTreeMap<(u64, u64), Arc>, @@ -49,10 +49,10 @@ impl SketchStore { pub(crate) fn read_frozen_exact_cohort( &self, generation: &Arc, - expected_definitions: &BTreeSet, + expected_definitions: &BTreeSet, requests: &[( u64, - SummaryDefinitionId, + StoredOutputId, BTreeSet<(u64, u64)>, BTreeMap, )], @@ -89,7 +89,7 @@ impl SketchStore { pub(crate) fn read_complete_raw_maintenance_cohort( &self, generation: &Arc, - definitions: &BTreeSet, + definitions: &BTreeSet, window: (u64, u64), ) -> Result { if definitions.is_empty() || window.0 >= window.1 { @@ -113,7 +113,7 @@ impl SketchStore { fn durable_maintenance_population_ids( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, ) -> Result, String> { let metadata = self .persistence_metadata @@ -126,7 +126,7 @@ impl SketchStore { .load_strict() .map_err(|error| error.to_string())? .into_iter() - .filter(|record| !record.removed && record.summary_definition_id == Some(definition)) + .filter(|record| !record.removed && record.stored_output_id == Some(definition)) .map(|record| record.sid) .collect()) } @@ -135,7 +135,7 @@ impl SketchStore { /// coverage and incarnation before reading or publishing any state. pub(crate) fn completed_maintenance_coordinates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, ) -> Result, BTreeSet<(u64, u64)>>>, String> { @@ -147,7 +147,7 @@ impl SketchStore { /// bind them to the current catalog. pub(crate) fn complete_raw_maintenance_population( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, ) -> Result, BTreeSet<(u64, u64)>>>, String> { @@ -156,7 +156,7 @@ impl SketchStore { fn maintenance_coordinates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, require_complete_population: bool, ) -> Result, BTreeSet<(u64, u64)>>>, String> @@ -260,7 +260,7 @@ impl SketchStore { pub(crate) fn read_frozen_exact_windows( &self, sid: u64, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &Arc, expected_windows: &BTreeSet<(u64, u64)>, group: &BTreeMap, @@ -435,7 +435,7 @@ impl SketchStore { if !finite_complete { return Err("complete raw publication requires the original finite closure".into()); } - let mut supplied = BTreeMap::>::new(); + let mut supplied = BTreeMap::>::new(); for input in cohort.inputs() { supplied .entry(input.definition) @@ -775,7 +775,7 @@ mod tests { let definition = config.policy_fingerprint().into(); let population = BTreeMap::from([("instance".to_string(), index.to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: definition, + stored_output_id: definition, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -891,7 +891,7 @@ mod tests { for (index, config) in configs.iter().take(2).enumerate() { let definition = config.policy_fingerprint().into(); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: definition, + stored_output_id: definition, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -926,7 +926,7 @@ mod tests { // Only the first population has the next window: completeness must // reject the partial cohort even after all writes are durably sealed. let extra_coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: configs[0].policy_fingerprint().into(), + stored_output_id: configs[0].policy_fingerprint().into(), time_range: HalfOpenTimeRange { start_ms: 1000, end_ms: 2000, @@ -1116,7 +1116,7 @@ mod tests { for (instance, value) in [("a", 5.0), ("b", 15.0)] { let population = BTreeMap::from([("instance".to_string(), instance.to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source.policy_fingerprint().into(), + stored_output_id: source.policy_fingerprint().into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, @@ -1315,7 +1315,7 @@ mod tests { let mut restored = restarted.start_persistence(restart_config).unwrap(); let population = BTreeMap::from([("instance".to_string(), "new".to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source_id, + stored_output_id: source_id, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index c45f274b9..e099f0a21 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -25,7 +25,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use asap_types::sds::{ CatalogGeneration, HalfOpenTimeRange, InstanceCompleteness, InstanceLifecycle, - ObservedSummaryInventory, SummaryDefinitionId, SummaryInstance, SummaryInstanceStatus, + ObservedSummaryInventory, StoredOutputId, SummaryInstance, SummaryInstanceStatus, SummaryPlacement, SummaryStateReference, }; use asap_types::PolicyFingerprint; @@ -698,7 +698,7 @@ pub struct SketchStore { u64, ( Option>, - Option, + Option, ), >, >, @@ -866,6 +866,11 @@ impl SketchStore { catalog: Arc, ) -> Result<(), String> { let reference = catalog.reference().map_err(|error| error.to_string())?; + if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { + writer + .persist_catalog(&catalog) + .map_err(|e| e.to_string())?; + } let mut inventory = self.admission.write().unwrap(); let generation = CatalogGeneration { schema_version: reference.schema_version, @@ -901,11 +906,10 @@ impl SketchStore { .descriptors .authoritative_catalog() .ok_or("summary admission requires an installed catalog")?; - if coordinates.iter().any(|coordinate| { - !catalog - .definitions - .contains_key(&coordinate.summary_definition_id) - }) { + if coordinates + .iter() + .any(|coordinate| !catalog.outputs.contains_key(&coordinate.stored_output_id)) + { return Err("summary admission references an uninstalled definition".into()); } self.admission @@ -950,7 +954,7 @@ impl SketchStore { .time_range .end_ms .saturating_sub(i64::try_from(replay_horizon_ms).unwrap_or(i64::MAX)); - inventory.retire_completed_before(coordinate.summary_definition_id, floor); + inventory.retire_completed_before(coordinate.stored_output_id, floor); Ok(()) } @@ -1030,7 +1034,7 @@ impl SketchStore { pub(crate) fn summary_window_known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -1048,7 +1052,7 @@ impl SketchStore { /// Overlapping neighboring snapshots do not establish population in this one. pub(crate) fn full_summary_window_known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -1079,7 +1083,7 @@ impl SketchStore { pub(crate) fn has_pending_summary_updates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, range: HalfOpenTimeRange, full_window: bool, ) -> bool { @@ -1198,7 +1202,7 @@ impl SketchStore { &self, reporter_id: &str, storage_node_id: &str, - producers: &BTreeMap, + producers: &BTreeMap, inventory_version: u64, observed_at_ms: i64, ) -> Result { @@ -1220,17 +1224,17 @@ impl SketchStore { if !Self::instance_visible_in_generation(binding, Some(&generation)) { continue; } - let summary_definition_id = SummaryDefinitionId::from(binding.metadata.policy_fp); + let stored_output_id = StoredOutputId::from(binding.metadata.policy_fp); if binding.metadata.policy_fp.is_unset() - || !catalog.definitions.contains_key(&summary_definition_id) + || !catalog.outputs.contains_key(&stored_output_id) { continue; } let (stored_output_id, producer_id) = - producers.get(&summary_definition_id).ok_or_else(|| { + producers.get(&stored_output_id).ok_or_else(|| { format!( "materialization {} has no producer in the active PrecomputePlan", - summary_definition_id.as_u64() + stored_output_id.as_u64() ) })?; let store = self @@ -1259,7 +1263,7 @@ impl SketchStore { 0, ); let instance_id = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id, + stored_output_id: *stored_output_id, time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values: group_values.clone(), } @@ -1268,7 +1272,12 @@ impl SketchStore { let instance = SummaryInstance { instance_id: instance_id.clone(), stored_output_id: *stored_output_id, - summary_definition_id, + summary_definition_id: binding + .stored_output_reference + .as_ref() + .ok_or("missing semantic binding")? + .definition_id + .clone(), summary_descriptor_id: binding.summary_descriptor.id().clone(), data_descriptor_id: binding.data_descriptor.id().clone(), time_range: HalfOpenTimeRange { start_ms, end_ms }, @@ -2692,7 +2701,9 @@ impl SketchStore { m.first_seen_unix_ms, ); if !m.policy_fp.is_unset() { - record.summary_definition_id = Some(SummaryDefinitionId::from(m.policy_fp)); + record.stored_output_id = Some(StoredOutputId::from(m.policy_fp)); + record.summary_definition_id = + Some(m.stored_output_reference.as_ref()?.definition_id.clone()); record.catalog_generation = Some(Arc::clone(m.catalog_generation.as_ref()?)); } record.retired_at_ms = m.retired_at_ms; @@ -2795,7 +2806,7 @@ impl SketchStore { pub(crate) fn authorize_series_reactivation( &self, sid: u64, - definition: SummaryDefinitionId, + definition: StoredOutputId, ) -> Result>, String> { if let Some(binding) = self .instances @@ -2814,7 +2825,7 @@ impl SketchStore { .authoritative_snapshot() .ok_or("series reactivation requires an authoritative catalog")?; if binding.metadata.policy_fp != definition.fingerprint() - || !catalog.definitions.contains_key(&definition) + || !catalog.outputs.contains_key(&definition) { return Err("series reactivation differs from its installed definition".into()); } @@ -2834,7 +2845,7 @@ impl SketchStore { .descriptors .authoritative_snapshot() .ok_or("series reactivation requires an authoritative catalog")?; - if *old_definition != Some(definition) || !catalog.definitions.contains_key(&definition) { + if *old_definition != Some(definition) || !catalog.outputs.contains_key(&definition) { return Err("series reactivation does not match the installed materialization".into()); } let old_generation = old_generation @@ -2875,7 +2886,7 @@ impl SketchStore { record .as_ref() .and_then(|value| value.catalog_generation.clone()), - record.and_then(|value| value.summary_definition_id), + record.and_then(|value| value.stored_output_id), ), ); } @@ -3232,6 +3243,9 @@ impl SketchStore { // concurrent lifecycle operation cannot succeed without persistence. let metadata_writer = Arc::new(persistence::metadata::SidMetadataStore::new(&cfg.disk_path)); + if let Some(catalog) = self.descriptors.authoritative_catalog() { + metadata_writer.persist_catalog(&catalog)?; + } // Restore the generation-wide raw admission barrier before exposing // recovered state or accepting another producer after restart. if let Some(closed) = metadata_writer.load_finite_closure()? { @@ -3255,7 +3269,7 @@ impl SketchStore { .map(|record| { ( record.sid, - (record.catalog_generation, record.summary_definition_id), + (record.catalog_generation, record.stored_output_id), ) }), ); @@ -3361,21 +3375,22 @@ impl SketchStore { continue; } let catalog = self.descriptors.authoritative_snapshot(); - let policy_fp = match ( - &rec.summary_definition_id, - &rec.catalog_generation, - &catalog, - ) { + let policy_fp = match (&rec.stored_output_id, &rec.catalog_generation, &catalog) { (Some(definition), Some(generation), Some((catalog, installed_generation))) => { + let persisted = persistence::metadata::SidMetadataStore::new(disk_path) + .load_catalog(generation); if generation != installed_generation - || !catalog.definitions.contains_key(definition) + || persisted.as_ref().ok() != Some(catalog.as_ref()) + || !catalog.outputs.get(definition).is_some_and(|output| { + Some(&output.definition_id) == rec.summary_definition_id.as_ref() + }) { // A new version must allocate a fresh physical series. Otherwise // the persisted resolver can route fresh input back to this // excluded series and its already-completed windows. if generation.plan_id == installed_generation.plan_id && generation.plan_version < installed_generation.plan_version - && catalog.definitions.contains_key(definition) + && catalog.outputs.contains_key(definition) { self.removed_sids .write() @@ -3682,6 +3697,15 @@ mod tests { meta_with_policy(sid, asap_types::PolicyFingerprint::UNSET) } + fn meta_for_config( + sid: u64, + config: &asap_types::PrecomputeMaterialization, + ) -> SummarySeriesMetadata { + let mut metadata = meta_with_policy(sid, config.policy_fingerprint()); + metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(config); + metadata + } + fn meta_with_policy( sid: u64, policy_fp: asap_types::PolicyFingerprint, @@ -3766,12 +3790,24 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(41, fingerprint)); + store.register(meta_for_config(41, &state_config)); store.append_sample( 41, BTreeMap::from([("job".to_string(), "api".to_string())]), @@ -3786,9 +3822,9 @@ mod tests { ); let producers = BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), + StoredOutputId::from(fingerprint), ( - asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) .stored_output_id, "producer-a".to_string(), ), @@ -3799,11 +3835,10 @@ mod tests { inventory.validate().unwrap(); assert_eq!(inventory.instances.len(), 2); let instance = inventory.instances.values().next().unwrap(); - assert_eq!(instance.summary_definition_id.fingerprint(), fingerprint); + assert_eq!(instance.stored_output_id.fingerprint(), fingerprint); assert_eq!( instance.stored_output_id, - asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) - .stored_output_id + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()).stored_output_id ); assert_eq!(instance.status, SummaryInstanceStatus::Ready); assert_eq!(instance.completeness, InstanceCompleteness::Unknown); @@ -3828,7 +3863,7 @@ mod tests { inventory.instances.keys().collect::>(), next_inventory.instances.keys().collect::>() ); - let catalog_identity = &plan.summary_catalog.definitions[&instance.summary_definition_id]; + let catalog_identity = &plan.summary_catalog.outputs[&instance.stored_output_id]; assert_eq!( instance.summary_descriptor_id, catalog_identity.summary_descriptor_id @@ -3849,16 +3884,28 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - store.register(meta_with_policy(42, fingerprint)); + store.register(meta_for_config(42, &state_config)); let producers = BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), + StoredOutputId::from(fingerprint), ( - asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) .stored_output_id, "producer-a".to_string(), ), @@ -5013,8 +5060,20 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); - let metadata = meta_with_policy(507, fingerprint); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); + let metadata = meta_for_config(507, &state_config); let record = SidMetaRecord::new( metadata.sid, metadata.metric_name.clone(), @@ -5032,7 +5091,7 @@ mod tests { assert_eq!(store.register_recovered_disk_series(tmp.path()), 0); assert!(store.instance(507).is_none()); let mut foreign = record; - foreign.summary_definition_id = Some(fingerprint.into()); + foreign.stored_output_id = Some(fingerprint.into()); let reference = plan.summary_catalog.reference().unwrap(); foreign.catalog_generation = Some(Arc::new(CatalogGeneration { schema_version: reference.schema_version, @@ -5055,12 +5114,24 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(509, fingerprint)); + store.register(meta_for_config(509, &state_config)); store.append_sample(509, BTreeMap::new(), (0, 10_000), sample(1)); let mut next = plan.summary_catalog; next.plan_version += 1; @@ -5076,14 +5147,14 @@ mod tests { .is_some(), "live version change must allocate a fresh physical series" ); - let output = asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + let output = asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) .stored_output_id; let inventory = store .observed_summary_inventory( "backend-a", "store-a", &BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), + StoredOutputId::from(fingerprint), (output, "producer-a".into()), )]), 1, @@ -5117,7 +5188,19 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let definition = fingerprint.into(); let mut next_catalog = plan.summary_catalog.clone(); next_catalog.plan_version += 1; @@ -5134,7 +5217,7 @@ mod tests { .unwrap(); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); old_sid = resolver.resolve("metric", "group", "family"); - store.register(meta_with_policy(old_sid, fingerprint)); + store.register(meta_for_config(old_sid, &state_config)); for pane in 0..4 { store.append_sample( old_sid, @@ -5162,7 +5245,7 @@ mod tests { }) .unwrap(); assert_ne!(new_sid, old_sid); - store.register(meta_with_policy(new_sid, fingerprint)); + store.register(meta_for_config(new_sid, &state_config)); for pane in 0..4 { store.append_sample( new_sid, @@ -5218,20 +5301,32 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(850, fingerprint)); + store.register(meta_for_config(850, &state_config)); let generation = store.active_catalog_generation().unwrap(); let writer = Arc::new(persistence::metadata::SidMetadataStore::new( directory.path(), )); *store.persistence_metadata.write().unwrap() = Some(writer.clone()); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: fingerprint.into(), + stored_output_id: fingerprint.into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 30_000, @@ -5269,7 +5364,7 @@ mod tests { let producers = BTreeMap::from([( fingerprint.into(), ( - asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) .stored_output_id, "producer".to_string(), ), @@ -5313,21 +5408,33 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); { let store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(851, fingerprint)); + store.register(meta_for_config(851, &state_config)); let mut config = durable_cfg(directory.path().to_path_buf()); config.hot_window_ms = None; config.seal_window_count = 100; let mut persistence = store.start_persistence(config).unwrap(); let generation = store.active_catalog_generation().unwrap(); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: fingerprint.into(), + stored_output_id: fingerprint.into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 30_000, @@ -5423,7 +5530,19 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); let disk = directory.path().to_path_buf(); let expected_retirement; @@ -5433,7 +5552,7 @@ mod tests { .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); for sid in [801, 802, 803] { - store.register(meta_with_policy(sid, fingerprint)); + store.register(meta_for_config(sid, &state_config)); } let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for sid in [801, 802, 803] { @@ -5460,7 +5579,7 @@ mod tests { expected_retirement = store.force_retire(801, Duration::from_secs(3600)).unwrap(); assert!(store.force_expire(802).is_some()); assert!(store.remove_instance(803).is_some()); - store.register(meta_with_policy(803, fingerprint)); + store.register(meta_for_config(803, &state_config)); assert!( store.instance(803).is_none(), "removed SID reused before restart" @@ -5490,7 +5609,7 @@ mod tests { recovered.instance(803).is_none(), "removed state resurrected" ); - recovered.register(meta_with_policy(803, fingerprint)); + recovered.register(meta_for_config(803, &state_config)); assert!( recovered.instance(803).is_none(), "removed SID reused after restart" @@ -5507,19 +5626,30 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); - let definition_id = SummaryDefinitionId::from(fingerprint); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); + let definition_id = StoredOutputId::from(fingerprint); let producers = BTreeMap::from([( definition_id, ( - asap_types::sds::StoredOutputReference::for_definition(definition_id) - .stored_output_id, + asap_types::sds::StoredOutputReference::for_output(definition_id).stored_output_id, "producer-a".to_string(), ), )]); let tmp = tempfile::TempDir::new().unwrap(); let disk = tmp.path().to_path_buf(); - let mut metadata = meta_with_policy(506, fingerprint); + let mut metadata = meta_for_config(506, &state_config); metadata.group_by_keys = ["job".to_string()].into_iter().collect(); { diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index d7de8c2bd..668aa99e9 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -202,6 +202,8 @@ mod tests { fn sum_agg_config(id: u64) -> AggregationConfig { AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs index dcce818b9..4ea55e319 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -34,6 +34,7 @@ fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> Persist || current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() + || current.stored_output_id != record.stored_output_id || current.summary_definition_id != record.summary_definition_id || current.catalog_generation != record.catalog_generation || current.metric_name != record.metric_name diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 552883c1c..9360d4eb1 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -304,6 +304,8 @@ pub struct SidMetaRecord { pub sid: u64, /// Authoritative identity and provenance, absent on legacy sidecars. #[serde(default)] + pub stored_output_id: Option, + #[serde(default)] pub summary_definition_id: Option, #[serde(default)] pub catalog_generation: Option>, @@ -341,6 +343,7 @@ impl SidMetaRecord { ) -> Self { Self { sid, + stored_output_id: None, summary_definition_id: None, catalog_generation: None, metric_name, @@ -412,6 +415,8 @@ struct DataDescriptorRec { struct SidBindingRec { sid: u64, #[serde(default)] + stored_output_id: Option, + #[serde(default)] summary_definition_id: Option, #[serde(default)] catalog_generation_sha256: Option, @@ -449,7 +454,7 @@ impl SdsSidecar { use crate::storage_engines::sketch_db::sds::{data_descriptor_id, summary_descriptor_id}; let mut sidecar = Self { - schema_version: 3, + schema_version: 4, catalog_generations: HashMap::new(), summary_descriptors: HashMap::new(), data_descriptors: HashMap::new(), @@ -492,6 +497,7 @@ impl SdsSidecar { record.sid.to_string(), SidBindingRec { sid: record.sid, + stored_output_id: record.stored_output_id, summary_definition_id: record.summary_definition_id, catalog_generation_sha256: generation_sha256, summary_descriptor_id: summary_id, @@ -533,6 +539,7 @@ impl SdsSidecar { })?; Ok(SidMetaRecord { sid: binding.sid, + stored_output_id: binding.stored_output_id, summary_definition_id: binding.summary_definition_id, catalog_generation: binding .catalog_generation_sha256 @@ -585,6 +592,67 @@ impl SidMetadataStore { } } + /// Definitions are durable before any output may refer to this generation. + pub(crate) fn persist_catalog( + &self, + catalog: &asap_types::summary_catalog::SummaryCatalog, + ) -> PersistResult<()> { + let reference = catalog + .reference() + .map_err(|e| PersistError::Format(e.to_string()))?; + let _writer = self + .writer + .lock() + .map_err(|_| PersistError::Internal("SID metadata writer poisoned".into()))?; + let path = self.path.with_file_name(format!( + "sds-definitions-{}.json", + reference.snapshot_sha256 + )); + if path.exists() { + let existing = self.load_catalog(&reference)?; + if existing != *catalog { + return Err(PersistError::Format( + "immutable SDS definitions changed".into(), + )); + } + return Ok(()); + } + let bytes = + serde_json::to_vec(catalog).map_err(|e| PersistError::Serialize(e.to_string()))?; + Self::write_atomic_at(&path, &bytes) + } + + pub(crate) fn load_catalog( + &self, + generation: &asap_types::sds::CatalogGeneration, + ) -> PersistResult { + if generation.snapshot_sha256.len() != 64 + || !generation + .snapshot_sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(PersistError::Format("invalid SDS generation digest".into())); + } + let path = self.path.with_file_name(format!( + "sds-definitions-{}.json", + generation.snapshot_sha256 + )); + let catalog: asap_types::summary_catalog::SummaryCatalog = + serde_json::from_slice(&fs::read(path)?) + .map_err(|e| PersistError::Format(e.to_string()))?; + if catalog + .reference() + .map_err(|e| PersistError::Format(e.to_string()))? + != *generation + { + return Err(PersistError::Format( + "persisted SDS definitions differ from generation".into(), + )); + } + Ok(catalog) + } + /// One bounded generation checkpoint closes raw producers across restart. /// Catalog activation with another generation does not inherit this seal. pub(crate) fn persist_finite_closure( @@ -647,7 +715,7 @@ impl SidMetadataStore { }; if matches!( value.get("schema_version").and_then(|v| v.as_u64()), - Some(2 | 3) + Some(2 | 3 | 4) ) { let sidecar: SdsSidecar = match serde_json::from_value(value) { Ok(sidecar) => sidecar, @@ -739,7 +807,7 @@ impl SidMetadataStore { let value: serde_json::Value = serde_json::from_slice(&bytes) .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; if let Some(version) = value.get("schema_version") { - if !matches!(version.as_u64(), Some(2 | 3)) { + if !matches!(version.as_u64(), Some(2 | 3 | 4)) { return Err(PersistError::Format( "unsupported SID metadata version".into(), )); @@ -845,6 +913,26 @@ mod tests { assert!(s.load().unwrap().is_empty()); } + // Restart must validate the immutable semantic document, not just its filename. + #[test] + fn persisted_definitions_roundtrip_and_reject_tampering() { + let directory = tempfile::tempdir().unwrap(); + let store = SidMetadataStore::new(directory.path()); + let catalog = asap_types::summary_catalog::SummaryCatalog::build(7, 2, []).unwrap(); + store.persist_catalog(&catalog).unwrap(); + let generation = catalog.reference().unwrap(); + assert_eq!(store.load_catalog(&generation).unwrap(), catalog); + let path = store.path.with_file_name(format!( + "sds-definitions-{}.json", + generation.snapshot_sha256 + )); + let mut altered = serde_json::to_value(&catalog).unwrap(); + altered["plan_version"] = serde_json::json!(3); + std::fs::write(&path, serde_json::to_vec(&altered).unwrap()).unwrap(); + assert!(store.load_catalog(&generation).is_err()); + assert!(store.persist_catalog(&catalog).is_err()); + } + #[test] fn authoritative_bindings_share_one_persisted_catalog_generation() { let directory = tempfile::tempdir().unwrap(); @@ -856,7 +944,7 @@ mod tests { snapshot_sha256: "catalog".into(), }); let mut first = sketch_meta(1); - first.summary_definition_id = Some(asap_types::PolicyFingerprint(7).into()); + first.stored_output_id = Some(asap_types::PolicyFingerprint(7).into()); first.catalog_generation = Some(std::sync::Arc::clone(&generation)); let mut second = first.clone(); second.sid = 2; @@ -871,7 +959,7 @@ mod tests { records[1].catalog_generation.as_ref().unwrap() )); assert_eq!( - records[0].summary_definition_id, + records[0].stored_output_id, Some(asap_types::PolicyFingerprint(7).into()) ); } @@ -890,7 +978,7 @@ mod tests { let persisted: serde_json::Value = serde_json::from_slice(&std::fs::read(s.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); + assert_eq!(persisted["schema_version"], 4); assert_eq!( persisted["summary_descriptors"].as_object().unwrap().len(), 2 @@ -950,7 +1038,7 @@ mod tests { store.upsert_all(&[exact_meta(2)]).unwrap(); let persisted: serde_json::Value = serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); + assert_eq!(persisted["schema_version"], 4); assert_eq!(store.load().unwrap().len(), 2); } diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index ffefbd250..ca7f7ec2d 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -75,12 +75,55 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { }) } +// A valid deployment reference does not authorize bytes in another state format. +fn operator_matches(summary: &SummaryDescriptor, actual: &AggKind) -> bool { + match &summary.operator { + SummaryOperator::Configured { + aggregation_type, + aggregation_sub_type, + parameters, + .. + } => { + let config = asap_types::PrecomputeMaterialization::new( + *aggregation_type, + aggregation_sub_type.clone(), + parameters.clone().into_iter().collect(), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::WindowKind::Tumbling, + String::new(), + String::new(), + None, + None, + None, + ); + super::data::agg_kind_for_config(&config).operator_canonical_string() + == actual.operator_canonical_string() + } + SummaryOperator::ExactAgg { + agg_type, + parameters_canonical, + } => matches!(actual, + AggKind::ExactAgg { agg_type: actual, parameters_canonical: parameters, .. } if actual == agg_type && parameters == parameters_canonical), + SummaryOperator::LegacyPartial { operator_canonical } => { + actual.operator_canonical_string() == *operator_canonical + } + // Bound runtime plans carry the complete Configured state contract. + SummaryOperator::Sketch { .. } => false, + } +} + /// Runtime foreign-key binding from one SeriesId to shared descriptors. Every pane /// row stored under the SeriesId is a Summary Instance: `(binding, interval, /// group-values, state)`. Descriptor references are normalized here instead of /// copied into every pane row. #[derive(Debug, Clone)] pub struct SdsBinding { + pub stored_output_reference: Option, pub metadata: Arc, pub summary_descriptor: Arc, pub data_descriptor: Arc, @@ -156,15 +199,19 @@ impl SummaryDescriptorRegistry { "materialization identity is required by the installed SummaryCatalog".into(), ); } - let materialization = asap_types::sds::SummaryDefinitionId::from(metadata.policy_fp); - let identity = catalog.definitions.get(&materialization).ok_or_else(|| { + let materialization = asap_types::sds::StoredOutputId::from(metadata.policy_fp); + let identity = catalog.outputs.get(&materialization).ok_or_else(|| { format!( "materialization {} is absent from the installed SummaryCatalog", materialization.as_u64() ) })?; + let summary = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !operator_matches(summary, &metadata.agg_kind) { + return Err("stored output state format differs from installed definition".into()); + } Some(( - catalog.summary_descriptors[&identity.summary_descriptor_id].clone(), + summary.clone(), catalog.data_descriptors[&identity.data_descriptor_id].clone(), )) } else { @@ -205,7 +252,16 @@ impl SummaryDescriptorRegistry { } }; + let stored_output_reference = authoritative + .as_ref() + .map(|(catalog, _)| { + catalog + .output_reference(metadata.policy_fp.into()) + .map_err(|e| e.to_string()) + }) + .transpose()?; Ok(SdsBinding { + stored_output_reference, metadata: Arc::new(metadata), summary_descriptor, data_descriptor, @@ -382,7 +438,7 @@ mod tests { let summary = SummaryDescriptor::new( SummaryOperator::ExactAgg { agg_type: AggregationType::Sum, - parameters_canonical: "authoritative=true".into(), + parameters_canonical: "pane=5000;".into(), }, FidelityGuarantee::Exact, 1, @@ -402,8 +458,14 @@ mod tests { let registry = SummaryDescriptorRegistry::default(); registry.install_catalog(Arc::new(catalog)).unwrap(); + assert!( + registry + .bind(metadata(1, "wrong-local-copy", "", AggregationType::Max, 7)) + .is_err(), + "an output ID must not authorize a different state family" + ); let binding = registry - .bind(metadata(1, "wrong-local-copy", "", AggregationType::Max, 7)) + .bind(metadata(1, "cpu", "", AggregationType::Sum, 7)) .unwrap(); assert_eq!(binding.summary_descriptor.as_ref(), &summary); assert_eq!(binding.data_descriptor.as_ref(), &data); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index d42457fe7..3d6a8683a 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -714,7 +714,7 @@ mod tests { endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: true, - require_summary_definition_identity: true, + require_stored_output_identity: true, require_registered_producer: true, }, schemas: Vec::new(), diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 5158398bd..3769f23af 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -90,6 +90,8 @@ pub fn create_engine_single_pop_with_aggregated( let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -176,6 +178,8 @@ pub fn create_engine_dual_input( // Value aggregation let value_agg_config = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: value_agg_type, aggregation_sub_type: String::new(), @@ -206,6 +210,8 @@ pub fn create_engine_dual_input( // Keys aggregation let keys_agg_config = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: key_agg_type, aggregation_sub_type: String::new(), @@ -301,6 +307,8 @@ pub fn create_engine_two_metrics( let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config_a = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: aggregation_type_a, aggregation_sub_type: String::new(), @@ -330,6 +338,8 @@ pub fn create_engine_two_metrics( materializations_by_policy_fingerprint.insert(id_a, agg_config_a); let agg_config_b = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: aggregation_type_b, aggregation_sub_type: String::new(), @@ -435,6 +445,8 @@ pub fn create_engine_three_metrics( (aggregation_type_c, &labels_c, metric_c), ] { let cfg = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: agg_type, aggregation_sub_type: String::new(), @@ -517,6 +529,8 @@ pub fn create_engine_multi_timestamp( let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -591,6 +605,8 @@ pub fn create_engine_multi_timestamp_with_window( let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index b86830b94..59fc84659 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -8,7 +8,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -187,39 +186,17 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: differential_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), - ) - .unwrap(); - let install = physical_fixture::artifact(&runtime); + let install = physical_fixture::artifact_from_materializations(vec![ddsketch_config()]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); + let bootstrap = output_dir.path().join("bootstrap.json"); + std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) .arg("--streaming-config") - .arg(config.path()) + .arg(&bootstrap) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") @@ -360,3 +337,194 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { ); } } + +fn ddsketch_config() -> asap_types::PrecomputeMaterialization { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + PrecomputeMaterialization::new( + AggregationType::DDSketch, + String::new(), + std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(ALPHA))]), + KeyByLabelNames::new(vec!["service".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 1, + 1, + WindowKind::Tumbling, + String::new(), + METRIC.into(), + None, + None, + None, + ) +} + +// A real process must range-read the bound output, never a semantic substitute. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn bound_sds_keeps_hot_and_rebuild_outputs_and_groups_isolated() { + use asap_types::query_plan::QueryPlanNode; + use asap_types::sds::StoredOutputId; + let mut hot = ddsketch_config(); + hot.stored_output_id = Some(StoredOutputId(1001)); + let mut rebuild = hot.clone(); + rebuild.stored_output_id = Some(StoredOutputId(1002)); + let mut install = physical_fixture::artifact_from_materializations(vec![hot, rebuild]); + assert_eq!(install.summary_catalog.definitions.len(), 1); + assert_eq!(install.summary_catalog.outputs.len(), 2); + for entry in install.query_plan.entries.values_mut() { + let output = if entry.canonical_query.contains("0.99") { + 1002 + } else { + 1001 + }; + for node in entry.nodes.values_mut() { + if let QueryPlanNode::ReadMaterialization { binding } = node { + binding.materialization = StoredOutputId(output); + } + } + } + install + .query_plan + .bind_catalog(&install.summary_catalog) + .unwrap(); + let query_port = unused_port(); + let otlp_http_port = unused_port(); + let directory = tempfile::tempdir().unwrap(); + let plan = directory.path().join("plan.json"); + std::fs::write(&plan, serde_json::to_vec(&install).unwrap()).unwrap(); + let bootstrap = directory.path().join("bootstrap.json"); + std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); + let mut child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--physical-plan") + .arg(&plan) + .arg("--streaming-config") + .arg(&bootstrap) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(directory.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(otlp_http_port.to_string()) + .arg("--otel-grpc-port") + .arg(unused_port().to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{query_port}"); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut child.0).await; + let end = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; + let send = |output: u64, second: u64, value: f64, service: &'static str| { + let mut stamp_plan = install.clone(); + stamp_plan + .precompute_plan + .schemas + .retain(|s| s.materialization == StoredOutputId(output)); + let client = client.clone(); + async move { + let bytes = ddsketch_export(METRIC, second * 1_000_000_000, &[value; 32]); + let mut message = ExportMetricsServiceRequest::decode(bytes.as_slice()).unwrap(); + let Some(Data::Ddsketch(sketch)) = + &mut message.resource_metrics[0].scope_metrics[0].metrics[0].data + else { + panic!("fixture") + }; + sketch.data_points[0].attributes[0].value = Some(AnyValue { + value: Some(any_value::Value::StringValue(service.into())), + }); + physical_fixture::stamp(&mut message, &stamp_plan); + let response = client + .post(format!("http://127.0.0.1:{otlp_http_port}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(message.encode_to_vec()) + .send() + .await + .unwrap(); + let status = response.status(); + assert!( + status.is_success(), + "output {output}: {status}: {}", + response.text().await.unwrap() + ); + } + }; + let query = |quantile: &str, range: &str| { + let expression = format!("quantile_over_time({quantile}, {METRIC}[{range}])"); + let client = client.clone(); + let base = base.clone(); + async move { + get_json( + &client, + &format!("{base}/api/v1/query"), + &[("query", expression), ("time", end.to_string())], + ) + .await + } + }; + for second in end - 2..=end { + send(1002, second, 100.0, SERVICE).await; + } + let mut rebuilt = Value::Null; + for _ in 0..50 { + rebuilt = query("0.99", "3s").await; + if first_instant(&rebuilt).is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_approx( + 100.0, + first_instant(&rebuilt).expect("bound rebuild result").2, + "rebuild", + ); + assert!( + first_instant(&query("0.5", "3s").await).is_none(), + "hot must not substitute rebuild state" + ); + send(1001, end - 2, 1.0, SERVICE).await; + send(1001, end, 1.0, SERVICE).await; + assert!( + first_instant(&query("0.5", "3s").await).is_none(), + "missing middle pane must not be an empty input" + ); + send(1001, end - 1, 1.0, SERVICE).await; + for second in end - 2..=end { + send(1001, second, 200.0, "payments").await; + } + let mut answer = Value::Null; + for _ in 0..50 { + answer = query("0.5", "3s").await; + if answer["data"]["result"] + .as_array() + .is_some_and(|r| r.len() == 2) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let rows = answer["data"]["result"] + .as_array() + .unwrap_or_else(|| panic!("{answer}")); + assert_eq!(rows.len(), 2, "{answer}"); + for row in rows { + let expected = if row["metric"]["service"] == SERVICE { + 1.0 + } else { + assert_eq!(row["metric"]["service"], "payments"); + 200.0 + }; + assert_approx( + expected, + row["value"][1].as_str().unwrap().parse().unwrap(), + "isolated group", + ); + } +} diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index defaf434b..909752886 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -136,7 +136,10 @@ async fn persisted_summary_restarts_without_live_reregistration() { .as_object() .unwrap() .values() - .all(|binding| !binding["summary_definition_id"].is_null() + .all(|binding| !binding["stored_output_id"].is_null() + && binding["summary_definition_id"] + .as_str() + .is_some_and(|id| id.starts_with("sds-v1:")) && !binding["catalog_generation_sha256"].is_null())); drop(first); let port = unused_port(); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index cdc04f320..f4858dc03 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -45,7 +45,7 @@ pub fn artifact_from_materializations( .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); - precompute.summary_catalog = Some(catalog.reference().unwrap()); + precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -131,7 +131,7 @@ pub fn artifact_from_materializations( full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), stored_output_reference: - asap_types::sds::StoredOutputReference::for_definition( + asap_types::sds::StoredOutputReference::for_output( config.policy_fingerprint().into(), ), output_grouping, @@ -160,6 +160,7 @@ pub fn artifact_from_materializations( ); } } + query_plan.bind_catalog(&catalog).unwrap(); PhysicalPlanInstallRequest { summary_catalog: catalog, collector_plans: vec![], @@ -233,7 +234,7 @@ pub fn stamp( ("backend_compat", BACKEND_COMPAT.into()), ( "materialization", - schema.materialization.0.as_u64().to_string(), + schema.materialization.as_u64().to_string(), ), ("schema_id", schema.schema_id.clone()), ("producer_id", "fixture".into()), diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 0fd76e35a..ce3355578 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -4,14 +4,23 @@ removed. The backend now stages and atomically activates one physical-plan envelope containing the authoritative `SummaryCatalog` snapshot plus the `PrecomputePlan`, optional `CollectorPlan`, `TransmissionPlan`, and `QueryPlan` -that reference catalog materialization IDs. - -The target semantic-definition and deployed-output separation is described in -[SDS architecture](../../design_docs/summary-catalog-sds-architecture.md). It is a -design contract, not evidence that the current catalog/storage path already -implements canonical semantic fingerprints or independent stored-output IDs. -The [migration gates](../../design_docs/asapplanner-migration-plan.md#5-bound-query-sds-implementation-across-the-pr-stack) -track the required code and recovery changes. +that reference installed stored outputs and their semantic definitions. + +Catalog schema 4 separates `StoredOutputId` (deployment routing) from +`SummaryDefinitionId` (versioned semantic SHA-256). `StoredOutputReference` +binds both. Planner exports the persisted output's typed semantic dependency +closure; explicit raw summary configurations use a restricted typed description. +Derived outputs require the complete Planner closure at installation. + +Writes must match the installed output's operator and format. Durable metadata +records both identities and the immutable catalog snapshot. Recovery validates +that snapshot before restoring authoritative state. Old payload decoders remain +available, but legacy metadata without semantic identity cannot authorize bound +reads. Reinstall/rebuild those outputs rather than guessing their meaning. + +See [SDS architecture](../../design_docs/summary-catalog-sds-architecture.md) +for the contract and [migration gates](../../design_docs/asapplanner-migration-plan.md#5-bound-query-sds-implementation-across-the-pr-stack) +for the downstream acceptance requirements. Ad-hoc discovery is not implemented. The HTTP lifecycle is exposed through `/api/v1/physical-plan`, `/api/v1/physical-plan/activate`, `/api/v1/physical-plan/discard`, and `/api/v1/physical-plan/status`. From 42602b90b5637da01a4fdbcce6b2d4da16a86261 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 16:58:02 +0000 Subject: [PATCH 07/25] feat(sds): require dataset-bound Planner definitions and versioned catalogs --- Cargo.lock | 11 +- .../examples/calibration_candidates.rs | 17 +-- .../examples/offline_planner_replay.rs | 8 +- control_plane/src/emit/mod.rs | 5 +- control_plane/src/main.rs | 2 + control_plane/src/physical/compiler.rs | 142 ++++++++++++------ .../src/physical/executable_binding.rs | 2 +- control_plane/src/physical/plan_dot.rs | 2 +- .../src/physical/post_asap/cost_model.rs | 4 +- control_plane/src/physical/post_asap/tests.rs | 7 +- .../src/physical/runtime_capability.rs | 8 +- control_plane/src/query_plan.rs | 134 +++++++++-------- control_plane/src/query_plan/residual.rs | 63 +++++--- control_plane/tests/offline_evidence.rs | 2 +- crates/asap_types/src/derived_input.rs | 2 +- crates/asap_types/src/executable_plan.rs | 15 +- crates/asap_types/src/precompute_plan.rs | 36 ++++- .../asap_types/src/precompute_plan/catalog.rs | 9 +- crates/asap_types/src/query_plan.rs | 24 +-- crates/asap_types/src/summary_catalog.rs | 2 +- crates/asap_types/src/summary_semantics.rs | 6 +- .../drivers/ingest/prometheus_remote_write.rs | 4 +- data_plane/src/drivers/query/servers/http.rs | 1 + data_plane/src/main.rs | 1 + .../precompute_engine/maintenance_runtime.rs | 39 ++--- .../src/precompute_engine/subdag_scheduler.rs | 20 ++- .../accelerator.rs | 2 +- .../relational_adapter.rs | 7 +- .../query_engines/asap_query_engine/engine.rs | 2 +- .../asap_query_engine/exact_subqueries.rs | 30 ++-- .../asap_query_engine/logical_dag.rs | 108 ++++++------- .../asap_query_engine/post_asap_readout.rs | 2 +- .../asap_query_engine/summary_exec.rs | 12 +- .../asap_query_engine/summary_executor.rs | 2 +- .../types/hot_reload_config.rs | 1 + .../design_docs/asapplanner-migration-plan.md | 4 +- .../catalog-physical-plan-runtime.md | 21 +-- ...asapquery-compatibility-demo-snapshot.json | 3 +- .../examples/asapquery-planning-snapshot.json | 3 +- tools/o11y-execution/calibrate_runtime.py | 42 +++--- .../o11y-execution/test_calibrate_runtime.py | 42 +++--- 41 files changed, 468 insertions(+), 379 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4dfd33fa..acc50d509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,15 +396,16 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "serde", "serde_json", + "sha2", "thiserror 2.0.20", ] diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 95f7f543b..5b3e3c36d 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -37,17 +37,6 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation vec![lhs, rhs], json!({"operator_debug":format!("{operator:?}"),"timing_debug":format!("{timing:?}")}), ), - SummaryExpr::CandidateTopK { - candidates, - values, - k, - grouping, - completeness, - } => ( - "CandidateTopK", - vec![candidates, values], - json!({"k":k,"grouping_debug":format!("{grouping:?}"),"completeness_debug":format!("{completeness:?}")}), - ), SummaryExpr::ValueOperation { child, operation, @@ -83,11 +72,11 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation left, right, kind, - pred, + pred, pruning, } => ( "RelationalJoin", vec![left, right], - json!({"kind_debug":format!("{kind:?}"),"predicate_debug":format!("{pred:?}")}), + json!({"kind_debug":format!("{kind:?}"),"predicate_debug":format!("{pred:?}"), "pruning_debug":format!("{pruning:?}")}), ), SummaryExpr::SummarySubtract { left, right } => { ("SummarySubtract", vec![left, right], json!({})) @@ -105,7 +94,7 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation vec![summary_input], json!({"query_debug":format!("{query:?}")}), ), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { ("SummaryMerge", children.iter().collect(), json!({})) } }; diff --git a/control_plane/examples/offline_planner_replay.rs b/control_plane/examples/offline_planner_replay.rs index 7a703ce18..78b6e1dd0 100644 --- a/control_plane/examples/offline_planner_replay.rs +++ b/control_plane/examples/offline_planner_replay.rs @@ -59,7 +59,7 @@ fn inspect( inspect(summary_input, model, seen, states, raw) } SummaryExpr::ValueOperation { child, .. } => inspect(child, model, seen, states, raw), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { inspect(child, model, seen, states, raw); } @@ -82,12 +82,6 @@ fn inspect( inspect(lhs, model, seen, states, raw); inspect(rhs, model, seen, states, raw); } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - inspect(candidates, model, seen, states, raw); - inspect(values, model, seen, states, raw); - } } } diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c2ede7ee3..c9908cc9b 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -54,11 +54,8 @@ fn extract_from_node(node: &Rc) -> Option { // `ExactAgg` case. SummaryExpr::SummaryAgg { .. } => None, SummaryExpr::SummaryEstimate { summary_input, .. } => extract_from_node(summary_input), - SummaryExpr::SummaryMerge { children } => children.iter().find_map(extract_from_node), + SummaryExpr::SummaryMerge { children, .. } => children.iter().find_map(extract_from_node), SummaryExpr::ValueOperation { child, .. } => extract_from_node(child), - SummaryExpr::CandidateTopK { - candidates, values, .. - } => extract_from_node(candidates).or_else(|| extract_from_node(values)), // Not surfaced by any `Bind*` path yet (gated on rules that // haven't landed — see `deployment_expr.rs`'s module docs). SummaryExpr::BinaryOp { .. } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index b0b3bf860..d0c759a17 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -199,6 +199,7 @@ struct CompileAndPublishPhysicalPlanRequest { workload_cost_evidence: Option, queries: Vec, data_workload: planner_types::workload::DataWorkload, + dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, #[serde(rename = "collector_ids", alias = "target_collector_ids")] target_collector_ids: Vec, capability_snapshot_id: String, @@ -630,6 +631,7 @@ fn compile_physical_plan_request( retained_summary_memory_budget_bytes: None, }; let environment = physical::compiler::PhysicalDeploymentContext { + dataset_identity: request.dataset_identity, target: request.target, target_collector_ids: request.target_collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index f5af7a460..f235e5433 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -199,6 +199,8 @@ pub struct TopKMembershipEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PhysicalDeploymentContext { + /// Semantic dataset served by this deployment's input channel; never an endpoint. + pub dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, pub target: PhysicalDeploymentTarget, #[serde(rename = "collector_ids")] pub target_collector_ids: Vec, @@ -219,7 +221,7 @@ pub enum PhysicalDeploymentTarget { } /// Startup and candidate-discovery input for backend-local planning. -/// Version 2 is the sole supported schema; deployment always requires quotes. +/// Version 3 requires an explicit logical dataset identity; it is the sole supported schema; deployment always requires quotes. /// Query/data semantics use ASAPPlanner's canonical workload types directly; /// this wrapper adds only backend-owned implementation evidence and lifecycle /// identity required to choose a concrete physical realization. @@ -569,9 +571,9 @@ impl BackendLocalPlanningInput { pub fn into_physical_compilation_request( self, ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { - if self.schema_version != 2 { + if self.schema_version != 3 { return Err(CompileError::Snapshot(format!( - "unsupported workload snapshot version {}; only version 2 is supported", + "unsupported workload snapshot version {}; only version 3 is supported", self.schema_version ))); } @@ -807,7 +809,7 @@ fn has_unsafe_raw_entity_leaf( SummaryExpr::SummaryEstimate { summary_input, .. } => { has_unsafe_raw_entity_leaf(summary_input, selected, false) } - SummaryExpr::SummaryMerge { children } => children + SummaryExpr::SummaryMerge { children, .. } => children .iter() .any(|child| has_unsafe_raw_entity_leaf(child, selected, false)), _ => false, @@ -963,6 +965,10 @@ impl DeploymentPlanCompiler { environment: PhysicalDeploymentContext, frontend: QueryFrontend, ) -> Result { + environment + .dataset_identity + .validate() + .map_err(CompileError::Snapshot)?; if let Some(data) = &request.data_workload { data.validate() .map_err(|error| CompileError::Snapshot(error.to_string()))?; @@ -1480,9 +1486,10 @@ impl DeploymentPlanCompiler { reason: "persisted semantic root is absent".into(), })?; runtime_materialization.semantic_fragment = Some( - asap_types::semantic_fragment::SemanticFragment::from_stored_output( + asap_types::semantic_fragment::SemanticFragment::from_stored_output_in_dataset( &compiled_dag.dag, semantic_root, + environment.dataset_identity.clone(), ) .map_err(|reason| CompileError::Query { query_id: query.query_id.clone(), @@ -2002,6 +2009,7 @@ impl DeploymentPlanCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; + precompute_plan.ingest.dataset_identity = Some(environment.dataset_identity.clone()); let mut transmission_plan = crate::physical::compiler::build_transmission_plan( envelope.clone(), &precompute_plan, @@ -2089,15 +2097,9 @@ fn summary_agg_metric(node: &SummaryNode) -> Option { } } SummaryExpr::SummaryAgg { child, .. } => walk(child, metrics), - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - walk(candidates, metrics); - walk(values, metrics); - } SummaryExpr::ValueOperation { child, .. } => walk(child, metrics), SummaryExpr::SummaryEstimate { summary_input, .. } => walk(summary_input, metrics), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { walk(child, metrics); } @@ -2165,10 +2167,9 @@ fn observed_population_matches_root( || !measures.iter().all(|intent| { matches!( intent, - AggIntent::Cardinality { col: None, .. } - | AggIntent::FrequencyL2 { col: None, .. } + AggIntent::FrequencyL2 { col: None, .. } | AggIntent::FrequencyEntropy { col: None, .. } - ) + ) || matches!(intent, AggIntent::Cardinality { cols, .. } if cols.is_empty()) }) { return false; @@ -2354,7 +2355,7 @@ fn requires_exact_erp_fallback( child: summary_input, .. } => walk(summary_input, out), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { children.iter().for_each(|child| walk(child, out)) } SummaryExpr::SummaryJoin { outer, inner, .. } => { @@ -2371,12 +2372,6 @@ fn requires_exact_erp_fallback( walk(left, out); walk(right, out); } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - walk(candidates, out); - walk(values, out); - } SummaryExpr::KeepPreAsap(_) => {} } } @@ -3241,7 +3236,7 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option Option @@ -3570,8 +3565,10 @@ fn collect_selected_materializations( } } match &node.expr { - SummaryExpr::CandidateTopK { - candidates, values, .. + SummaryExpr::RelationalJoin { + right: candidates, left: values, + kind: planner_types::pre_asap::JoinKind::Semi, + pruning: Some(_), .. } => { walk(candidates, readout, composable, grouping.clone(), selected)?; // In a hybrid TopK, the sketch is only a candidate-membership @@ -3624,7 +3621,7 @@ fn collect_selected_materializations( grouping.clone(), selected, )?, - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { walk(child, readout, composable, grouping.clone(), selected)?; } @@ -3665,16 +3662,6 @@ fn collect_selected_materializations( SummaryInputExpr::Column( planner_types::pre_asap::ColumnRef::SampleValue, ) => "value", - SummaryInputExpr::ResetAwareCounterDelta { .. } - if matches!( - input.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { - proof: planner_types::post_asap::NonNegativeWeightProof::ResetAwareCounterDerivative - } - ) => "counter_delta", - SummaryInputExpr::ResetAwareCounterDelta { .. } => { - return Err("counter-delta TopK input lacks a non-negative reset-aware proof".into()) - } _ => return Err("unsupported TopK SummaryUpdate weight".into()), }; parameters["weight_mode"] = mode.into(); @@ -4257,7 +4244,7 @@ pub(crate) mod tests { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - for version in [0, 1, 3] { + for version in [0, 1, 2, 4] { let mut old = snapshot.clone(); old.schema_version = version; assert!(old @@ -4265,7 +4252,7 @@ pub(crate) mod tests { .into_physical_compilation_request() .unwrap_err() .to_string() - .contains("only version 2")); + .contains("only version 3")); assert!(old.compile_promql().is_err()); } assert!(snapshot.into_physical_compilation_request().is_ok()); @@ -4496,10 +4483,17 @@ pub(crate) mod tests { .compile_promql(request, environment(10_000)) .unwrap(); let entry = plan.query_plan.entries.values().next().unwrap(); - let crate::query_plan::QueryPlanNode::CandidateTopK { inputs, .. } = - &entry.nodes[&entry.root] + let crate::query_plan::QueryPlanNode::Logical { + operator: asap_types::query_plan::residual::ResidualQueryOperator::TopKSelection { .. }, + inputs, + } = &entry.nodes[&entry.root] else { - panic!("Planner weighted TopK must lower to CandidateTopK: {entry:#?}"); + panic!("expected ordinary TopK root") + }; + let crate::query_plan::QueryPlanNode::MembershipFilter { inputs, .. } = + &entry.nodes[&inputs[0]] + else { + panic!("Planner weighted TopK must lower to MembershipFilter: {entry:#?}"); }; assert!(matches!( entry.nodes[&inputs[0]], @@ -4596,7 +4590,14 @@ pub(crate) mod tests { asap_types::AggregationType::CountMinSketchWithHeap ); let entry = plan.query_plan.lookup(query).unwrap(); - let QueryPlanNode::CandidateTopK { inputs, .. } = &entry.nodes[&entry.root] else { + let QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { .. }, + inputs, + } = &entry.nodes[&entry.root] + else { + panic!("expected ordinary TopK root") + }; + let QueryPlanNode::MembershipFilter { inputs, .. } = &entry.nodes[&inputs[0]] else { panic!("expected candidate TopK: {entry:#?}"); }; assert!(matches!( @@ -4637,7 +4638,7 @@ pub(crate) mod tests { .nodes .iter() .all(|node| node.output_state.timing - == planner_types::post_asap::ExecutionTiming::MaintenanceTime)); + == planner_types::post_asap::ExecutionTiming::IngestionTime)); assert_eq!(installed.binding.query_plan_sink, entry.root); let mut mismatched = plan.to_publication_artifact().unwrap(); let projected = mismatched @@ -4747,8 +4748,51 @@ pub(crate) mod tests { ); } + /// Dataset changes alter persisted meaning; relocating the same input does not. + #[test] + fn dataset_identity_survives_binding_and_rejects_wrong_input() { + let compile = |env| { + DeploymentPlanCompiler + .compile_promql(request("q", "sum_over_time(m[1m])"), env) + .unwrap() + }; + let first = compile(environment(10_000)); + assert!(!first.summary_catalog.definitions.is_empty()); + let mut other = environment(10_000); + other.dataset_identity.namespace = "other-tenant".into(); + let second = compile(other); + assert_ne!( + first.summary_catalog.definitions.keys().collect::>(), + second + .summary_catalog + .definitions + .keys() + .collect::>() + ); + let mut relocated = environment(10_000); + relocated.target_collector_ids = vec!["relocated-source".into()]; + let relocated = compile(relocated); + assert_eq!( + first.summary_catalog.definitions, + relocated.summary_catalog.definitions + ); + let mut forged = first.precompute_plan.clone(); + forged.ingest.dataset_identity.as_mut().unwrap().namespace = "other-tenant".into(); + assert!(forged + .validate() + .unwrap_err() + .to_string() + .contains("dataset")); + forged.ingest.dataset_identity = None; + assert!(forged.validate().is_err()); + } + fn environment(now: u64) -> PhysicalDeploymentContext { PhysicalDeploymentContext { + dataset_identity: planner_types::post_asap::LogicalDatasetIdentity { + namespace: "test".into(), + dataset: "metrics".into(), + }, target: PhysicalDeploymentTarget::DistributedCollectors, target_collector_ids: vec!["edge-a".into(), "edge-b".into()], capability_snapshot_id: "caps-7".into(), @@ -5820,7 +5864,7 @@ pub(crate) mod tests { }; request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: selected.clone(), rhs: selected.clone(), operator: planner_types::post_asap::BinaryOperator { @@ -6021,7 +6065,7 @@ pub(crate) mod tests { let right = right.queries[0].selected_plan_root.clone(); let right = Rc::new(SummaryNode { expr: SummaryExpr::ValueOperation { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, child: right.clone(), }, @@ -6030,7 +6074,7 @@ pub(crate) mod tests { }); request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: left.clone(), rhs: right, operator: planner_types::post_asap::BinaryOperator { @@ -7296,6 +7340,7 @@ pub(crate) mod tests { ) .unwrap(); envelope_plan.ingest = IngestContract { + dataset_identity: envelope_plan.ingest.dataset_identity.clone(), protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, @@ -7383,7 +7428,7 @@ pub(crate) mod tests { .queries .remove(0); let snapshot = BackendLocalPlanningInput { - schema_version: 2, + schema_version: 3, workload_cost_evidence: None, query_workload, data_workload, @@ -7652,6 +7697,7 @@ pub(crate) mod tests { }; let merge = Rc::new(SummaryNode { expr: SummaryExpr::SummaryMerge { + timing: planner_types::post_asap::ExecutionTiming::QueryTime, children: vec![left.clone(), right.clone()], }, schema: left.schema.clone(), diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 639dfb344..378967d87 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -19,7 +19,7 @@ pub fn install_selected_dag( precompute_sinks.push(node.id); BackendNodeBinding::Materialization { stored_output } } else if node.output_state.timing - == planner_types::post_asap::ExecutionTiming::MaintenanceTime + == planner_types::post_asap::ExecutionTiming::IngestionTime { BackendNodeBinding::MaintenanceInput } else { diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index 67bad1b79..19d602413 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -154,7 +154,7 @@ fn query_node_label(node: &QueryPlanNode) -> String { QueryPlanNode::SummaryEstimate { query, .. } => format!("SummaryEstimate\n{query:?}"), QueryPlanNode::ExactReadout { readout, .. } => format!("ExactReadout\n{readout:?}"), QueryPlanNode::SummaryMerge { .. } => "SummaryMerge".into(), - QueryPlanNode::CandidateTopK { k, .. } => format!("CandidateTopK\nk={k}"), + QueryPlanNode::MembershipFilter { .. } => "MembershipFilter".into(), QueryPlanNode::ExternalExact { .. } => "ExternalExact".into(), QueryPlanNode::ExactFallback { reason } => format!("ExactFallback\n{reason}"), } diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 571c054c3..5fa6c75d9 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -521,8 +521,8 @@ impl CostModel for ControlPlaneCostModel { fn value_operation_capabilities(&self) -> ValueOperationCapabilities { ValueOperationCapabilities { - read_time: true, - maintenance_time: false, + query_time: true, + ingestion_time: false, } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index e2eec8b92..5e226419d 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -112,13 +112,10 @@ fn node_is_archive(node: &Rc) -> bool { SummaryExpr::SummaryAgg { child, .. } => node_is_archive(child), SummaryExpr::ValueOperation { child, .. } => node_is_archive(child), SummaryExpr::SummaryEstimate { summary_input, .. } => node_is_archive(summary_input), - SummaryExpr::SummaryMerge { children } => children.iter().any(node_is_archive), + SummaryExpr::SummaryMerge { children, .. } => children.iter().any(node_is_archive), SummaryExpr::SummaryJoin { outer, inner, .. } => { node_is_archive(outer) || node_is_archive(inner) } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => node_is_archive(candidates) || node_is_archive(values), SummaryExpr::SummarySubtract { left, right } | SummaryExpr::RelationalJoin { left, right, .. } | SummaryExpr::BinaryOp { @@ -332,7 +329,7 @@ fn uncertified_hll_keeps_exact_execution() { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, measures: vec![AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Epsilon(0.01), }], output_names: Vec::new(), diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 5b9f407c5..136594268 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -462,7 +462,7 @@ mod tests { #[test] fn capability_for_cardinality_with_epsilon_returns_cardinality_approx() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Epsilon(0.01), }; assert_eq!(capability_for(&intent), Some(Capability::CardinalityApprox)); @@ -471,7 +471,7 @@ mod tests { #[test] fn capability_for_cardinality_with_epsilon_delta_returns_cardinality_approx() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::EpsilonDelta { epsilon: 0.01, delta: 0.001, @@ -483,7 +483,7 @@ mod tests { #[test] fn capability_for_cardinality_with_exact_returns_none() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Exact, }; assert_eq!(capability_for(&intent), None); @@ -520,7 +520,7 @@ mod tests { accuracy: approximate.clone(), }); let cardinality = capability_for(&AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: approximate, }); assert_eq!(count, Some(Capability::FrequencyEstimate(None))); diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index db7557c26..84916c422 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -179,7 +179,7 @@ where *input = remap[input]; } } - QueryPlanNode::CandidateTopK { inputs, .. } + QueryPlanNode::MembershipFilter { inputs, .. } | QueryPlanNode::Binary { inputs, .. } | QueryPlanNode::RelationalJoin { inputs, .. } => { for input in inputs { @@ -257,6 +257,7 @@ where right, kind, pred, + pruning: None, } if self.preserve_relational => QueryPlanNode::RelationalJoin { inputs: [self.lower(left)?, self.lower(right)?], join_kind: kind.clone(), @@ -269,9 +270,6 @@ where right_schema: right.schema.clone(), output_schema: node.schema.clone(), }, - SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { - reason: "read-time relational join requires the relational compiler".into(), - }, SummaryExpr::ValueOperation { child, operation, .. } if self.preserve_relational @@ -308,7 +306,7 @@ where .. }, ), - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if measures.len() == 1 => { use planner_types::pre_asap::AggIntent; let operation = match &measures[0] { @@ -367,19 +365,22 @@ where } SummaryExpr::ValueOperation { child: sort, - operation: planner_types::post_asap::ValueOperation::Limit { n, offset: 0 }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + operation: planner_types::post_asap::ValueOperation::Limit { n, offset: 0, partition_by: limit_partition }, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } => { let SummaryExpr::ValueOperation { child, operation: planner_types::post_asap::ValueOperation::Sort { keys, partition_by }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } = &sort.expr else { return Err(QueryPlanError::Invalid( "query-time Limit must consume a query-time Sort".into(), )); }; + if limit_partition != partition_by { + return Err(QueryPlanError::Invalid("TopK Limit and Sort grouping differ".into())); + } if keys.len() != 1 || keys[0].ascending { return Err(QueryPlanError::Invalid( "only descending value-ranked TopK is executable".into(), @@ -434,7 +435,7 @@ where SummaryExpr::ValueOperation { child, operation: planner_types::post_asap::ValueOperation::Sort { keys, .. }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if keys.len() == 1 => QueryPlanNode::Logical { operator: residual::ResidualQueryOperator::Sort { descending: !keys[0].ascending, @@ -444,43 +445,15 @@ where SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { reason: "unsupported post-ASAP value operation".into(), }, - SummaryExpr::CandidateTopK { - candidates, - values, - k, - grouping, - completeness, - } => { - let labels = grouping - .keys() - .iter() - .map(|&column| { - values - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid( - "unresolved CandidateTopK grouping column".into(), - ) - }) - }) - .collect::, _>>()?; + SummaryExpr::RelationalJoin { + right: candidates, left: values, + kind: planner_types::pre_asap::JoinKind::Semi, + pruning: Some(completeness), pred, + } if !self.preserve_relational => { + validate_membership_join(pred, &values.schema, &candidates.schema)?; let candidate_input = self.lower(candidates)?; let value_input = if let Some(original) = &self.logical_source { - let parsed = promql_parser::parser::parse(original) - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - let promql_parser::parser::Expr::Aggregate(aggregate) = parsed else { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires a top-level PromQL aggregate".into(), - )); - }; - if aggregate.op.to_string() != "topk" { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires a topk source expression".into(), - )); - } + let exact_expression = residual::selected_native_expression(original, values)?; fn item_label(node: &SummaryNode) -> Option { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => { @@ -500,7 +473,7 @@ where } let item_label = item_label(candidates).ok_or_else(|| { QueryPlanError::Invalid( - "CandidateTopK membership has no named item label".into(), + "MembershipFilter membership has no named item label".into(), ) })?; let value_id = QueryNodeId(self.next_id); @@ -510,7 +483,7 @@ where QueryPlanNode::ExternalExact { request: ExternalExactRequest { language: QueryLanguage::PromQl, - expression: aggregate.expr.to_string(), + expression: exact_expression.to_string(), output: ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, @@ -526,23 +499,19 @@ where } else { self.lower(values)? }; - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [candidate_input, value_input], - k: u64::try_from(*k).map_err(|_| { - QueryPlanError::Invalid("CandidateTopK k exceeds u64".into()) - })?, - grouping: residual::Grouping { - labels, - without: grouping.is_without(), - }, completeness: completeness.clone(), } } + SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { + reason: "read-time relational join requires the relational compiler".into(), + }, SummaryExpr::BinaryOp { lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if self.logical_source.is_some() || operator.checked_relative_division || operator.checked_finite_division => @@ -624,7 +593,7 @@ where lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if exact_value_executable(node) => { let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind else { @@ -782,7 +751,7 @@ where input: self.lower(summary_input)?, query: query.clone().into(), }, - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { if children.is_empty() { QueryPlanNode::ExactFallback { reason: "empty summary_merge".into(), @@ -881,7 +850,7 @@ pub(crate) fn exact_value_executable(node: &SummaryNode) -> bool { lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } => { matches!( operator.kind, @@ -1390,7 +1359,7 @@ mod tests { } #[test] - fn candidate_topk_rejects_invalid_completeness_contract() { + fn membership_filter_rejects_invalid_completeness_contract() { let leaf = QueryPlanNode::ExactFallback { reason: "prepared".into(), }; @@ -1405,13 +1374,8 @@ mod tests { (QueryNodeId(1), leaf), ( QueryNodeId(2), - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [QueryNodeId(0), QueryNodeId(1)], - k: 2, - grouping: residual::Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::Certified { guarantee: planner_types::post_asap::ResultGuarantee { metric: planner_types::post_asap::ErrorMetric::Frequency, @@ -1438,3 +1402,43 @@ mod tests { assert!(entry.validate(&BTreeSet::new()).is_err()); } } + +// The vector adapter matches complete label identities. Reject joins whose +// predicate would require a different projection instead of silently widening it. +fn validate_membership_join( + pred: &planner_types::pre_asap::Predicate, + left: &planner_types::post_asap::SummarySchema, + right: &planner_types::post_asap::SummarySchema, +) -> Result<(), QueryPlanError> { + use std::collections::BTreeSet; + use planner_types::pre_asap::{QueryExpr, CompareOpKind, DataType}; + use planner_types::post_asap::SummaryFamilyType; + fn collect(expr: &QueryExpr, width: usize, keys: &mut Vec<(usize, usize)>) -> Result<(), QueryPlanError> { + match expr { + QueryExpr::BoolAnd(parts) => { for part in parts { collect(part, width, keys)?; } } + QueryExpr::Compare { left, op: CompareOpKind::Eq, right } => { + let (QueryExpr::Column(a), QueryExpr::Column(b)) = (left.as_ref(), right.as_ref()) else { + return Err(QueryPlanError::Invalid("membership join requires column equality".into())); + }; + let (a,b) = if a < b { (*a,*b) } else { (*b,*a) }; + if a >= width || b < width { return Err(QueryPlanError::Invalid("membership join requires cross-input keys".into())); } + keys.push((a,b-width)); + } + _ => return Err(QueryPlanError::Invalid("unsupported membership join predicate".into())), + } + Ok(()) + } + let labels = |schema: &planner_types::post_asap::SummarySchema| schema.fields.iter().filter(|f| f.name != "__name__" && matches!(f.dtype, SummaryFamilyType::Plain(DataType::Utf8))).map(|f| f.name.clone()).collect::>(); + let mut keys = Vec::new(); + collect(&pred.0, left.fields.len(), &mut keys)?; + let mut matched = BTreeSet::new(); + for (a,b) in keys { + let Some((a,b)) = left.fields.get(a).zip(right.fields.get(b)) else { return Err(QueryPlanError::Invalid("membership join key out of bounds".into())); }; + if a.name != b.name { return Err(QueryPlanError::Invalid("membership join requires matching label names".into())); } + matched.insert(a.name.clone()); + } + if matched.is_empty() || matched != labels(left) || matched != labels(right) { + return Err(QueryPlanError::Invalid("membership join must match the complete label identity".into())); + } + Ok(()) +} diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index b5d0af51f..e8e1df437 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -442,11 +442,34 @@ pub(crate) fn selected_residual_nodes( original: &str, selected: &planner_types::post_asap::SummaryNode, ) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { + let expression = selected_native_expression(original, selected)?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expression)?; + Ok((root, lower.nodes)) +} + +/// Resolve the selected exact subtree to a verified native expression before +/// binding an external input. Never substitute the top-level query's child. +pub(super) fn selected_native_expression( + original: &str, + selected: &planner_types::post_asap::SummaryNode, +) -> Result { if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { return Err(invalid( "native residual substitution requires an exact selected value", )); } + let selected = match &selected.expr { + planner_types::post_asap::SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + .. + } => child.as_ref(), + _ => selected, + }; fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { output.push(expr); match expr { @@ -485,11 +508,6 @@ pub(crate) fn selected_residual_nodes( rhs: right, .. } - | SummaryExpr::CandidateTopK { - candidates: left, - values: right, - .. - } | SummaryExpr::RelationalJoin { left, right, .. } | SummaryExpr::SummaryJoin { outer: left, @@ -500,7 +518,7 @@ pub(crate) fn selected_residual_nodes( selected_horizons(left, out); selected_horizons(right, out); } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { selected_horizons(child, out); } @@ -530,12 +548,7 @@ pub(crate) fn selected_residual_nodes( let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) .replacements(&TargetSubDAG::new(&root)); if candidates.iter().any(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if node.as_ref() == selected)) { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(expression)?; - let candidate = (root, lower.nodes); + let candidate = expression.clone(); if matched .as_ref() .is_some_and(|previous| previous != &candidate) @@ -575,6 +588,24 @@ pub(super) fn selected_aggregate_operator( mod hybrid_tests { use super::*; use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; + #[test] + fn external_binding_rejects_an_unrelated_selected_exact_subtree() { + let exact = crate::query_parser::parse_query_expr_with_interval( + "sum_over_time(other_metric[5m])", + planner_types::types::AccuracyTarget::Exact, + 1_000, + ) + .unwrap(); + let selected = crate::planner_selection::plan_test_query(&exact).unwrap(); + assert!(selected_native_expression("topk(2, sum_over_time(m[5m]))", &selected).is_err()); + assert_eq!( + selected_native_expression("sum_over_time(other_metric[5m])", &selected) + .unwrap() + .to_string(), + "sum_over_time(other_metric[5m])" + ); + } + #[test] fn selected_summary_and_filtered_residual_share_installed_binary() { // Both filtered and unfiltered leaves bind independently. @@ -1085,19 +1116,13 @@ pub fn eligible_materialization_keys( visit(original, left, keys)?; visit(original, right, keys)?; } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - visit(original, candidates, keys)?; - visit(original, values, keys)?; - } SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryEstimate { summary_input, .. } | SummaryExpr::SummaryDelete { summary_input, .. } => { visit(original, summary_input, keys)? } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { visit(original, child, keys)?; } diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 5e334d261..a07a9a163 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -351,7 +351,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: child.clone(), rhs: child.clone(), operator: BinaryOperator { diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index eb1976b1f..b3556e69d 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -215,7 +215,7 @@ mod tests { use planner_types::post_asap::{ EdgeRole, ExecutionDataState, GroupingEdgeCompatibility, WindowEdgeCompatibility, }; - let state = ExecutionDataState::MAINTENANCE_SUMMARY; + let state = ExecutionDataState::INGESTION_SUMMARY; OwnedPostAsapDag { schema_version: crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, query_id: "query-a".into(), diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index c7462edd1..03e2193f9 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -20,8 +20,8 @@ use serde::{Deserialize, Serialize}; #[serde(transparent)] pub struct QueryNodeId(pub u64); -pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 2; -pub const MAINTENANCE_DAG_SCHEMA_VERSION: u32 = 3; +pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 3; +pub const MAINTENANCE_DAG_SCHEMA_VERSION: u32 = 4; /// Versioned, language-neutral Planner DAG persisted with an installed plan. /// Plan lifecycle belongs to the enclosing `PrecomputePlan`; this document @@ -293,7 +293,7 @@ impl BackendExecutableBinding { for node in &dag.nodes { match (node.output_state.timing, self.node(node.id)) { ( - ExecutionTiming::MaintenanceTime, + ExecutionTiming::IngestionTime, Some( BackendNodeBinding::MaintenanceInput | BackendNodeBinding::Materialization { .. }, @@ -331,11 +331,10 @@ impl BackendExecutableBinding { } for node in &dag.nodes { match (node.output_state.timing, self.node(node.id).unwrap()) { - (ExecutionTiming::ReadTime, BackendNodeBinding::Query { .. }) - | (ExecutionTiming::ReadTime, BackendNodeBinding::QueryInput) - | (ExecutionTiming::MaintenanceTime, BackendNodeBinding::MaintenanceInput) - | (ExecutionTiming::MaintenanceTime, BackendNodeBinding::Materialization { .. }) => { - } + (ExecutionTiming::QueryTime, BackendNodeBinding::Query { .. }) + | (ExecutionTiming::QueryTime, BackendNodeBinding::QueryInput) + | (ExecutionTiming::IngestionTime, BackendNodeBinding::MaintenanceInput) + | (ExecutionTiming::IngestionTime, BackendNodeBinding::Materialization { .. }) => {} _ => { return Err(format!( "backend placement disagrees with node {} mode", diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 67ebe26f0..547677956 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -139,6 +139,8 @@ pub enum TimestampUnit { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct IngestContract { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dataset_identity: Option, pub protocol: IngestProtocol, pub endpoint_path: String, pub timestamp_unit: TimestampUnit, @@ -372,11 +374,16 @@ impl PrecomputePlan { }) }) .collect(); + let dataset_identity = materializations + .iter() + .filter_map(|m| m.semantic_fragment.as_ref()?.dataset_identity.clone()) + .next(); let plan = Self { summary_catalog: None, envelope, ingest: if backend_local { IngestContract { + dataset_identity: dataset_identity.clone(), protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, @@ -386,6 +393,7 @@ impl PrecomputePlan { } } else { IngestContract { + dataset_identity: dataset_identity.clone(), protocol: IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: TimestampUnit::UnixNanoseconds, @@ -468,6 +476,23 @@ impl PrecomputePlan { } pub fn validate(&self) -> Result<(), PrecomputePlanError> { + if let Some(dataset) = &self.ingest.dataset_identity { + dataset + .validate() + .map_err(PrecomputePlanError::CatalogContract)?; + } + for materialization in &self.materializations { + if let Some(fragment) = &materialization.semantic_fragment { + fragment + .validate() + .map_err(PrecomputePlanError::CatalogContract)?; + if fragment.dataset_identity != self.ingest.dataset_identity { + return Err(PrecomputePlanError::CatalogContract( + "semantic dataset differs from installed input binding".into(), + )); + } + } + } let valid_ingest = match self.ingest.protocol { IngestProtocol::ModifiedOtlpMetricsV1 => { self.ingest.endpoint_path == "/v1/metrics" @@ -622,22 +647,20 @@ impl PrecomputePlan { .filter(|edge| edge.consumer == id) .collect(); use planner_types::post_asap::{ - ExecutableOperatorPayload as Payload, ExecutionTiming, ValueOperation, + ExecutableOperatorPayload as Payload, ValueOperation, }; if node.output_state - != planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS + != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS { return Err(invalid()); } match &node.payload { Payload::Value { operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::MaintenanceTime, } if children.len() == 1 && frontiers.contains_key(&children[0].producer) => {} Payload::Binary { operator, - timing: ExecutionTiming::MaintenanceTime, } if children.len() == 2 && children .iter() @@ -1048,7 +1071,7 @@ mod source_window_cohort_tests { reduction: Reduction::by(vec![]), grouping: Default::default(), }, - output_state: ExecutionDataState::MAINTENANCE_SUMMARY, + output_state: ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: vec![], time_index: None, @@ -1069,7 +1092,8 @@ mod source_window_cohort_tests { assert!(validate_maintenance_reduction(&config, &node).is_err()); config.partitioning = None; assert!(validate_maintenance_reduction(&config, &node).is_err()); - node.payload = ExecutableOperatorPayload::SummaryMerge; + node.payload = ExecutableOperatorPayload::SummaryMerge { + }; assert!(validate_maintenance_reduction(&config, &node).is_err()); } diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index f2322cd93..bad95e7e5 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -70,11 +70,10 @@ impl PrecomputePlan { if stored_output.fingerprint() == config.policy_fingerprint()) { found = true; - let actual = - crate::semantic_fragment::SemanticFragment::from_stored_output( - &dag, *id, - ) - .map_err(invalid)?; + let actual = match &self.ingest.dataset_identity { + Some(dataset) => crate::semantic_fragment::SemanticFragment::from_stored_output_in_dataset(&dag, *id, dataset.clone()), + None => crate::semantic_fragment::SemanticFragment::from_stored_output(&dag, *id), + }.map_err(invalid)?; if &actual != expected { return Err(invalid( "semantic definition differs from Planner-selected producer", diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 2135d6540..93eb4ae08 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -410,15 +410,7 @@ impl QueryPlanEntry { )); } } - if let QueryPlanNode::CandidateTopK { - k, completeness, .. - } = node - { - if *k == 0 { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires k > 0".into(), - )); - } + if let QueryPlanNode::MembershipFilter { completeness, .. } = node { if matches!( completeness, CandidateCompleteness::Certified { guarantee } @@ -428,7 +420,7 @@ impl QueryPlanEntry { || guarantee.failure_probability.evaluate().is_none() ) { return Err(QueryPlanError::Invalid( - "invalid CandidateTopK completeness certificate".into(), + "invalid MembershipFilter completeness certificate".into(), )); } } @@ -630,13 +622,11 @@ pub enum QueryPlanNode { SummaryMerge { inputs: Vec, }, - /// Use an approximate heap only as a membership sidecar, then rerank the - /// matching exact counter readouts. `inputs[0]` is candidate membership; - /// `inputs[1]` is the authoritative exact value vector. - CandidateTopK { + /// Semijoin value rows against membership identities, preserving their values + /// and order. Inputs are membership and authoritative values respectively. + /// Ranking, grouping and limiting are separate downstream operators. + MembershipFilter { inputs: [QueryNodeId; 2], - k: u64, - grouping: residual::Grouping, completeness: CandidateCompleteness, }, /// An exact subtree evaluated outside ASAP. Its results enter the query DAG @@ -664,7 +654,7 @@ impl QueryPlanNode { Self::SummaryMerge { inputs } | Self::Logical { inputs, .. } | Self::ExternalExact { inputs, .. } => inputs, - Self::CandidateTopK { inputs, .. } => inputs, + Self::MembershipFilter { inputs, .. } => inputs, } } } diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4cc0e985a..ec924e704 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -12,7 +12,7 @@ use crate::sds::{ use crate::PolicyFingerprint; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 4; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 6; /// Canonical definition binds operator and population descriptors. Writer /// layout and concrete state belong to installed plans and runtime instances. diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs index 387967dd2..757a7e15b 100644 --- a/crates/asap_types/src/summary_semantics.rs +++ b/crates/asap_types/src/summary_semantics.rs @@ -22,7 +22,7 @@ pub enum SummarySemantics { /// Restricted raw-input adapter for explicit native summary configurations. /// General expressions must use the Planner fragment variant. Configured { - computation: SummaryComputation, + computation: Box, value_source_column: Option, aggregated_labels: crate::KeyByLabelNames, rollup_labels: crate::KeyByLabelNames, @@ -59,7 +59,7 @@ impl SummaryDefinition { value_source_column: None, aggregated_labels: crate::KeyByLabelNames::empty(), rollup_labels: crate::KeyByLabelNames::empty(), - computation: SummaryComputation { + computation: Box::new(SummaryComputation { operator: summary.operator.clone(), source: data.source.clone(), value: data.value_projection.clone(), @@ -67,7 +67,7 @@ impl SummaryDefinition { grouping: data.group_by_keys.clone(), timestamp_column: data.timestamp_column.clone(), observation_semantics: data.observation_semantics.clone(), - }, + }), }, }) } diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 5411f0a10..6c373e6a2 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -988,6 +988,7 @@ mod tests { summary_catalog: Some(generation), envelope: envelope.clone(), ingest: IngestContract { + dataset_identity: None, protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, @@ -1207,7 +1208,8 @@ mod tests { table_timestamp_column: None, partitioning: None, value_source_column: None, - }; + } + }; let cms = config( AggregationType::CountMinSketchWithHeap, vec![], diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 9ba8e6b2e..144e63012 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2524,6 +2524,7 @@ mod tests { summary_catalog: Some(generation.clone()), envelope: envelope.clone(), ingest: IngestContract { + dataset_identity: None, protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 9218db907..1448cc337 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -746,6 +746,7 @@ async fn main() -> Result<()> { capability_snapshot_id: "bootstrap".into(), }, ingest: asap_types::precompute_plan::IngestContract { + dataset_identity: None, protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 393aab5b8..08de6788e 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -166,15 +166,18 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { node: &ExecutableDagNode, inputs: &[Arc], ) -> Result { + if node.output_state.timing != planner_types::post_asap::ExecutionTiming::IngestionTime { + return Err("maintenance runtime requires ingestion-time nodes".into()); + } match &node.payload { - ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs), + ExecutableOperatorPayload::SummaryMerge { + } => merge_inputs(inputs), ExecutableOperatorPayload::Binary { operator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, } => { if !self.inputs.frozen_inputs().is_some() || node.output_state - != planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS + != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS { return Err("maintenance binary requires immutable completed row inputs".into()); } @@ -183,7 +186,6 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { ExecutableOperatorPayload::Value { operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, } => { if !self.inputs.frozen_inputs().is_some() { return Err( @@ -2160,8 +2162,7 @@ mod tests { .nodes .iter() .filter(|node| { - node.output_state.timing - == planner_types::post_asap::ExecutionTiming::MaintenanceTime + node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime }) .map(|node| node.id) .collect::>(); @@ -2238,8 +2239,9 @@ mod tests { fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), - payload: ExecutableOperatorPayload::SummaryMerge, - output_state: planner_types::post_asap::ExecutionDataState::MAINTENANCE_SUMMARY, + payload: ExecutableOperatorPayload::SummaryMerge { + }, + output_state: planner_types::post_asap::ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: vec![], time_index: None, @@ -2257,7 +2259,7 @@ mod tests { fields: vec![], time_index: None, }, - data_state: planner_types::post_asap::ExecutionDataState::MAINTENANCE_SUMMARY, + data_state: planner_types::post_asap::ExecutionDataState::INGESTION_SUMMARY, grouping: GroupingEdgeCompatibility::Identical, window: WindowEdgeCompatibility::NotApplicable, } @@ -2433,7 +2435,6 @@ mod tests { let mut read = node(2); read.payload = ExecutableOperatorPayload::Value { operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, }; read.output_schema.fields = vec![SummaryField { name: "value".into(), @@ -2478,14 +2479,14 @@ mod tests { dtype: SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), nullable: false, }]; - read.output_state = planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS; + read.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; aggregate.output_schema.fields = vec![SummaryField { name: "state".into(), dtype: configs[1].accumulator_spec().unwrap().family, nullable: false, }]; let mut query = node(4); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let mut first_edge = edge(1, 2); first_edge.intermediate_schema = source_node.output_schema.clone(); let mut second_edge = edge(2, 3); @@ -2896,7 +2897,8 @@ mod tests { second_node.id = PostAsapNodeId(5); let mut merge = second_node.clone(); merge.id = PostAsapNodeId(6); - merge.payload = ExecutableOperatorPayload::SummaryMerge; + merge.payload = ExecutableOperatorPayload::SummaryMerge { + }; dag.nodes.extend([second_node, merge]); let original = dag .edges @@ -3384,9 +3386,8 @@ mod tests { }; operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, }; - operation.output_state = planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS; + operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; assert!(frozen .execute(&operation, &[left.clone(), right.clone()]) .is_ok()); @@ -3403,8 +3404,8 @@ mod tests { .is_err()); operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), - timing: planner_types::post_asap::ExecutionTiming::ReadTime, }; + operation.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; assert!(frozen .execute(&operation, &[left.clone(), right.clone()]) .is_err()); @@ -3859,7 +3860,7 @@ mod tests { .policy_fingerprint() .into(); let mut query = node(2); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![node(0), node(1), query], edges: vec![edge(0, 1), edge(1, 2)], @@ -3999,7 +4000,7 @@ mod tests { // source 0 is shared by both branches; root therefore contains two // copies of its value while node 0 itself is evaluated once. let mut query = node(4); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: (0..4).map(node).chain([query]).collect(), edges: vec![edge(0, 1), edge(0, 2), edge(1, 3), edge(2, 3), edge(3, 4)], @@ -4071,7 +4072,7 @@ mod tests { let mut unsupported = node(1); unsupported.payload = ExecutableOperatorPayload::SummarySubtract; let mut query = node(2); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![node(0), unsupported, query], edges: vec![edge(0, 1), edge(1, 2)], diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 3d75003e8..660ef8906 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -153,7 +153,7 @@ where let node = nodes .get(&id) .ok_or_else(|| ScheduleError::Invalid(format!("missing node {id}")))?; - if node.output_state == ExecutionDataState::READ_ROWS { + if node.output_state == ExecutionDataState::QUERY_ROWS { return Err(ScheduleError::Invalid(format!( "query-time node {id} in precompute dependency path" ))); @@ -236,8 +236,7 @@ mod tests { .nodes .iter() .filter(|node| { - node.output_state.timing - == planner_types::post_asap::ExecutionTiming::MaintenanceTime + node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime }) .map(|node| node.id) .collect::>(); @@ -253,7 +252,7 @@ mod tests { ExecutableDagNode { id: PostAsapNodeId(id), payload: ExecutableOperatorPayload::SummarySubtract, - output_state: ExecutionDataState::MAINTENANCE_SUMMARY, + output_state: ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: Vec::new(), time_index: None, @@ -271,7 +270,7 @@ mod tests { fields: Vec::new(), time_index: None, }, - data_state: ExecutionDataState::MAINTENANCE_SUMMARY, + data_state: ExecutionDataState::INGESTION_SUMMARY, grouping: GroupingEdgeCompatibility::Identical, window: WindowEdgeCompatibility::NotApplicable, } @@ -347,7 +346,6 @@ mod tests { } let mut binary = node(3); binary.payload = ExecutableOperatorPayload::Binary { - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, operator: BinaryOperator { checked_relative_division: false, checked_finite_division: false, @@ -362,7 +360,7 @@ mod tests { let mut dag = ExecutableDag { nodes: vec![node(0), node(1), node(2), binary, { let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; query }], edges: vec![right, left], @@ -398,7 +396,7 @@ mod tests { .map(node) .chain([{ let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; query }]) .collect(), @@ -443,9 +441,9 @@ mod tests { } } let mut raw = node(0); - raw.output_state = ExecutionDataState::READ_ROWS; + raw.output_state = ExecutionDataState::QUERY_ROWS; let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![raw, node(1), node(2), node(3), query], edges: vec![edge(0, 1), edge(1, 2), edge(1, 3), edge(2, 3), edge(3, 4)], @@ -471,7 +469,7 @@ mod tests { #[test] fn rejects_query_node_in_precompute_path_and_mismatched_lineage_key() { let mut query_child = node(0); - query_child.output_state = ExecutionDataState::READ_ROWS; + query_child.output_state = ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![query_child, node(1)], edges: vec![edge(0, 1)], diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index e402f6343..0ef02672d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -668,7 +668,7 @@ mod tests { root, QueryPlanNode::Relational { input: sort, - operation: serde_json::to_value(ValueOperation::Limit { n: 1, offset: 0 }) + operation: serde_json::to_value(ValueOperation::Limit { n: 1, offset: 0, partition_by: planner_types::pre_asap::GroupKeys::by(vec![]) }) .unwrap(), input_schema: projected_schema.clone(), output_schema: projected_schema, diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index c47b7b48e..27abba680 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -484,7 +484,10 @@ impl ClickHouseRelationalAdapter { .rows .sort_by(|left, right| compare_sort_keys(left, right, keys, &schema)); } - ValueOperation::Limit { n, offset } => { + ValueOperation::Limit { n, offset, partition_by } => { + if !partition_by.keys().is_empty() || partition_by.is_without() { + return Err(ClickHouseRelationalError::Unsupported("partitioned Limit".into())); + } input.rows = input.rows.into_iter().skip(*offset).take(*n).collect(); } other => return Err(ClickHouseRelationalError::Unsupported(format!("{other:?}"))), @@ -1604,7 +1607,7 @@ mod tests { }], partition_by: GroupKeys::none(), }, - ValueOperation::Limit { n: 1, offset: 0 }, + ValueOperation::Limit { n: 1, offset: 0, partition_by: planner_types::pre_asap::GroupKeys::by(vec![]) }, ] { relation = adapter .apply_operation(&operation, &projected_schema, relation) diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index cae8ff229..cc6407ee8 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -286,7 +286,7 @@ impl ASAPQueryEngine { // Candidate-filtered exact cuts have a data dependency: read the // installed membership subtree once, then use that vector to build the // Prometheus selector. Keeping the result as a prepared leaf also means - // CandidateTopK reuses the same membership readout during composition. + // MembershipFilter reuses the same membership readout during composition. let dependencies = super::exact_subqueries::external_dependencies(entry, times)?; let mut prepared = super::logical_dag::PreparedLeaves::new(); let unique_inputs = dependencies diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index b15da276b..7b2350907 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -85,9 +85,9 @@ fn leaves( } _ => pending.extend(inputs.iter().map(|input| (*input, at))), }, - // CandidateTopK is a typed composition node rather than a Logical + // MembershipFilter is a typed composition node rather than a Logical // wrapper, but its value input can still be a Prometheus leaf. - QueryPlanNode::CandidateTopK { inputs, .. } => { + QueryPlanNode::MembershipFilter { inputs, .. } => { pending.extend(inputs.iter().map(|input| (*input, at))); } QueryPlanNode::ExternalExact { request, inputs } => { @@ -650,22 +650,30 @@ mod tests { } #[tokio::test] - async fn candidate_exact_is_discovered_and_prepared_behind_candidate_topk_root() { - use asap_types::query_plan::{residual::Grouping, CandidateCompleteness}; + async fn candidate_exact_is_discovered_and_prepared_behind_membership_filter_root() { + use asap_types::query_plan::CandidateCompleteness; let mut entry = candidate_entry("sum by (job) (rate(m[5m]))"); entry.nodes.insert( QueryNodeId(2), - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [QueryNodeId(1), QueryNodeId(0)], - k: 2, - grouping: Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::BestEffort { guarantee: None }, }, ); - entry.root = QueryNodeId(2); + entry.nodes.insert( + QueryNodeId(3), + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { + k: 2, + grouping: asap_types::query_plan::residual::Grouping { + labels: vec![], + without: false, + }, + }, + inputs: vec![QueryNodeId(2)], + }, + ); + entry.root = QueryNodeId(3); let dependencies = external_dependencies(&entry, &[1_000]).unwrap(); assert_eq!(dependencies, vec![(QueryNodeId(0), QueryNodeId(1), 1_000)]); let prepared = prepare_external( diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 4b77a5a8f..2bf916356 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -205,16 +205,13 @@ impl Result> Evaluator<' } self.logical(operator, &inputs, at)? } - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs, - k, - grouping, completeness, } => { let candidates = vector(self.eval(inputs[0], at)?)?; let values = vector(self.eval(inputs[1], at)?)?; - let (selected, warning) = - candidate_topk(k, &grouping, candidates, values, &completeness)?; + let (selected, warning) = membership_filter(candidates, values, &completeness)?; if let Some(warning) = warning { self.warnings.push(warning); } @@ -423,9 +420,7 @@ impl Result> Evaluator<' } } -fn candidate_topk( - k: u64, - grouping: &Grouping, +fn membership_filter( candidates: Vector, values: Vector, completeness: &CandidateCompleteness, @@ -435,30 +430,21 @@ fn candidate_topk( labels.remove("__name__"); labels }; - let candidate_ids: BTreeSet<_> = candidates - .iter() - .map(|(labels, _)| identity(labels)) - .collect(); + let candidate_ids: BTreeSet<_> = candidates.iter().map(|(labels, _)| identity(labels)).collect(); let value_ids: BTreeSet<_> = values.iter().map(|(labels, _)| identity(labels)).collect(); - let dangling = candidate_ids - .iter() - .any(|candidate| !value_ids.contains(candidate)); - if dangling && matches!(completeness, CandidateCompleteness::Certified { .. }) { - return Err(miss("certified TopK candidate has no exact counter value")); + let missing: BTreeSet<_> = candidate_ids.difference(&value_ids).collect(); + let selected = values.into_iter().filter(|(labels, _)| candidate_ids.contains(&identity(labels))).collect(); + if !missing.is_empty() && matches!(completeness, CandidateCompleteness::Certified { .. }) { + return Err(miss("certified membership key has no authoritative value")); } - let matched = values - .into_iter() - .filter(|(labels, _)| candidate_ids.contains(&identity(labels))) - .collect(); - let selected = topk_selection(k, grouping, matched); let warning = match completeness { CandidateCompleteness::Certified { .. } => None, CandidateCompleteness::BestEffort { guarantee } => Some(match guarantee { Some(guarantee) => format!( - "ASAP TopK candidate membership is approximate: {:?}", + "ASAP membership pruning is approximate: {:?}", guarantee.metric ), - None => "ASAP TopK candidate membership is approximate and uncertified".into(), + None => "ASAP membership pruning is approximate and uncertified".into(), }), }; Ok((selected, warning)) @@ -908,6 +894,14 @@ mod topk_tests { (labels(&[("series", "high")]), 3.0), ], ); + let selected = topk_selection( + 2, + &Grouping { + labels: vec![], + without: false, + }, + selected, + ); assert_eq!( selected .iter() @@ -1177,12 +1171,7 @@ mod topk_tests { (labels(&[("pod", "b")]), 8.0), (labels(&[("pod", "c")]), 9.0), ]; - let (selected, warning) = candidate_topk( - 2, - &Grouping { - labels: vec![], - without: false, - }, + let (selected, warning) = membership_filter( candidates, exact, &CandidateCompleteness::Certified { @@ -1190,6 +1179,14 @@ mod topk_tests { }, ) .unwrap(); + let selected = topk_selection( + 2, + &Grouping { + labels: vec![], + without: false, + }, + selected, + ); assert_eq!( selected .iter() @@ -1204,7 +1201,8 @@ mod topk_tests { fn installed_candidate_sidecar_reads_both_summary_inputs() { let candidate_id = QueryNodeId(0); let value_id = QueryNodeId(1); - let root = QueryNodeId(2); + let filter = QueryNodeId(2); + let root = QueryNodeId(3); let entry = QueryPlanEntry { language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), @@ -1225,19 +1223,27 @@ mod topk_tests { }, ), ( - root, - QueryPlanNode::CandidateTopK { + filter, + QueryPlanNode::MembershipFilter { inputs: [candidate_id, value_id], - k: 1, - grouping: Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::Certified { guarantee: topk_membership_guarantee(), }, }, ), + ( + root, + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { + k: 1, + grouping: Grouping { + labels: vec![], + without: false, + }, + }, + inputs: vec![filter], + }, + ), ]), instant: InstantExecution { lookback_ms: 300_000, @@ -1251,7 +1257,10 @@ mod topk_tests { ( (candidate_id, at), PreparedLeaf { - value: Value::Vector(vec![(labels(&[("pod", "b")]), 100.0)]), + value: Value::Vector(vec![ + (labels(&[("pod", "b")]), 100.0), + (labels(&[("pod", "c")]), 1.0), + ]), remote: false, remote_evaluations: 0, remote_rpcs: 0, @@ -1263,6 +1272,7 @@ mod topk_tests { value: Value::Vector(vec![ (labels(&[("pod", "a")]), 2.0), (labels(&[("pod", "b")]), 1.0), + (labels(&[("pod", "c")]), 3.0), ]), remote: false, remote_evaluations: 0, @@ -1278,8 +1288,8 @@ mod topk_tests { panic!("vector expected") }; assert_eq!(result.values.len(), 1); - assert_eq!(result.values[0].value, 1.0, "exact value is authoritative"); - assert_eq!(result.values[0].labels.labels, vec!["b"]); + assert_eq!(result.values[0].value, 3.0, "exact value is authoritative"); + assert_eq!(result.values[0].labels.labels, vec!["c"]); assert_eq!(stats.summary_readout_evaluations, 2); assert!(result.warnings.is_empty()); } @@ -1288,30 +1298,20 @@ mod topk_tests { fn uncertified_candidate_sidecar_warns_or_falls_back_explicitly() { let candidates = vec![(labels(&[("pod", "a")]), 1.0)]; let exact = vec![(labels(&[("pod", "a")]), 2.0)]; - let (_, warning) = candidate_topk( - 1, - &Grouping { - labels: vec![], - without: false, - }, + let (_, warning) = membership_filter( candidates.clone(), exact.clone(), &CandidateCompleteness::BestEffort { guarantee: None }, ) .unwrap(); assert!(warning.unwrap().contains("approximate")); - // Exact queries never lower an uncertified CandidateTopK. The Planner + // Exact queries never lower an uncertified MembershipFilter. The Planner // emits its ordinary exact fallback instead; this runtime node is only // valid for certified or explicitly approximate plans. let certified = CandidateCompleteness::Certified { guarantee: topk_membership_guarantee(), }; - assert!(candidate_topk( - 1, - &Grouping { - labels: vec![], - without: false - }, + assert!(membership_filter( vec![(labels(&[("pod", "missing")]), 1.0)], exact, &certified, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 366517767..0bf056e18 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -305,7 +305,7 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { }) } QueryPlanNode::Logical { .. } - | QueryPlanNode::CandidateTopK { .. } + | QueryPlanNode::MembershipFilter { .. } | QueryPlanNode::Relational { .. } | QueryPlanNode::ExternalExact { .. } | QueryPlanNode::RelationalJoin { .. } => Err(PhysicalNodeError::Fallback( diff --git a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs index fb75ae6bc..803653718 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs @@ -184,7 +184,7 @@ pub fn execute( Ok(ExecOutcome::Value(out)) } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { if children.is_empty() { return Err(ExecError::EmptyMerge); } @@ -223,11 +223,10 @@ pub fn execute( SummaryExpr::SummaryJoin { .. } => Err(ExecError::NotYetSupported("SummaryJoin")), SummaryExpr::RelationalJoin { .. } => Err(ExecError::NotYetSupported("RelationalJoin")), - // CandidateTopK is lowered to the deployed QueryPlan DAG, where both + // MembershipFilter is lowered to the deployed QueryPlan DAG, where both // row inputs retain labels for intersection and exact reranking. This // legacy generic adapter exposes opaque GroupKey values and cannot // implement that contract without losing label identity. - SummaryExpr::CandidateTopK { .. } => Err(ExecError::NotYetSupported("CandidateTopK")), SummaryExpr::BinaryOp { .. } => Err(ExecError::NotYetSupported("BinaryOp")), SummaryExpr::ValueOperation { .. } => Err(ExecError::NotYetSupported("ValueOperation")), SummaryExpr::SummarySubtract { .. } => Err(ExecError::NotYetSupported("SummarySubtract")), @@ -350,7 +349,10 @@ mod tests { fn merge_node(children: Vec>) -> Rc { Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { children }, + expr: SummaryExpr::SummaryMerge { + children, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, + }, schema: lift(vec!["value"]), guarantee: None, }) @@ -522,7 +524,7 @@ mod tests { let child = logical_node(); let tree = SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: child.clone(), rhs: child.clone(), operator: planner_types::post_asap::BinaryOperator { diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index f3015afb8..979f3c845 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1420,7 +1420,7 @@ fn find_metric(node: &SummaryNode) -> Option { SummaryExpr::KeepPreAsap(qe) => find_metric_in_query_expr(qe), SummaryExpr::SummaryAgg { child, .. } => find_metric(child), SummaryExpr::SummaryEstimate { summary_input, .. } => find_metric(summary_input), - SummaryExpr::SummaryMerge { children } => children.first().and_then(|c| find_metric(c)), + SummaryExpr::SummaryMerge { children, .. } => children.first().and_then(|c| find_metric(c)), _ => None, } } diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 3d6a8683a..d1d15c6e0 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -710,6 +710,7 @@ mod tests { summary_catalog: None, envelope: envelope.clone(), ingest: asap_types::precompute_plan::IngestContract { + dataset_identity: None, protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, diff --git a/docs/design_docs/asapplanner-migration-plan.md b/docs/design_docs/asapplanner-migration-plan.md index 0c51cda9a..afbd49ca4 100644 --- a/docs/design_docs/asapplanner-migration-plan.md +++ b/docs/design_docs/asapplanner-migration-plan.md @@ -165,8 +165,8 @@ Ad-hoc discovery is deferred. | Implementation owner | Required change | Regression/acceptance gate | | --- | --- | --- | | Planner shared types and physical integration (#462) | Export a versioned canonical semantic description for a selected persisted output; exclude placement and temporary node IDs. | Different input expressions differ; renumbering preserves identity; state definitions exclude downstream readout parameters. | -| Backend plan/schema foundation (#749), completed with the shared semantic contract in #774 | Separate semantic definitions from deployed-output bindings; remove the requirement that stored-output ID equals definition ID; version the changed plan contract. | Same-version hot/rebuild outputs can share one definition without aliasing; tampered definitions and mismatched bindings fail installation. | -| Planner dependency integration (#774) | Consume the shared semantic export and propagate it from selected physical outputs into deployment compilation. | No backend expression normalization or synthetic semantic fingerprint from incomplete config fields. | +| Backend plan/schema and SDS identity foundation (#749) | Separate semantic definitions from deployed-output bindings; remove the requirement that stored-output ID equals definition ID; version the changed plan contract. | Same-version hot/rebuild outputs can share one definition without aliasing; tampered definitions and mismatched bindings fail installation. | +| Planner semantic integration (#749); shared execution integration (#774) | #749 consumes the shared semantic export and propagates dataset-bound definitions from selected outputs. #774 integrates the shared physical executor. | No backend expression normalization or synthetic semantic fingerprint from incomplete config fields. | | Precompute/storage integration (#763) | Persist definitions and output-scoped records; authorize writes against installed bindings and recover them consistently. | Restart retains semantic descriptions; wrong-output writes fail; replacement metadata and payload remain consistent. | | Query integration (#765) | Resolve the installed deployed output and validate definition, revision, format and coverage before invoking shared execution. | A hot-bound query never reads rebuild state; stale, missing or incompatible records take the explicit failure route. | | Acceptance PRs (#728, #742, #775) | Update fixtures and process tests for the new contract; validate individual queries and ensembles using synthetic costs. | End-to-end producer → persisted definition/record → recovery → bound read, with negative identity and coverage cases. | diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index ce3355578..57b6f7f79 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -6,7 +6,7 @@ envelope containing the authoritative `SummaryCatalog` snapshot plus the `PrecomputePlan`, optional `CollectorPlan`, `TransmissionPlan`, and `QueryPlan` that reference installed stored outputs and their semantic definitions. -Catalog schema 4 separates `StoredOutputId` (deployment routing) from +Catalog schema 6 separates `StoredOutputId` (deployment routing) from `SummaryDefinitionId` (versioned semantic SHA-256). `StoredOutputReference` binds both. Planner exports the persisted output's typed semantic dependency closure; explicit raw summary configurations use a restricted typed description. @@ -31,14 +31,17 @@ This branch validates that maintenance and query plans belong to one installed catalog generation, retain the selected DAG provenance, and agree on writer/read bindings and physical windows. It removes the Collector runtime dependency. -The identity representation is transitional: catalog schema 3 still uses policy -fingerprints and couples the stored-output ID to that identifier. It cannot yet -represent independent hot/rebuild outputs with one semantic definition or prove -that equal metric names refer to the same logical dataset. Do not use this stage -as the completed SDS implementation. #774 supplies the Planner semantic export, -independent definition/output identities and explicit logical dataset binding; -#763/#765 supply the remaining storage and execution integration. These are -required before claiming the target contract or supporting multi-dataset reuse. +#749 establishes the SDS identity contract. Planner-generated definitions contain +an explicit logical dataset/tenant identity and the canonical typed dependency +closure of the persisted output. Endpoint relocation preserves this identity; +a different dataset changes it even when the metric and expression are equal. +The installed input binding must agree with the exported dataset identity. + +Semantic definition IDs and deployed output IDs are independent. Hot and rebuild +outputs can share one definition, but an installed query reads only its selected +output. Catalog schema 6 and planning snapshot schema 3 reject older metadata +rather than inventing a missing semantic identity. Later runtime PRs consume this +contract; they do not replace its identity model. Recovery at this stage is limited to the same installed catalog generation. Installing a new plan version never adopts previous-version state, even for an diff --git a/docs/examples/asapquery-compatibility-demo-snapshot.json b/docs/examples/asapquery-compatibility-demo-snapshot.json index e381d536e..c24e976c4 100644 --- a/docs/examples/asapquery-compatibility-demo-snapshot.json +++ b/docs/examples/asapquery-compatibility-demo-snapshot.json @@ -1,5 +1,5 @@ { - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": { "language": "promql", "query_batch": null, @@ -111,6 +111,7 @@ } }, "environment": { + "dataset_identity": {"namespace": "example", "dataset": "metrics"}, "target": "backend_local_remote_write", "collector_ids": [], "capability_snapshot_id": "asapquery-compatibility-demo-v1", diff --git a/docs/examples/asapquery-planning-snapshot.json b/docs/examples/asapquery-planning-snapshot.json index 471c8ad02..736857c49 100644 --- a/docs/examples/asapquery-planning-snapshot.json +++ b/docs/examples/asapquery-planning-snapshot.json @@ -1,5 +1,5 @@ { - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": { "language": "promql", "query_batch": null, @@ -62,6 +62,7 @@ "scrape_interval_ms": 5000 }, "environment": { + "dataset_identity": {"namespace": "example", "dataset": "metrics"}, "target": "backend_local_remote_write", "collector_ids": [], "capability_snapshot_id": "asapquery-local-v1", diff --git a/tools/o11y-execution/calibrate_runtime.py b/tools/o11y-execution/calibrate_runtime.py index 5587dca6e..7b263db7c 100644 --- a/tools/o11y-execution/calibrate_runtime.py +++ b/tools/o11y-execution/calibrate_runtime.py @@ -212,7 +212,7 @@ def launch(name, command): query_phase = phase(folder, "query-" + qid, before, after, time.perf_counter_ns() - start) raw = folder / f"queries-{qid}.json" runner.write_json(raw, records) - validate_candidate_topk_execution(artifact, records) + validate_membership_filter_execution(artifact, records) routes = {record["execution"] for record in records} correct = all(record["comparison"]["equal"] and record["exact"]["http_status"] == 200 for record in records) row["queries"][qid] = {"cpu_ns": query_phase["cpu_ns"], "evaluations": len(records), @@ -267,20 +267,20 @@ def launch(name, command): -def _candidate_topk_inputs(nodes, root): +def _membership_filter_inputs(nodes, root): bindings, visiting, visited = set(), set(), set() def visit(node_id): node_id = str(node_id) if node_id in visiting: - raise ValueError("CandidateTopK input DAG contains a cycle") + raise ValueError("MembershipFilter input DAG contains a cycle") if node_id in visited: return if node_id not in nodes: - raise ValueError(f"CandidateTopK input DAG references missing node {node_id}") + raise ValueError(f"MembershipFilter input DAG references missing node {node_id}") visiting.add(node_id) node = nodes[node_id] if node.get("op") == "exact_fallback": - raise ValueError("CandidateTopK input contains ExactFallback") + raise ValueError("MembershipFilter input contains ExactFallback") if node.get("op") == "read_materialization": bindings.add(str(node["binding"]["materialization"])) children = [str(value) for value in node.get("inputs", [])] @@ -294,21 +294,21 @@ def visit(node_id): return bindings -def validate_candidate_topk_artifact(artifact): - """Reject CandidateTopK plans whose membership sidecar is not locally installed.""" +def validate_membership_filter_artifact(artifact): + """Reject MembershipFilter plans whose membership sidecar is not locally installed.""" request = artifact.get("install_request", {}) schemas = {str(row["materialization"]): row for row in request.get("precompute_plan", {}).get("schemas", [])} modes = set() for entry in request.get("query_plan", {}).get("entries", {}).values(): nodes = entry.get("nodes", {}) for node in nodes.values(): - if node.get("op") != "candidate_top_k": + if node.get("op") != "membership_filter": continue inputs = node.get("inputs", []) if len(inputs) != 2: - raise ValueError("CandidateTopK requires membership and exact-value inputs") - membership_bindings = _candidate_topk_inputs(nodes, inputs[0]) - value_bindings = _candidate_topk_inputs(nodes, inputs[1]) + raise ValueError("MembershipFilter requires membership and exact-value inputs") + membership_bindings = _membership_filter_inputs(nodes, inputs[0]) + value_bindings = _membership_filter_inputs(nodes, inputs[1]) heap_bindings = [] for materialization in membership_bindings: schema = schemas.get(materialization) @@ -316,7 +316,7 @@ def validate_candidate_topk_artifact(artifact): if "CmsWithHeap" in family or "CountSketchWithHeap" in family: heap_bindings.append(materialization) if not heap_bindings: - raise ValueError("CandidateTopK membership input has no installed heap materialization") + raise ValueError("MembershipFilter membership input has no installed heap materialization") value_node = nodes.get(str(inputs[1]), {}) operator = value_node.get("operator", {}) if value_node.get("op") == "logical" else {} if operator.get("kind") == "candidate_exact_subquery": @@ -339,21 +339,21 @@ def validate_candidate_topk_artifact(artifact): and any(kind in json.dumps((schemas.get(mid) or {}).get("family", {})).lower() for kind in ("counter", "rate", "increase")) for mid in value_bindings): - raise ValueError("CandidateTopK value input has no installed ExactCounter materialization") + raise ValueError("MembershipFilter value input has no installed ExactCounter materialization") return modes -def validate_candidate_topk_execution(artifact, records): - modes = validate_candidate_topk_artifact(artifact) +def validate_membership_filter_execution(artifact, records): + modes = validate_membership_filter_artifact(artifact) if not modes: return if len(modes) != 1: - raise ValueError("mixed CandidateTopK execution contracts are not calibratable together") + raise ValueError("mixed MembershipFilter execution contracts are not calibratable together") mode = next(iter(modes)) for record in records: provenance = record.get("execution_provenance", {}) if provenance.get("raw_scan_evaluations", 0) not in (0, None): - raise ValueError("CandidateTopK execution used a forbidden local raw scan") + raise ValueError("MembershipFilter execution used a forbidden local raw scan") if mode == "candidate_filtered_exact": if record.get("execution") != "hybrid" or provenance.get("detail") != "hybrid": raise ValueError("candidate-filtered TopK did not report hybrid execution") @@ -364,12 +364,12 @@ def validate_candidate_topk_execution(artifact, records): raise ValueError(f"candidate-filtered TopK has invalid provenance: {key}") else: if record.get("execution") != "warm" or provenance.get("detail") not in (None, "asap"): - raise ValueError("local CandidateTopK execution was not warm") + raise ValueError("local MembershipFilter execution was not warm") for key in ("exact_subquery_rpcs", "exact_subquery_evaluations", "exact_branch_evaluations"): if provenance.get(key, 0) != 0: - raise ValueError(f"CandidateTopK execution used exact path: {key}") + raise ValueError(f"MembershipFilter execution used exact path: {key}") if provenance.get("summary_readout_evaluations", 0) < 2: - raise ValueError("CandidateTopK execution did not read both summary branches") + raise ValueError("MembershipFilter execution did not read both summary branches") def main(): @@ -413,7 +413,7 @@ def main(): candidates = candidate_document["candidates"] for candidate in candidates: if "manifest" in candidate and "install_request" in candidate: - validate_candidate_topk_artifact(candidate) + validate_membership_filter_artifact(candidate) for index, candidate in enumerate(candidates): if "manifest" not in candidate or "install_request" not in candidate: continue diff --git a/tools/o11y-execution/test_calibrate_runtime.py b/tools/o11y-execution/test_calibrate_runtime.py index 74a0931dc..891829f36 100644 --- a/tools/o11y-execution/test_calibrate_runtime.py +++ b/tools/o11y-execution/test_calibrate_runtime.py @@ -1,14 +1,14 @@ -"""Fail-closed validation for calibrated CandidateTopK artifacts.""" +"""Fail-closed validation for calibrated MembershipFilter artifacts.""" import unittest -from calibrate_runtime import validate_candidate_topk_artifact, validate_candidate_topk_execution +from calibrate_runtime import validate_membership_filter_artifact, validate_membership_filter_execution -class CandidateTopKArtifactTests(unittest.TestCase): +class MembershipFilterArtifactTests(unittest.TestCase): def artifact(self, membership): return {"install_request": { "query_plan": {"entries": {"q": {"nodes": { - "0": {"op": "candidate_top_k", "inputs": [1, 3]}, + "0": {"op": "membership_filter", "inputs": [1, 3]}, "1": {"op": "summary_estimate", "input": 2}, "2": membership, "3": {"op": "exact_readout", "input": 4}, @@ -35,20 +35,20 @@ def candidate_filtered_artifact(self): def test_rejects_exact_membership_fallback(self): with self.assertRaisesRegex(ValueError, "contains ExactFallback"): - validate_candidate_topk_artifact(self.artifact({"op": "exact_fallback", "reason": "unsupported"})) + validate_membership_filter_artifact(self.artifact({"op": "exact_fallback", "reason": "unsupported"})) def test_rejects_uninstalled_heap_membership(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 9}}) with self.assertRaisesRegex(ValueError, "no installed heap"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_accepts_heap_membership_and_exact_values(self): - validate_candidate_topk_artifact( + validate_membership_filter_artifact( self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) ) - def test_ignores_plans_without_candidate_topk(self): - validate_candidate_topk_artifact({"install_request": { + def test_ignores_plans_without_membership_filter(self): + validate_membership_filter_artifact({"install_request": { "query_plan": {"entries": {"q": {"nodes": {"0": {"op": "exact_fallback"}}}}}, "precompute_plan": {"schemas": []}, }}) @@ -57,56 +57,56 @@ def test_rejects_exact_value_fallback(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) artifact["install_request"]["query_plan"]["entries"]["q"]["nodes"]["3"] = {"op": "exact_fallback"} with self.assertRaisesRegex(ValueError, "contains ExactFallback"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_rejects_missing_or_cyclic_input_nodes(self): artifact = self.artifact({"op": "summary_estimate", "input": 99}) with self.assertRaisesRegex(ValueError, "missing node 99"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) artifact = self.artifact({"op": "summary_estimate", "input": 2}) with self.assertRaisesRegex(ValueError, "contains a cycle"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_requires_two_local_summary_readouts_at_runtime(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) provenance = {"summary_readout_evaluations": 2, "exact_subquery_rpcs": 0, "exact_subquery_evaluations": 0, "exact_branch_evaluations": 0} - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": provenance}]) + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": provenance}]) with self.assertRaisesRegex(ValueError, "both summary branches"): - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": { + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": { **provenance, "summary_readout_evaluations": 1}}]) with self.assertRaisesRegex(ValueError, "used exact path"): - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": { + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": { **provenance, "exact_subquery_rpcs": 1}}]) def test_accepts_one_heap_and_candidate_filtered_external_exact(self): - validate_candidate_topk_artifact(self.candidate_filtered_artifact()) + validate_membership_filter_artifact(self.candidate_filtered_artifact()) def test_candidate_filtered_contract_rejects_local_exact_state_or_unshared_input(self): artifact = self.candidate_filtered_artifact() artifact["install_request"]["precompute_plan"]["schemas"].append( {"materialization": 8, "family": {"family": "exact", "kind": "increase"}}) with self.assertRaisesRegex(ValueError, "must not install"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) artifact = self.candidate_filtered_artifact() artifact["install_request"]["query_plan"]["entries"]["q"]["nodes"]["3"]["inputs"] = [2] with self.assertRaisesRegex(ValueError, "shared membership"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_candidate_filtered_execution_requires_hybrid_one_rpc_and_one_summary_read(self): artifact = self.candidate_filtered_artifact() provenance = {"detail": "hybrid", "raw_scan_evaluations": 0, "summary_readout_evaluations": 1, "exact_subquery_rpcs": 1, "exact_subquery_evaluations": 1, "exact_branch_evaluations": 1} - validate_candidate_topk_execution( + validate_membership_filter_execution( artifact, [{"execution": "hybrid", "execution_provenance": provenance}]) for key in ("summary_readout_evaluations", "exact_subquery_rpcs", "exact_subquery_evaluations", "exact_branch_evaluations"): with self.subTest(key=key), self.assertRaisesRegex(ValueError, "invalid provenance"): - validate_candidate_topk_execution(artifact, [{"execution": "hybrid", + validate_membership_filter_execution(artifact, [{"execution": "hybrid", "execution_provenance": {**provenance, key: 0}}]) with self.assertRaisesRegex(ValueError, "hybrid execution"): - validate_candidate_topk_execution( + validate_membership_filter_execution( artifact, [{"execution": "hybrid", "execution_provenance": { **provenance, "detail": "external_exact"}}]) From 9e0a892fef50f9153219af2e089ca16aa05a0ae9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 24 Sep 2026 12:50:16 +0000 Subject: [PATCH 08/25] refactor: implement typed summary semantics and physical plan lowering --- control_plane/src/clickhouse.rs | 4 +- control_plane/src/emit/backend_wire.rs | 2 +- control_plane/src/emit/mod.rs | 2 +- control_plane/src/physical/backend_stage.rs | 2 +- control_plane/src/physical/compiler.rs | 68 +-- control_plane/src/physical/pane_reuse.rs | 5 +- control_plane/src/physical/post_asap/lower.rs | 40 +- control_plane/src/physical/post_asap/tests.rs | 32 +- .../src/physical/runtime_capability.rs | 113 ++-- control_plane/src/workload.rs | 86 +-- crates/asap_types/src/accumulator_spec.rs | 184 ++---- crates/asap_types/src/aggregation_config.rs | 109 ++-- crates/asap_types/src/aggregation_type.rs | 91 ++- crates/asap_types/src/key_by_label_names.rs | 2 +- crates/asap_types/src/monitor_spec.rs | 2 +- crates/asap_types/src/policy_fingerprint.rs | 22 +- crates/asap_types/src/policy_registry.rs | 22 +- crates/asap_types/src/precompute_plan.rs | 21 +- crates/asap_types/src/query_plan.rs | 16 + crates/asap_types/src/routing_index.rs | 19 +- crates/asap_types/src/sds.rs | 72 ++- data_plane/benches/sketch_db.rs | 4 +- data_plane/src/drivers/ingest/otel.rs | 35 +- .../drivers/ingest/prometheus_remote_write.rs | 12 +- data_plane/src/drivers/query/servers/http.rs | 465 +++------------ data_plane/src/lib.rs | 8 +- .../precompute_engine/accumulator_factory.rs | 538 +++++++++++++----- .../src/precompute_engine/erp_observer.rs | 6 +- .../src/precompute_engine/ingest_handler.rs | 22 +- .../precompute_engine/maintenance_runtime.rs | 13 +- data_plane/src/precompute_engine/mod.rs | 1 + .../operators/exact_accumulator.rs | 327 +++++++++++ ..._accumulator.rs => keyed_counter_state.rs} | 85 ++- ..._max_accumulator.rs => keyed_max_state.rs} | 69 ++- ..._min_accumulator.rs => keyed_min_state.rs} | 69 ++- ...ator.rs => keyed_sum_count_accumulator.rs} | 268 +++++++-- .../src/precompute_engine/operators/mod.rs | 17 +- .../operators/sum_accumulator.rs | 22 +- .../src/precompute_engine/output_sink.rs | 8 +- data_plane/src/precompute_engine/raw_dag.rs | 295 ++++++++++ .../src/precompute_engine/series_router.rs | 8 +- .../src/precompute_engine/window_manager.rs | 2 +- data_plane/src/precompute_engine/worker.rs | 450 ++++++++++----- .../asap_query_engine/catalog_resolver.rs | 56 +- .../asap_query_engine/exact_subqueries.rs | 4 +- .../asap_query_engine/post_asap_readout.rs | 4 +- .../asap_query_engine/summary_executor.rs | 194 ++++--- data_plane/src/query_engines/query_result.rs | 2 +- .../src/storage_engines/sketch_db/accuracy.rs | 45 +- .../storage_engines/sketch_db/backfill/mod.rs | 8 +- .../sketch_db/backfill/processor.rs | 61 +- .../sketch_db/backfill/raw_sample_reader.rs | 2 +- .../sketch_db/backfill/service.rs | 8 +- .../sketch_db/backfill/window_builder.rs | 43 +- .../src/storage_engines/sketch_db/data/mod.rs | 8 +- .../storage_engines/sketch_db/index/mod.rs | 126 +++- .../sketch_db/lifecycle/eviction.rs | 6 +- .../sketch_db/lifecycle/reconcile.rs | 14 +- .../src/storage_engines/sketch_db/mod.rs | 12 +- .../types/hot_reload_config.rs | 6 +- data_plane/src/storage_engines/types/mod.rs | 2 +- .../types/precomputed_output.rs | 4 +- .../storage_engines/types/streaming_config.rs | 298 +++------- .../accuracy_empirical_validation_tests.rs | 6 +- .../tests/test_utilities/engine_factories.rs | 36 +- data_plane/src/tests/trait_design_tests.rs | 8 +- data_plane/src/utils/file_io.rs | 7 +- .../asapquery_compatibility_process_e2e.rs | 9 +- ...e2e_controller_plans_and_backend_serves.rs | 12 +- docs/design_docs/precompute-dag-execution.md | 24 + 70 files changed, 2763 insertions(+), 1880 deletions(-) create mode 100644 data_plane/src/precompute_engine/operators/exact_accumulator.rs rename data_plane/src/precompute_engine/operators/{multiple_increase_accumulator.rs => keyed_counter_state.rs} (86%) rename data_plane/src/precompute_engine/operators/{multiple_max_accumulator.rs => keyed_max_state.rs} (81%) rename data_plane/src/precompute_engine/operators/{multiple_min_accumulator.rs => keyed_min_state.rs} (81%) rename data_plane/src/precompute_engine/operators/{multiple_sum_accumulator.rs => keyed_sum_count_accumulator.rs} (50%) create mode 100644 data_plane/src/precompute_engine/raw_dag.rs create mode 100644 docs/design_docs/precompute-dag-execution.md diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 140fb5b3e..0bbbbba3b 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -392,7 +392,7 @@ fn materialize_selected_sql( let aggregation = BackendAggregation { aggregation_id: String::new(), metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: crate::physical::compiler::physical_materialization_family(family), + family: family.clone(), window_secs, spatial_filter: String::new(), grouping: grouping.names(), @@ -617,7 +617,7 @@ fn bind_selected_node( .. } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = crate::physical::compiler::physical_materialization_family(family); + let expected = family.clone(); let selected = select_materialization( &request.precompute_plan.materializations, &table_ref, diff --git a/control_plane/src/emit/backend_wire.rs b/control_plane/src/emit/backend_wire.rs index 829d1be41..ff294e8ce 100644 --- a/control_plane/src/emit/backend_wire.rs +++ b/control_plane/src/emit/backend_wire.rs @@ -5,7 +5,7 @@ //! * the storage-routing table, which maps each metric's materialized summary //! families to the query shapes the ASAP tier serves natively versus the //! ones that belong to the archive; -//! * the aggregation and readout JSON the backend's `AggregationConfig` +//! * the aggregation and readout JSON the backend's `PrecomputeMaterialization` //! parser consumes. //! //! `backend_plan::from_stage_config` reuses [`build_backend_aggregation_json`] diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c9908cc9b..bf51c16e5 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1,7 +1,7 @@ //! Backend-facing emission for a compiled physical plan. //! //! * [`backend_wire`] builds the storage-routing table and the aggregation / -//! readout JSON the backend's `AggregationConfig` parser consumes. +//! readout JSON the backend's `PrecomputeMaterialization` parser consumes. //! * [`monitor`] carries the CDM monitor declarations. pub mod backend_wire; diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs index 165c6d111..bc60cf45a 100644 --- a/control_plane/src/physical/backend_stage.rs +++ b/control_plane/src/physical/backend_stage.rs @@ -35,7 +35,7 @@ pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, /// Source metric the aggregation runs over. Required by the backend's - /// `AggregationConfig` parser. + /// `PrecomputeMaterialization` parser. pub metric_name: String, /// Planner-owned committed summary identity. Sketch entries carry a /// validated `SketchKind` (category + algorithm + params); exact entries diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index f235e5433..77b74d16b 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1262,10 +1262,7 @@ impl DeploymentPlanCompiler { .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - // Rate is a readout over the same reset-aware counter state - // as Increase. Keep that semantic distinction in QueryPlan, - // while the physical store binds both to Increase state. - let physical_family = physical_materialization_family(&selected.family); + let physical_family = selected.family.clone(); let physical_algorithm = match &physical_family { SummaryFamilyType::ExactAggregate(kind, _) => { format!("{kind:?}").to_ascii_lowercase() @@ -1735,7 +1732,7 @@ impl DeploymentPlanCompiler { .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? .family; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != physical_materialization_family(node_family) + if materialization_family != *node_family || window_ms == 0 || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 @@ -2868,13 +2865,11 @@ pub(super) fn estimated_state_bytes( A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), A::DDSketch => 64 * 1024, A::Sum + | A::Count | A::Increase + | A::Rate | A::Min | A::Max - | A::MultipleSum - | A::MultipleIncrease - | A::MultipleMin - | A::MultipleMax | A::SingleSubpopulation | A::MultipleSubpopulation => 256, } @@ -2892,7 +2887,7 @@ fn retained_partition_count( if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( materialization.aggregation_type, - A::Increase | A::MultipleIncrease | A::Min | A::Max | A::MultipleMin | A::MultipleMax + A::Increase | A::Rate | A::Min | A::Max ) || !materialization.grouping_labels.names().is_empty() { @@ -3139,7 +3134,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -fn raw_time_series_input_contract( +pub fn raw_time_series_input_contract( expr: &QueryExpr, exact: bool, ) -> Result<(String, Option, String), String> { @@ -3335,7 +3330,7 @@ fn physical_aggregation( BackendAggregation { aggregation_id, metric_name: selected.metric.clone(), - family: physical_materialization_family(&selected.family), + family: selected.family.clone(), window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected @@ -3737,26 +3732,6 @@ fn collect_selected_materializations( Ok(selected) } -pub(crate) fn physical_materialization_family(family: &SummaryFamilyType) -> SummaryFamilyType { - match family { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) => { - // The SummaryStore Sum accumulator retains the observation count - // alongside its sum. Both logical states can share this producer. - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ) - } - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Rate, _) => { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, - ) - } - _ => family.clone(), - } -} - pub(super) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { @@ -4553,8 +4528,7 @@ pub(crate) mod tests { .find(|materialization| { matches!( materialization.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Rate ) }) .expect("reset-aware exact counter"); @@ -5175,8 +5149,7 @@ pub(crate) mod tests { .iter() .all(|m| !matches!( m.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Increase | asap_types::AggregationType::Rate ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); @@ -5734,7 +5707,7 @@ pub(crate) mod tests { } #[test] - fn rate_and_increase_share_physical_counter_state() { + fn rate_and_increase_keep_planner_families_distinct() { let mut workload = request("rate", "rate(m[1m])"); workload .queries @@ -5743,10 +5716,14 @@ pub(crate) mod tests { .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert_eq!(collector.materializations[0].algorithm, "increase"); + let algorithms: std::collections::BTreeSet<_> = collector + .materializations + .iter() + .map(|materialization| materialization.algorithm.as_str()) + .collect(); + assert_eq!(algorithms, ["increase", "rate"].into()); } } @@ -5766,8 +5743,7 @@ pub(crate) mod tests { } #[test] - fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { - // Both dashboard roots use one packed raw accumulator, with explicit readouts. + fn exact_dashboard_preserves_distinct_sum_and_count_producers() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5783,7 +5759,7 @@ pub(crate) mod tests { entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { assert!( @@ -5793,7 +5769,7 @@ pub(crate) mod tests { )), "{entry:?}" ); - assert_eq!(entry.materialization_bindings().len(), 1); + assert!(!entry.materialization_bindings().is_empty()); } assert!(bundle .query_plan @@ -7559,8 +7535,8 @@ pub(crate) mod tests { assert_eq!( materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, + planner_types::post_asap::ExactKind::Rate, + planner_types::post_asap::ExactParams::Rate, ) ); } diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index b76cdbf00..e1e29a4dc 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -34,10 +34,7 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!( - m.aggregation_type, - AggregationType::Sum | AggregationType::MultipleSum - ) + || !matches!(m.aggregation_type, AggregationType::Sum) { continue; } diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index cdff8f355..aa572cdf8 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -1,5 +1,4 @@ //! Query binding delegates selection to Planner's costed workload search. -//! Backend-specific rate normalization remains part of the physical binding. #![allow(dead_code)] @@ -118,41 +117,8 @@ fn bind_recursive( )) } - _ => { - let rewritten = rewrite_rate_to_increase(expr); - let node = crate::planner_selection::select_query(&rewritten, cost_model)?; - Ok(PostAsapPlan::Summary(node)) - } - } -} - -/// Rewrite Rate to Increase along the aggregate spine traversed by Planner. -/// This deployment computes rate by dividing the Increase readout by window -/// seconds, rather than storing a separate Rate accumulator. -fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs - .iter() - .map(|intent| { - if matches!(intent, AggIntent::Rate) { - AggIntent::Increase - } else { - intent.clone() - } - }) - .collect(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(rewrite_rate_to_increase(child)), - }, - other => other.clone(), + _ => Ok(PostAsapPlan::Summary( + crate::planner_selection::select_query(expr, cost_model)?, + )), } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 5e226419d..344b60534 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -471,13 +471,9 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { /// `ONLY_SPATIAL` — `sum by (host) (m)`. /// Control plane path: `Aggregate{Sum, by=[host]}` over a bare `Scan`. /// -/// The old locally-defined `AggregationType::MultipleSum` (keyed vs -/// unkeyed sum) identity no longer exists at the L4 IR level — -/// `SummaryKind::Sum` covers both; the keyed/unkeyed distinction now -/// lives on `SummaryAgg::by` (non-empty ⇒ the old "MultipleSum" shape), -/// per `emit::mod.rs`'s exact-accumulator classification notes. +/// Family remains Sum; the reduction carries the grouping columns. #[test] -fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { +fn phase_b_pattern_only_spatial_aggregate_binds_to_grouped_sum() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), // service column measures: vec![AggIntent::Sum { col: None }], @@ -498,7 +494,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { assert_eq!( reduction.group_keys().map(|k| k.keys()), Some(&[1][..]), - "keyed sum must carry the group-by column (the MultipleSum-equivalent signal)" + "Sum reduction must retain the group-by column" ); } other => panic!("expected SummaryAgg(Sum, by=[1]), got {other:?}"), @@ -508,14 +504,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { } /// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// `bind_query_expr` (not `implement_tree` directly) rewrites -/// `AggIntent::Rate` to `AggIntent::Increase` before binding (see -/// `lower.rs`'s `rewrite_rate_to_increase` — this deployment's data -/// plane has no Rate accumulator). The old -/// `AggregationType::MultipleIncrease` identity is now -/// `SummaryKind::Increase` with a non-empty `by`. +/// Planner preserves the Rate family and the `by` reduction independently. #[test] -fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { +fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), measures: vec![AggIntent::Rate], @@ -531,11 +522,11 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { } => { assert_eq!( family, - &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } - other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), + other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), }, other => panic!("expected Committed(Summary(_)), got {other:?}"), } @@ -668,12 +659,9 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { ); } -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Control plane path: `Aggregate{Rate}` over -/// `Window` → `bind_query_expr` rewrites `Rate` to `Increase` and binds an -/// exact accumulator (`SummaryAgg{Increase}`) — no approximate summary -/// family. Both paths produce a single non-summary streaming row; the L5 -/// emitter is the one that picks the actual MultipleIncrease processor. +/// A Rate query keeps Planner's exact Rate family through binding. The +/// physical emitter chooses the runtime processor without changing that +/// family identity. #[test] fn phase_b_e2e_rate_falls_through_to_logical() { let bound = pipeline_l1_to_l4( diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 136594268..8dc48e18d 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -101,7 +101,7 @@ pub enum Capability { /// * Sum-over-time requires archive execution because cumulative samples /// cannot be reconstructed from delta state alone. /// -/// Rate and Increase require the Increase capability; plain sum requires Sum. +/// Rate and Increase have distinct exact-family capabilities. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -275,8 +275,7 @@ impl Capability { /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` /// has no wildcard concept of its own; family-matching subsumes it. - /// `ExactAgg` is intentionally NOT routed through this path — see - /// [`multi_pop_satisfies_single`]'s doc for why. + /// Exact families require identity; keyed layout is checked separately. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { @@ -313,22 +312,9 @@ impl Capability { SketchAlgorithm::CmsWithHeap, ) } - // Exact-aggregation family: the agg_type must match - // exactly OR be the single-pop ⇆ multi-pop equivalent. A - // `MultipleSum` policy can serve a `Sum` query by - // re-aggregating across keys; the `find_matching_policies` - // group_by ⊆ policy_grouping_labels check is what - // ultimately decides whether the re-aggregation is - // semantically valid. The reverse direction (single-pop - // serving multi-pop) is NOT allowed — the single-pop - // policy has lost the key dimension and can't recover it. - // - // Exact counter summaries are a distinct state contract. A sum of - // cumulative sample values cannot reconstruct reset correction or - // Prometheus boundary extrapolation. - (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have || multi_pop_satisfies_single(*req, *have) - } + // Exact family identity must match. Grouping compatibility is + // checked separately by population routing. + (Capability::ExactAgg(req), Capability::ExactAgg(have)) => req == have, _ => false, } } @@ -348,30 +334,12 @@ fn sketch_algorithms_compatible( sketch_family_satisfied(required, available) } -/// True when `available` is the multi-population equivalent of -/// `required`'s single-population variant — i.e. a `MultipleSum` -/// policy can serve a `Sum` query (via re-aggregation across keys), -/// `MultipleIncrease` can serve `Increase`, `MultipleMax` can -/// serve `Max`. Asymmetric: this returns `false` for the reverse -/// direction (single-pop can't recover keys that have been collapsed -/// away). -fn multi_pop_satisfies_single(required: AggregationType, available: AggregationType) -> bool { - matches!( - (required, available), - (AggregationType::Sum, AggregationType::MultipleSum) - | (AggregationType::Increase, AggregationType::MultipleIncrease) - | (AggregationType::Min, AggregationType::MultipleMin) - | (AggregationType::Max, AggregationType::MultipleMax) - ) -} - // ── AggIntent → Capability bridge ──────────────────────────────────────────── #[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` for intents that have no ASAP-tier sketch -/// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent -/// — see [`AggIntent::archive_only`]). +/// answer it. Returns `None` when no deployed ASAP-tier capability can +/// satisfy the intent. /// /// This is a runtime routing requirement, not a summary-selection rule. /// ASAPPlanner owns legal implementations and candidate enumeration; this @@ -395,15 +363,17 @@ pub fn capability_for(intent: &AggIntent) -> Option { } match intent { AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Count { accuracy } if is_exact(accuracy) => { + Some(Capability::ExactAgg(AggregationType::Count)) + } // Direction is part of the capability: a stored minimum cannot // answer `max_over_time` and vice versa, so these must not // collapse onto one `ExactAgg` the way they did while Planner // had a single `MinMax` accumulator. AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase | AggIntent::Rate => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } + AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), + AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::QuantileApprox(None)) } @@ -532,18 +502,14 @@ mod tests { } #[test] - fn capability_for_count_exact_routes_to_archive() { - // `count_over_time` lowers to `Count{accuracy:Exact}`. The - // PR #200/#201 follow-up briefly routed this to - // `ExactAgg(Sum)`, but the data plane has no count - // accumulator — `SumAccumulator` returns its `sum` for both - // `Statistic::Sum` and `Statistic::Count`, so the result was - // sum-of-values, not sample-count. Reverted to `None` (archive - // routing) until a real `SumCountAccumulator` lands. + fn capability_for_count_exact_preserves_count_family() { let intent = AggIntent::Count { accuracy: AccuracyTarget::Exact, }; - assert_eq!(capability_for(&intent), None); + assert_eq!( + capability_for(&intent), + Some(Capability::ExactAgg(AggregationType::Count)) + ); } #[test] @@ -593,16 +559,16 @@ mod tests { assert!(!Capability::ExactAgg(AggregationType::Max) .is_satisfied_by(&Capability::ExactAgg(AggregationType::Min))); assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleMax))); + .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); } #[test] - fn capability_for_rate_increase_route_to_exact_agg_increase() { - // PR-6 follow-up: Rate and Increase route to ASAP-tier - // ExactAgg(Increase) — the counter-reset-aware exact precompute. - // Pre-follow-up this returned `None`. + fn capability_for_rate_and_increase_preserves_family() { let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!(capability_for(&AggIntent::Rate), exact_inc); + assert_eq!( + capability_for(&AggIntent::Rate), + Some(Capability::ExactAgg(AggregationType::Rate)) + ); assert_eq!(capability_for(&AggIntent::Increase), exact_inc); } @@ -849,10 +815,6 @@ mod tests { AggregationType::Min, AggregationType::Max, AggregationType::DatasketchesKLL, - AggregationType::MultipleSum, - AggregationType::MultipleIncrease, - AggregationType::MultipleMin, - AggregationType::MultipleMax, AggregationType::HydraKLL, AggregationType::CountMinSketch, AggregationType::CountMinSketchWithHeap, @@ -873,21 +835,16 @@ mod tests { fn sum_family_cannot_impersonate_exact_counter_state() { let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); - assert!( - !required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum)) - ); + let required_multi = Capability::ExactAgg(AggregationType::Increase); + assert!(!required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } #[test] fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { - // Same single/multi-population direction as multi_pop_satisfies_single: - // a single-pop available (Sum) can't serve a multi-pop required - // capability (MultipleIncrease) -- it already lost the per-key - // breakdown a multi-pop caller needs. - let required = Capability::ExactAgg(AggregationType::MultipleIncrease); + // A different exact family cannot supply counter state. + let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } @@ -905,30 +862,26 @@ mod tests { // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] - fn exact_agg_routing_covers_sum_rate_increase_only() { - // `Capability::ExactAgg` routing covers the three intents the - // data plane has a real accumulator for: `Sum` (SumAccumulator) - // and `Rate` / `Increase` (IncreaseAccumulator). + fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { assert_eq!( capability_for(&AggIntent::Sum { col: None }), Some(Capability::ExactAgg(AggregationType::Sum)) ); assert_eq!( capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Increase)) + Some(Capability::ExactAgg(AggregationType::Rate)) ); assert_eq!( capability_for(&AggIntent::Increase), Some(Capability::ExactAgg(AggregationType::Increase)) ); - // `Count{Exact}` (count_over_time) and `Avg` both need a real - // count accumulator that doesn't exist yet — they route to - // archive until `SumCountAccumulator` lands. + // Exact count follows the Planner Count family; Avg still needs + // its own composition contract. assert_eq!( capability_for(&AggIntent::Count { accuracy: AccuracyTarget::Exact, }), - None + Some(Capability::ExactAgg(AggregationType::Count)) ); assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); } diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 6bb7fc327..d6b74cd45 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -18,8 +18,7 @@ use planner_types::pre_asap::AggIntent; /// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` /// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): /// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Sum`] -/// (rate binds to ExactAgg(Sum)-shaped capability) +/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] /// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] /// /// Before this enum: the `WorkloadStore` was keyed by metric name alone @@ -30,7 +29,7 @@ use planner_types::pre_asap::AggIntent; /// shape). /// /// After: the store is keyed by `(metric, role)` so each shape gets its -/// own plan, its own `AggregationConfig` on the backend's streaming +/// own plan, its own `PrecomputeMaterialization` on the backend's streaming /// config, and its own routing-connector pipeline. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -39,15 +38,15 @@ pub enum AggRole { /// or workload entries with `sketch_family_override: DDSketch | KLL`. /// Routes to a quantile-shaped sketch (DDSketch / KLL). Quantile, - /// Bare counter selector, `sum(...)`, `sum_over_time(...)`, - /// `rate(...)`, `increase(...)`. All bind to ExactAgg(Sum)-shaped - /// capability on the data plane; the streaming-config emits an - /// `aggregation_type: Sum` rather than a sketch. + /// Bare selector or Sum-shaped exact aggregation. Sum, + /// Reset-aware per-second counter rate. + Rate, + /// Reset-aware counter increase over the selected window. + Increase, /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to a Sum-as-count - /// exact-aggregation. + /// to HLL when a sketch is appropriate, otherwise to exact Count. Count, /// `topk(...)`, or workload entries with /// `sketch_family_override: CountSketch | CountMinSketch`. Routes @@ -70,6 +69,8 @@ impl AggRole { match self { AggRole::Quantile => "quantile", AggRole::Sum => "sum", + AggRole::Rate => "rate", + AggRole::Increase => "increase", AggRole::Count => "count", AggRole::Topk => "topk", AggRole::Other => "other", @@ -97,15 +98,16 @@ impl std::fmt::Display for AggRole { /// through the same canonical pipeline the live serving path uses /// (`query_parser::parse_query_expr_canonical` → /// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent (the one bound to the data-plane capability) is matched: +/// intent is matched, except that a Sum wrapping a counter function +/// keeps the inner Rate or Increase role: /// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] /// * [`AggIntent::TopK`] → [`AggRole::Topk`] /// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the /// windowed-Count-as-Frequency extension /// (`intent_algebra::as_frequency`) → [`AggRole::Count`] /// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → [`AggRole::Sum`] -/// * Anything else recognised but not one of the four shapes above +/// → their respective roles +/// * Anything else recognised but not one of the listed shapes above /// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → /// [`AggRole::Other`]. /// * Bare metric selector (no `Aggregate` node at all) → @@ -125,11 +127,7 @@ impl std::fmt::Display for AggRole { /// semantics, whichever the lowerer picks). The Sum-shaped /// alternative is rare in practice; users who want it write /// `sum_over_time(count(...))` which classifies as Sum. -/// * `rate` / `irate` / `increase` — Sum. `irate` folds onto -/// `AggIntent::Rate` at L3 same as `rate`; both bind to -/// ExactAgg(Increase) on the data plane (see -/// `data_plane/src/precompute_engine/ingest_handler.rs`'s handling -/// of `AggKind::ExactAgg { Increase }`). +/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { // 1. `sketch_family_override` wins. if let Some(family) = entry.sketch_family_override.as_ref() { @@ -167,11 +165,30 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { if crate::planner_selection::as_frequency(outer).is_some() { return AggRole::Count; } + // A spatial `sum by (...)` around a counter function still requires the + // counter family's state; using Sum as the registration key would let it + // overwrite a bare Sum workload for the same metric. + if matches!(outer, AggIntent::Sum { .. }) { + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Rate)) + { + return AggRole::Rate; + } + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Increase)) + { + return AggRole::Increase; + } + } match outer { AggIntent::Quantile { .. } => AggRole::Quantile, AggIntent::TopK { .. } => AggRole::Topk, AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } | AggIntent::Rate | AggIntent::Increase => AggRole::Sum, + AggIntent::Sum { .. } => AggRole::Sum, + AggIntent::Rate => AggRole::Rate, + AggIntent::Increase => AggRole::Increase, _ => AggRole::Other, } } @@ -970,13 +987,7 @@ mod tests { #[test] fn agg_role_sum_query_strings() { - for q in [ - "sum by (zone) (m)", - "sum_over_time(m[5m])", - "rate(m[5m])", - "increase(m[5m])", - "sum by (zone) (rate(m[5m]))", - ] { + for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { assert_eq!( derive_agg_role(&entry("m", Some(q), None)), AggRole::Sum, @@ -985,6 +996,18 @@ mod tests { } } + #[test] + fn counter_functions_have_distinct_workload_roles() { + for (query, expected) in [ + ("rate(m[5m])", AggRole::Rate), + ("sum by (zone) (rate(m[5m]))", AggRole::Rate), + ("increase(m[5m])", AggRole::Increase), + ("sum by (zone) (increase(m[5m]))", AggRole::Increase), + ] { + assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); + } + } + #[test] fn agg_role_count_query_strings() { for q in [ @@ -1095,7 +1118,7 @@ mod tests { } #[test] - fn three_synthetic_http_requests_total_entries_classify_to_two_distinct_roles() { + fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { // Synthetic mirror of `deploy/configs/mvp-workload.yaml` // entries 2/3/4 — proves `derive_agg_role` produces distinct // roles for the three http_requests_total shapes. Pre-B2 the @@ -1120,15 +1143,8 @@ mod tests { ), ]; let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Sum, AggRole::Count]); - // The store distinguishes Sum vs Count keys, so two of the - // three entries (the two Sum-shaped ones) still collide - // under (metric, role). That's the documented behaviour — - // two YAML entries with the SAME (metric, role) overwrite, - // which is the legitimate "operator updated their workload" - // path. The fix scope is collisions across DIFFERENT shapes, - // not idempotent re-registers. + assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); + assert_eq!(distinct.len(), 3); } } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 482ef6d67..bdf5949b4 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,68 +1,12 @@ -//! Typed accumulator dispatch derived from legacy streaming config. +//! Validate stored materialization descriptors against Planner summary families. //! -//! The semantic identity is ASAPPlanner's [`SummaryFamilyType`]. This module -//! only adds the backend execution concern of keyed versus unkeyed state and -//! adapts the stable legacy wire fields into that canonical representation. -//! -//! ## This is an additive representation, not a replacement (yet) -//! -//! `AggregationConfig` keeps its `aggregation_type` / `aggregation_sub_type` -//! / `parameters` fields untouched. Two hard constraints ruled out full -//! removal in this pass: -//! -//! 1. **`PolicyFingerprint` hash stability.** [`crate::policy_fingerprint`] -//! hashes `aggregation_type` / `aggregation_sub_type` / `parameters` -//! directly, and its own module doc is explicit that the byte layout -//! it produces is a stability *contract* ("Don't reorder fields... -//! any such change invalidates every deployed fingerprint and forces -//! a cold-start rebuild"). Changing what feeds that hash — even by -//! routing it through an equivalent typed shape — risks producing a -//! different byte sequence for the same logical policy, which strands -//! on-disk sids after a deploy. `policy_fingerprint.rs` is -//! deliberately **not touched** by this module; it keeps reading the -//! original three fields, unchanged. -//! 2. **Consumer fan-out.** `AggregationType` is read by ~40 files across -//! `data_plane` and `asap_types` — persistence (`sid_metadata.json` -//! round-trip), query-time capability matching -//! (`capability_matching.rs`, unrelated to accumulator dispatch), -//! the query engine, reconciliation, index maintenance — not just -//! `accumulator_factory.rs` (the single highest-risk consumer, and -//! the one this module targets). Migrating all of them in one PR was -//! judged too large to land and review safely; that's tracked as -//! follow-up, not done here. -//! -//! So: `AccumulatorSpec` is *computed from* `AggregationConfig`'s -//! existing fields via [`AggregationConfig::accumulator_spec`], and -//! consumed by `data_plane::precompute_engine::accumulator_factory` -//! instead of the raw fields. The wire format (`aggregationType` / -//! `aggregationSubType` / `parameters` JSON/YAML keys) is completely -//! unaffected — nothing here changes how `AggregationConfig::from_yaml` -//! / `from_json` parse or how `serialize_to_json` emits. -//! -//! Backend-specific execution details remain deliberately separate: -//! -//! - **Min/max direction.** Direction is part of the family now, not a -//! string riding alongside it: `AggregationType::{Min, Max}` (and the -//! keyed `{MultipleMin, MultipleMax}`) map to `ExactKind::Min` and -//! `ExactKind::Max` respectively — upstream still spells its -//! maximum accumulator `MinMax`, but it is a maximum. Nothing reads -//! `AggregationConfig::aggregation_sub_type` for the direction any -//! more, so a min state can no longer content-address onto a max one. -//! - **HydraKLL's `(row, col)` tiling.** `SketchParams::Kll` carries -//! only `k` — upstream has no concept of the CMS-like grid-of-KLL-cells -//! layout `HydraKllSketchAccumulator` uses to parallelize a keyed KLL -//! across many populations. `accumulator_factory.rs` calls -//! [`cms_params`] directly for keyed KLL execution -//! arm, same extraction the plain CMS arms use, because `w`/`d` are -//! genuinely the same wire keys for both. -//! - **Top-k ranking mode (`weight_mode`).** Not a sketch structural -//! parameter — a data_plane-only "what to accumulate" axis -//! (`accumulator_factory::TopkWeight`) with no upstream equivalent. -//! Stays a raw-`parameters`-reading helper in `accumulator_factory.rs`. +//! This projection supports catalog identity and imported state metadata. It is +//! not an execution program. Raw and maintenance execution dispatch directly +//! on the selected post-ASAP DAG payload; the descriptor must agree with it. use serde_json::Value; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; use planner_types::post_asap::{ @@ -75,8 +19,8 @@ use planner_types::post_asap::{ /// accumulator to run (`kind`), with what tuning (`params`), and /// whether it's keyed by a group-by label set (`grouping`). /// -/// Computed on demand from an [`AggregationConfig`] via -/// [`AggregationConfig::accumulator_spec`] — not stored on the config +/// Computed on demand from an [`PrecomputeMaterialization`] via +/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config /// itself, so there is exactly one source of truth for the fields that /// feed [`crate::policy_fingerprint::PolicyFingerprint`]. #[derive(Debug, Clone, PartialEq)] @@ -85,10 +29,7 @@ pub struct AccumulatorSpec { /// validated `SketchKind` (category + algorithm + params), following the /// ASAP-aware-mapping vocabulary. pub family: SummaryFamilyType, - /// `Some(labels)` for a keyed (multi-population) accumulator, - /// `None` for a single-population one. This is the axis - /// `AggregationType` wrongly folded into identity (`Sum` vs - /// `MultipleSum`) — here it's a sibling field instead. + /// Physical keyed-state layout, independent of semantic family. pub grouping: Option, } @@ -96,8 +37,8 @@ pub struct AccumulatorSpec { /// /// This is execution semantics, separate from the summary family: the same /// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Legacy streaming artifacts still encode the rule in -/// `parameters`; callers use [`AggregationConfig::sample_update_rule`] so the +/// counter deltas. Stored descriptors encode the rule in +/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the /// runtime does not branch on ad-hoc strings. #[derive(Debug, Clone, Copy, PartialEq)] pub enum SampleUpdateRule { @@ -134,7 +75,7 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) } -impl AggregationConfig { +impl PrecomputeMaterialization { pub fn sample_update_rule(&self) -> SampleUpdateRule { let scale = self .parameters @@ -157,23 +98,14 @@ impl AggregationConfig { } } -/// Why [`AggregationConfig::accumulator_spec`] couldn't resolve a config -/// into an [`AccumulatorSpec`]. Each variant matches one of the three -/// distinct fallback paths `accumulator_factory::create_accumulator_updater` -/// took pre-Step-5 — preserved verbatim (including which default -/// updater and which warning text each one produced) so this refactor -/// changes *how* the dispatch is expressed, not what it does for any -/// input. +/// A storage descriptor cannot be resolved to a supported Planner family. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccumulatorSpecError { /// `aggregation_type` was `SingleSubpopulation` with an /// `aggregation_sub_type` string not in the recognized alias list. - /// Pre-Step-5 this defaulted to `SumAccumulatorUpdater`. UnknownSingleSubpopulationSubType(String), /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. Pre-Step-5 this defaulted - /// to `MultipleSumAccumulatorUpdater` (note: a *different* default - /// than the `SingleSubpopulation` case). + /// unrecognized `aggregation_sub_type`. UnknownMultipleSubpopulationSubType(String), /// `aggregation_type` itself has no accumulator-dispatch mapping. /// Also returned for an invalid HLL precision. A resolved family identifies @@ -185,36 +117,24 @@ impl std::fmt::Display for AccumulatorSpecError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::UnknownSingleSubpopulationSubType(s) => { - write!( - f, - "Unknown SingleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown SingleSubpopulation sub_type '{s}'") } Self::UnknownMultipleSubpopulationSubType(s) => { - write!( - f, - "Unknown MultipleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") } - Self::UnmappedAggregationType(t) => write!( - f, - "Unknown aggregation_type '{t:?}', defaulting to SingleSubpopulation Sum" - ), + Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), } } } impl std::error::Error for AccumulatorSpecError {} -impl AggregationConfig { +impl PrecomputeMaterialization { /// Resolve this config's `(aggregation_type, aggregation_sub_type, /// parameters)` triple into a typed [`AccumulatorSpec`]. /// - /// Mirrors `accumulator_factory::create_accumulator_updater`'s - /// pre-Step-5 dispatch exactly — same sub_type alias lists, same - /// numeric defaults, same three fallback paths (see - /// [`AccumulatorSpecError`]) — just re-expressed as data instead of - /// as a 14-arm match baked into the accumulator constructor. + /// Unsupported descriptors return an error; this projection never chooses + /// a fallback family and cannot authorize DAG execution. pub fn accumulator_spec(&self) -> Result { use AggregationType::*; @@ -227,21 +147,9 @@ impl AggregationConfig { ) }; let (family, keyed) = match self.aggregation_type { - Sum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - Increase => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - Min => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - Max => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, + Sum | Count | Increase | Rate | Min | Max => ( + self.aggregation_type.planner_exact_family().unwrap(), + !self.aggregated_labels.is_empty(), ), DatasketchesKLL => ( independent_sketch( @@ -252,22 +160,6 @@ impl AggregationConfig { ), false, ), - MultipleSum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - MultipleIncrease => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - MultipleMin => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - MultipleMax => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), HydraKLL => { let k = kll_k_param(self) as u32; ( @@ -497,7 +389,7 @@ impl AggregationConfig { /// Extract the KLL `k` parameter. Capital `"K"` takes precedence over /// lowercase `"k"` to match the convention used by the top-level /// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &AggregationConfig) -> u16 { +pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { config .parameters .get("K") @@ -513,7 +405,7 @@ pub fn kll_k_param(config: &AggregationConfig) -> u16 { /// matches what the control plane's `sketch_params_to_json` emits and /// what `sketch_config_to_params` uses for OTLP policy_fp content /// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { +pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { let row_num = config .parameters .get("d") @@ -530,7 +422,7 @@ pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { /// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` /// from `parameters`; defaults to 20 (the heap holds the top-k /// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &AggregationConfig) -> usize { +pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { config .parameters .get("heap_size") @@ -545,7 +437,7 @@ pub fn heap_size_param(config: &AggregationConfig) -> usize { /// Pull `relativeAccuracy` (or canonical aliases) out of a /// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, /// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { +pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { let parsed = param_f64(config, "relativeAccuracy") .or_else(|| param_f64(config, "relative_accuracy")) .or_else(|| param_f64(config, "alpha")) @@ -561,7 +453,7 @@ pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { } } -fn param_f64(config: &AggregationConfig, key: &str) -> Option { +fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { config.parameters.get(key).and_then(Value::as_f64) } @@ -599,8 +491,8 @@ mod tests { sub_type: &str, params: HashMap, grouping_labels: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), params, @@ -630,13 +522,9 @@ mod tests { } #[test] - fn multiple_sum_is_keyed_sum() { - let cfg = make_config( - AggregationType::MultipleSum, - "", - HashMap::new(), - vec!["zone"], - ); + fn keyed_layout_preserves_sum_family() { + let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); + cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); let spec = cfg.accumulator_spec().expect("resolves"); assert_exact(&spec, ExactKind::Sum); assert_eq!( @@ -862,16 +750,16 @@ mod tests { assert_eq!( AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown SingleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown MultipleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL', defaulting to SingleSubpopulation Sum" + "Unknown aggregation_type 'HLL'" ); } @@ -905,7 +793,7 @@ mod tests { // ---- PolicyFingerprint stability guard --------------------------- /// `accumulator_spec()` must be a pure, additional *read* of - /// `AggregationConfig` — it must not change what + /// `PrecomputeMaterialization` — it must not change what /// `PolicyFingerprint::from_config` hashes. This locks in a fixed /// fingerprint for a fixed config as a tripwire: if this test ever /// needs its expected constant updated, `policy_fingerprint.rs` diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index d16b6df7e..51f9813e9 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -87,8 +87,9 @@ impl WindowMaterializationLayout { } } -/// Per-aggregation policy with content-derived [`PolicyFingerprint`] identity. -/// An `aggregationId` field in input YAML is ignored for compatibility. +/// Physical materialization metadata with content-derived identity. +/// This descriptor cannot authorize execution: the enclosing PrecomputePlan +/// must bind it to a compatible Planner DAG producer. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputeMaterialization { /// Explicit deployment output allocation; independent of semantic identity. @@ -202,10 +203,6 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} -/// Compatibility name for legacy streaming-config and precompute call sites. -/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. -pub type AggregationConfig = PrecomputeMaterialization; - impl PrecomputeMaterialization { pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection @@ -346,7 +343,7 @@ impl PrecomputeMaterialization { /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s - /// `HashMap` keys). **Always** equal to + /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. pub fn policy_fp_u64(&self) -> u64 { @@ -826,12 +823,18 @@ mod tests { /// SAME config as a fixture without it. #[test] fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = - AggregationConfig::from_yaml_data(&sample_yaml(true), None, QueryLanguage::PromQl) - .expect("parse ok"); - let without = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse ok"); + let with = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(true), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); + let without = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); assert_eq!( with.policy_fingerprint(), without.policy_fingerprint(), @@ -842,10 +845,18 @@ mod tests { /// Round-tripping the same content yields the same fingerprint. #[test] fn fingerprint_is_deterministic_per_content() { - let a = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse a"); - let b = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse b"); + let a = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse a"); + let b = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse b"); assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); assert_ne!( a.policy_fingerprint().as_u64(), @@ -870,37 +881,44 @@ mod tests { ] { yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); let config = - AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) + .unwrap(); assert_eq!(config.window_layout, layout); let mut wire = config.serialize_to_json(); wire["groupingLabels"] = serde_json::to_value(&config.grouping_labels).unwrap(); wire["aggregatedLabels"] = serde_json::to_value(&config.aggregated_labels.labels).unwrap(); wire["rollupLabels"] = serde_json::to_value(&config.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&wire).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); assert_eq!(decoded.window_layout, layout); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); wire["window_layout"] = wire["windowLayout"].clone(); - assert!(AggregationConfig::deserialize_from_json(&wire).is_err()); + assert!(PrecomputeMaterialization::deserialize_from_json(&wire).is_err()); } yaml.as_mapping_mut() .unwrap() .remove(serde_yaml::Value::from("windowLayout")); - let legacy = AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + let legacy = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); assert_eq!( legacy.window_layout, WindowMaterializationLayout::Pane { pane_secs: 10 } ); yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!(AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err()); + assert!( + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() + ); } #[test] fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let mut epoch = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let unknown = epoch.policy_fingerprint(); epoch.pane_origin_ms = Some(7_000); let planned = epoch.policy_fingerprint(); @@ -918,13 +936,13 @@ mod tests { .as_object_mut() .unwrap() .insert("paneOriginMs".into(), origin); - let decoded: AggregationConfig = serde_json::from_value(derived.clone()).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(derived.clone()).unwrap(); assert_eq!(decoded.pane_origin_ms, Some(7_000)); let mut legacy = derived; legacy.as_object_mut().unwrap().remove("paneOriginMs"); assert_eq!( - serde_json::from_value::(legacy) + serde_json::from_value::(legacy) .expect("decode legacy wire") .pane_origin_ms, None @@ -934,18 +952,24 @@ mod tests { /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. #[test] fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); } /// PR 5: `serialize_to_json` no longer emits `aggregationId`. #[test] fn serialize_to_json_omits_aggregation_id() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let json = cfg.serialize_to_json(); assert!( json.get("aggregationId").is_none(), @@ -957,9 +981,12 @@ mod tests { fn typed_projection_roundtrips_and_legacy_column_keeps_identity() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .unwrap(); + let mut config = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .unwrap(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -968,7 +995,7 @@ mod tests { let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); - let decoded: AggregationConfig = serde_json::from_value(legacy).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(legacy).unwrap(); assert_eq!(decoded.policy_fingerprint(), column_identity); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), @@ -985,8 +1012,8 @@ mod tests { "rollup": config.rollup_labels.serialize_to_json(), }); assert!(wire.get("valueColumn").is_none()); - let json = AggregationConfig::deserialize_from_json(&wire).unwrap(); - let yaml = AggregationConfig::from_yaml_data( + let json = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); + let yaml = PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(&wire).unwrap(), None, QueryLanguage::ClickHouseSql, @@ -1002,8 +1029,8 @@ mod tests { ); let mut conflicting = wire; conflicting["valueColumn"] = serde_json::json!("other_column"); - assert!(AggregationConfig::deserialize_from_json(&conflicting).is_err()); - assert!(AggregationConfig::from_yaml_data( + assert!(PrecomputeMaterialization::deserialize_from_json(&conflicting).is_err()); + assert!(PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(conflicting).unwrap(), None, QueryLanguage::ClickHouseSql diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ccdcbec0d..647f7604f 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -14,15 +14,13 @@ use std::str::FromStr; pub enum AggregationType { // ---------- single-population (non-keyed) ---------- Sum, + Count, Increase, + Rate, Min, Max, DatasketchesKLL, // ---------- multi-population (keyed) ---------- - MultipleSum, - MultipleIncrease, - MultipleMin, - MultipleMax, HydraKLL, CountMinSketch, CountMinSketchWithHeap, @@ -38,17 +36,31 @@ pub enum AggregationType { } impl AggregationType { + /// Adapt a storage/processor tag to Planner's exact family. Keyed storage + /// changes the payload layout, not the semantic family. + pub fn planner_exact_family(self) -> Option { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + _ => return None, + }; + Some(SummaryFamilyType::ExactAggregate(kind, params)) + } + pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", + AggregationType::Count => "Count", AggregationType::Increase => "Increase", + AggregationType::Rate => "Rate", AggregationType::Min => "Min", AggregationType::Max => "Max", AggregationType::DatasketchesKLL => "DatasketchesKLL", - AggregationType::MultipleSum => "MultipleSum", - AggregationType::MultipleIncrease => "MultipleIncrease", - AggregationType::MultipleMin => "MultipleMin", - AggregationType::MultipleMax => "MultipleMax", AggregationType::HydraKLL => "HydraKLL", AggregationType::CountMinSketch => "CountMinSketch", AggregationType::CountMinSketchWithHeap => "CountMinSketchWithHeap", @@ -67,10 +79,6 @@ impl AggregationType { matches!( self, AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax | AggregationType::CountMinSketch | AggregationType::CountMinSketchWithHeap | AggregationType::CountSketch @@ -93,14 +101,12 @@ impl FromStr for AggregationType { match s { // Canonical names "Sum" => Ok(AggregationType::Sum), + "Count" => Ok(AggregationType::Count), "Increase" => Ok(AggregationType::Increase), + "Rate" => Ok(AggregationType::Rate), "Min" => Ok(AggregationType::Min), "Max" => Ok(AggregationType::Max), "DatasketchesKLL" => Ok(AggregationType::DatasketchesKLL), - "MultipleSum" => Ok(AggregationType::MultipleSum), - "MultipleIncrease" => Ok(AggregationType::MultipleIncrease), - "MultipleMin" => Ok(AggregationType::MultipleMin), - "MultipleMax" => Ok(AggregationType::MultipleMax), "HydraKLL" => Ok(AggregationType::HydraKLL), "CountMinSketch" => Ok(AggregationType::CountMinSketch), "CountMinSketchWithHeap" => Ok(AggregationType::CountMinSketchWithHeap), @@ -121,12 +127,6 @@ impl FromStr for AggregationType { "DatasketchesKLLAccumulator" | "KLL" | "kll" | "datasketches_kll" => { Ok(AggregationType::DatasketchesKLL) } - "MultipleSumAccumulator" | "multiple_sum" => Ok(AggregationType::MultipleSum), - "MultipleIncreaseAccumulator" | "multiple_increase" => { - Ok(AggregationType::MultipleIncrease) - } - "MultipleMinAccumulator" | "multiple_min" => Ok(AggregationType::MultipleMin), - "MultipleMaxAccumulator" | "multiple_max" => Ok(AggregationType::MultipleMax), "HydraKllSketchAccumulator" | "hydra_kll" => Ok(AggregationType::HydraKLL), "CountMinSketchAccumulator" | "CMS" | "cms" | "count_min_sketch" => { Ok(AggregationType::CountMinSketch) @@ -149,7 +149,7 @@ impl FromStr for AggregationType { | "MultipleMinMaxAccumulator" | "multiple_min_max" => Err(format!( "Retired aggregation type: '{s}' -- min and max are separate types now, \ - use 'Min'/'Max' (or 'MultipleMin'/'MultipleMax')" + use 'Min'/'Max'" )), _ => Err(format!("Unknown aggregation type: '{s}'")), } @@ -168,3 +168,48 @@ impl<'de> Deserialize<'de> for AggregationType { s.parse().map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + /// Removed layout tags cannot be installed as semantic families. + #[test] + fn rejects_keyed_family_aliases() { + for name in [ + "MultipleSum", + "MultipleIncrease", + "MultipleMin", + "MultipleMax", + ] { + assert!(name.parse::().is_err(), "{name}"); + } + } + + #[test] + fn storage_layout_tags_do_not_create_planner_families() { + for (storage, expected) in [ + (AggregationType::Sum, ExactKind::Sum), + (AggregationType::Count, ExactKind::Count), + (AggregationType::Increase, ExactKind::Increase), + (AggregationType::Rate, ExactKind::Rate), + ] { + let family = storage.planner_exact_family().unwrap(); + assert!( + matches!(family, SummaryFamilyType::ExactAggregate(kind, _) if kind == expected) + ); + } + assert_eq!( + AggregationType::Rate.planner_exact_family(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Rate, + ExactParams::Rate + )) + ); + assert_ne!( + AggregationType::Rate.planner_exact_family(), + AggregationType::Increase.planner_exact_family() + ); + } +} diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index deb7fe3f1..5cd902b7d 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -2,7 +2,7 @@ //! //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` -//! (`AggregationConfig::grouping_labels`, `PolicyFingerprint`, +//! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, //! `PolicyRegistry`, `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index 535ec3ef0..a22352be6 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -44,7 +44,7 @@ impl MonitorFunctional { /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both -/// sides can reach — same reasoning as `AggregationConfig`/`PolicyFingerprint`. +/// sides can reach — same reasoning as `PrecomputeMaterialization`/`PolicyFingerprint`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct MonitorSpec { pub agg_id: u64, diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index 18847d73b..02d55a5fc 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -1,6 +1,6 @@ //! Legacy routing wrapper for a deployed stored output. //! -//! An explicit `AggregationConfig::stored_output_id` takes precedence. +//! An explicit `PrecomputeMaterialization::stored_output_id` takes precedence. //! Otherwise the compiler allocates a deterministic default from the existing //! policy fields (including pane layout and cadence). This identifier is not //! semantic identity: `SummaryDefinitionId` hashes the versioned semantic @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use xxhash_rust::xxh64::xxh64; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; /// Routing handle for one deployed stored output. See the module contract. #[derive( @@ -33,14 +33,14 @@ impl PolicyFingerprint { } impl PolicyFingerprint { - /// Compute the fingerprint of an [`AggregationConfig`]. + /// Compute the fingerprint of an [`PrecomputeMaterialization`]. /// /// Hash inputs are concatenated with `\0` byte separators and /// canonicalized so that map/iteration order can't affect the /// outcome. Parameter values are rendered via `serde_json::to_string` /// for nested-shape determinism (matches the existing /// `parameters_canonical` form used in `AggKind::ExactAgg`). - pub fn from_config(cfg: &AggregationConfig) -> Self { + pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { if let Some(output) = cfg.stored_output_id { return output.fingerprint(); } @@ -226,8 +226,8 @@ mod tests { group_by: Vec<&str>, window_size: u64, spatial_filter: &str, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -259,7 +259,7 @@ mod tests { ); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); let mut canonical = legacy.clone(); @@ -267,7 +267,7 @@ mod tests { assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); use crate::traits::SerializableToSink; let mut sink = canonical.serialize_to_json(); @@ -276,7 +276,7 @@ mod tests { sink["aggregatedLabels"] = serde_json::to_value(&canonical.aggregated_labels.labels).unwrap(); sink["rollupLabels"] = serde_json::to_value(&canonical.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&sink).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&sink).unwrap(); assert_eq!( decoded.population_key_encoding, canonical.population_key_encoding @@ -423,7 +423,7 @@ mod tests { ); } - /// Pre-PR-5 the `aggregation_id` field on `AggregationConfig` was + /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was /// excluded from the fingerprint hash. PR 5 deletes the field /// entirely — identity *is* the fingerprint — so this is now /// vacuously true. Kept as a doc-comment anchor; no runtime test @@ -486,7 +486,7 @@ mod tests { fn spatial_filter_canonicalization_drives_fingerprint() { // Two filters that differ only in matcher ordering produce the // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `AggregationConfig::new` (via + // canonicalization step in `PrecomputeMaterialization::new` (via // `normalize_spatial_filter`) sorts matchers by key. let a = cfg( "http_lat", diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 4b4f9e95c..dde5e5cce 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -1,7 +1,7 @@ //! Content-addressed policy registry. //! -//! Derived view over a collection of `AggregationConfig`s that maps -//! [`PolicyFingerprint`] → [`AggregationConfig`]. This is the +//! Derived view over a collection of `PrecomputeMaterialization`s that maps +//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated //! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` //! carries (see `data_plane::storage_engines::types::streaming_config`'s @@ -20,7 +20,7 @@ //! //! ## Identity invariants //! -//! Two `AggregationConfig`s that produce the same `PolicyFingerprint` +//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* //! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write @@ -30,13 +30,13 @@ use std::collections::HashMap; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; /// Content-addressed lookup table for active aggregation policies. #[derive(Debug, Clone, Default)] pub struct PolicyRegistry { - policies: HashMap, + policies: HashMap, } impl PolicyRegistry { @@ -46,7 +46,7 @@ impl PolicyRegistry { /// them. pub fn from_configs(configs: I) -> Self where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); for cfg in configs { @@ -63,7 +63,7 @@ impl PolicyRegistry { /// surfacing. pub fn from_configs_with_collisions(configs: I) -> (Self, usize) where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); let mut collisions = 0usize; @@ -77,12 +77,12 @@ impl PolicyRegistry { } /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.policies.get(&fp) } /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.policies.iter() } @@ -110,11 +110,11 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(_id: u64, metric: &str) -> AggregationConfig { + fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is derived from // content. Kept as a parameter so existing call sites in the // tests below don't churn. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 547677956..de0641754 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -101,11 +101,10 @@ pub struct PlanEnvelope { pub capability_snapshot_id: String, } -/// Backend-side materialization projection consumed by the streaming -/// precompute engine. This is deliberately config-driven: it contains no -/// PromQL string or ad-hoc scheduler job. The aggregation definitions are -/// emitted to `/api/v1/streaming-config`, where the runtime matches incoming -/// series, maintains windows, and writes content-addressed materializations. +/// DAG-format precompute installation. Planner node payloads and dependency +/// edges define execution; materializations attach storage/window placement. +/// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths +/// execute through the maintenance DAG scheduler at stored-state frontiers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -156,6 +155,8 @@ pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, ExactAccumulatorV1, + /// Persisted backend state with explicit Planner family and population layout. + PlannerExactAccumulatorV1, ExactCounterAccumulatorV2, } @@ -934,8 +935,14 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate, _, - ) => vec![StateEncoding::ExactCounterAccumulatorV2], - SummaryFamilyType::ExactAggregate(..) => vec![StateEncoding::ExactAccumulatorV1], + ) => vec![ + StateEncoding::ExactCounterAccumulatorV2, + StateEncoding::PlannerExactAccumulatorV1, + ], + SummaryFamilyType::ExactAggregate(..) => vec![ + StateEncoding::ExactAccumulatorV1, + StateEncoding::PlannerExactAccumulatorV1, + ], SummaryFamilyType::Sketch(kind, _) if matches!( kind.algorithm(), diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 93eb4ae08..445f21daa 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -672,6 +672,22 @@ pub enum ExactReadout { Max, } +impl ExactReadout { + /// Planner family required by this installed DAG readout node. + pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + }; + SummaryFamilyType::ExactAggregate(kind, params) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryReadout { diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 4e0dd5153..40a90fe36 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `AggregationConfig`s, so it represents planned policy +//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -32,7 +32,7 @@ use std::collections::{BTreeSet, HashMap}; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; use crate::policy_registry::PolicyRegistry; @@ -62,7 +62,7 @@ impl RoutingIndex { /// Tier 1 — exact fingerprint lookup. Delegates to the underlying /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.registry.get(fp) } @@ -136,8 +136,8 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(metric: &str) -> AggregationConfig { - AggregationConfig::new( + fn cfg(metric: &str) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -172,7 +172,7 @@ mod tests { fn multiple_policies_for_the_same_metric_all_bucket_together() { // Same metric, distinct group-by shapes -> distinct fingerprints, // same bucket. - let a = AggregationConfig::new( + let a = PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -220,8 +220,9 @@ mod tests { #[test] fn len_and_is_empty_match_registry() { - let idx = - RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let idx = RoutingIndex::build(PolicyRegistry::from_configs( + Vec::::new(), + )); assert!(idx.is_empty()); assert_eq!(idx.len(), 0); @@ -234,7 +235,7 @@ mod tests { fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { let mut parameters = StdHashMap::new(); parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::DDSketch, String::new(), parameters, diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 9f4a50cfc..752d49a45 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -469,6 +469,9 @@ pub enum SummaryOperator { /// Complete planner materialization configuration, including heap/Hydra /// dimensions and readout/update subtype. Never equal to a legacy projection. Configured { + /// Planner-selected semantic family; grouping and pane layout live in + /// the data descriptor and summary definition, respectively. + family: planner_types::post_asap::SummaryFamilyType, aggregation_type: AggregationType, aggregation_sub_type: String, parameters: BTreeMap, @@ -684,13 +687,48 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if let SummaryOperator::Configured { + family, + aggregation_type, + .. + } = &operator + { + if let Some(expected) = aggregation_type.planner_exact_family() { + if family != &expected { + return Err(SdsError( + "configured storage type disagrees with Planner family".into(), + )); + } + } else { + use AggregationType as A; + let expected = match aggregation_type { + A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), + A::CountMinSketch => Some(SketchAlgorithm::Cms), + A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), + A::CountSketch => Some(SketchAlgorithm::CountSketch), + A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), + A::DDSketch => Some(SketchAlgorithm::DDSketch), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let Some(expected) = expected { + if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) + { + return Err(SdsError( + "configured sketch storage disagrees with Planner family".into(), + )); + } + } + } + } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), )); } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); + let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); Ok(Self { id, operator, @@ -726,6 +764,10 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { + family: config + .accumulator_spec() + .map_err(|error| SdsError(error.to_string()))? + .family, aggregation_type: config.aggregation_type, aggregation_sub_type: config.aggregation_sub_type.clone(), parameters: config @@ -753,11 +795,8 @@ impl FidelityGuarantee { matches!( (aggregation_type, self), (A::UnivMon, UnivMonFrequency { .. }) - | ( - A::Sum | A::MultipleSum | A::Min | A::Max | A::MultipleMin | A::MultipleMax, - Exact - ) - | (A::Increase | A::MultipleIncrease, ExactCounter { .. }) + | (A::Sum | A::Count | A::Min | A::Max, Exact) + | (A::Increase | A::Rate, ExactCounter { .. }) | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) | (A::DDSketch, DdSketchRelativeError { .. }) | (A::HLL, HllCardinalityError { .. }) @@ -1583,6 +1622,7 @@ mod tests { .is_err()); assert!(SummaryDescriptor::new( SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::new(), @@ -1607,6 +1647,24 @@ mod tests { ) .is_err()); } + + #[test] + fn configured_descriptor_rejects_family_storage_disagreement() { + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + family: AggregationType::Rate.planner_exact_family().unwrap(), + aggregation_type: AggregationType::Increase, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, + 2, + ) + .is_err()); + } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); @@ -1663,11 +1721,13 @@ mod tests { #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), }; let b = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 01162475e..11cbfa4a0 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -395,13 +395,13 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; use std::collections::HashMap; - let cfg = AggregationConfig::new( + let cfg = PrecomputeMaterialization::new( AT::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 070ea25b6..fea49a0a2 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -582,7 +582,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t } /// Resolve the bucket sid (and `policy_fp`) for a single data point -/// against a single matching `AggregationConfig`. +/// against a single matching `PrecomputeMaterialization`. /// /// B7.6 — sid is the bucket identity in the precompute engine; this /// helper folds `(config, grouping-label-values)` into a single u64 via @@ -602,7 +602,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t /// their separate wire-level identity protocol. fn resolve_bucket_sid_for_agg_config( ingest_state: &Arc, - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, point_labels: &HashMap, captured_generation: Option<&asap_types::sds::CatalogGeneration>, ) -> Result<(u64, asap_types::PolicyFingerprint), String> { @@ -1798,15 +1798,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Detection is independent of the legacy dual-write // (it only drives the routed/unconfigured accounting), // so we walk it whether or not the worker push fires. - let matching_configs: Vec<&asap_types::aggregation_config::AggregationConfig> = - agg_configs - .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) - .collect(); + let matching_configs: Vec< + &asap_types::aggregation_config::PrecomputeMaterialization, + > = agg_configs + .values() + .filter(|config| { + config.metric == canonical_name + || config.spatial_filter_normalized == canonical_name + || config.spatial_filter == canonical_name + }) + .collect(); let matched_any = !matching_configs.is_empty(); // CQ-2 — only pay the worker push (and the per-config @@ -1884,7 +1885,7 @@ async fn route_modified_otlp_sketches_to_precompute( routed += 1; } else { // CQ-6 — a decoded sketch that matched no - // AggregationConfig in the running streaming config. + // PrecomputeMaterialization in the running streaming config. ingest_state .observability .dropped_unconfigured @@ -1929,7 +1930,7 @@ async fn route_modified_otlp_sketches_to_precompute( /// `AggregationType`. Inverse direction is in /// `sketch_algorithm_for` above. Used by /// [`derive_sketch_policy_fp`] to find the policy whose -/// `AggregationConfig.aggregation_type` matches a freshly-ingested +/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested /// sketch. /// /// `Any` is a control-plane analysis-time wildcard — it doesn't @@ -3440,7 +3441,7 @@ mod policy_fp_lookup_tests { fn sketch_config_to_params_uses_canonical_keys() { // The param-name vocabulary must match what the control plane // writes in streaming-config YAML (see - // `asap_types::aggregation_config::AggregationConfig::from_yaml_data`). + // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). // Drift surfaces as `find_policy_by_content` missing matches. let dd = sketch_config_to_params(&SketchConfig::DDSketch { relative_accuracy: 0.01, @@ -4541,7 +4542,7 @@ mod sid_bucketing_tests { metric::Data, number_data_point::Value as NumberValue, Gauge as PbGauge, Metric as PbMetric, NumberDataPoint, ResourceMetrics, ScopeMetrics, }; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -4558,8 +4559,8 @@ mod sid_bucketing_tests { } } - fn sum_agg_config(metric: &str, grouping: &[&str]) -> AggregationConfig { - AggregationConfig::new( + fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 6c373e6a2..3ddd40607 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -717,11 +717,9 @@ fn route_messages( && matches!( config.aggregation_type, asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + | asap_types::AggregationType::Rate | asap_types::AggregationType::Min | asap_types::AggregationType::Max - | asap_types::AggregationType::MultipleMin - | asap_types::AggregationType::MultipleMax )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -1041,8 +1039,8 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; - use asap_types::{AggregationType, KeyByLabelNames, AggregationConfig}; - let aggregation = AggregationConfig { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; + let aggregation = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -1168,10 +1166,10 @@ mod tests { #[test] fn global_topk_cms_routes_once_while_counters_remain_per_series() { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; let config = |aggregation_type, grouping: Vec, aggregated: Vec| { - AggregationConfig { + PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 144e63012..2eba6811a 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1004,9 +1004,9 @@ fn metric_has_exact_agg_sum_sid( cap, Capability::ExactAgg( AggregationType::Sum - | AggregationType::MultipleSum + | AggregationType::Count | AggregationType::Increase - | AggregationType::MultipleIncrease + | AggregationType::Rate ) ) { return true; @@ -2849,100 +2849,18 @@ mod tests { /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and /// the GET snapshot emission. #[tokio::test] - async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; - let client = Client::new(); - - // Initial GET: empty config, 0 entries. - let initial = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .send() - .await - .expect("GET failed"); - assert!(initial.status().is_success()); - let initial_body: serde_json::Value = initial.json().await.unwrap(); - assert_eq!(initial_body["aggregation_count"], 0); - - // POST a new config with two aggregation_ids. The YAML shape - // matches what `StreamingConfig::from_yaml_data` parses — see - // `asap-common/dependencies/rs/asap_types/src/streaming_config.rs`. - let new_config_yaml = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 102 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host, region] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 120 - windowType: tumbling - spatialFilter: '' -"#; - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(new_config_yaml.to_string()) - .send() - .await - .expect("POST failed"); - let post_status = post_resp.status(); - let post_body: serde_json::Value = post_resp.json().await.unwrap(); - assert!( - post_status.is_success(), - "POST returned {post_status}: {post_body}" - ); - assert_eq!(post_body["status"], "success"); - assert_eq!(post_body["new_aggregation_count"], 2); - // PR 5: the YAML's `aggregationId` fields are silently - // dropped — `agg_ids_added` carries fingerprint u64s. - let added = post_body["agg_ids_added"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(added.len(), 2, "exactly two distinct aggs were added"); - assert!(added.iter().all(|id| *id != 0), "fingerprints are non-zero"); - - // GET again: should reflect the two new ids. - let after = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) + async fn flat_streaming_config_is_rejected_without_mutating_active_state() { + let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; + let before = handle.snapshot(); + let response = Client::new() + .post(format!("http://127.0.0.1:{port}/api/v1/streaming-config")) + .body("aggregations: [{aggregationType: Sum, metric: m}]") .send() .await - .expect("GET after swap failed"); - assert!(after.status().is_success()); - let after_body: serde_json::Value = after.json().await.unwrap(); - assert_eq!(after_body["aggregation_count"], 2); - - // The underlying StreamingConfigHandle handle (cloned into - // the server at setup) also reflects the swap — proving that - // downstream consumers that re-snapshot would see the new - // state. PR 5: the map is keyed on fingerprints, so just - // assert the entry count. - let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert!(Arc::ptr_eq(&before, &handle.snapshot())); } #[tokio::test] @@ -2985,7 +2903,7 @@ aggregations: .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), reqwest::StatusCode::GONE); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "error"); } @@ -3050,192 +2968,6 @@ aggregations: }); } - #[tokio::test] - async fn test_streaming_config_swap_drives_sid_reconcile() { - // Schema retirement final cut: the swap handler now drives a - // single sid-level reconcile (no `SchemaRegistry`). Sids that - // already exist in the catalog and whose content signature - // does not appear in the new config get force-retired; the - // response surfaces them under `sids_retired`. There is no - // `sids_added` — sids are minted lazily by the ingest path, - // not by the swap handler. - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::sketch_db::AggStatus; - - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let summary_store = Arc::new(SketchStore::new()); - // Pre-register two Active sids whose signatures match the - // first config below; only sid 1 will survive the second - // swap. - register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); - register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); - let server_port = setup_test_server_with_hot_reload_and_sketch_index( - hot_reload.clone(), - summary_store.clone(), - ) - .await; - let client = Client::new(); - - // POST a config whose signatures cover both pre-registered - // sids. Nothing should retire. - let yaml_two = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 202 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_two.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - let retired_ids = body["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert!( - retired_ids.is_empty(), - "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", - ); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Active - ); - - // Swap to a config that drops `mem_usage`. Sid 2's signature - // is now orphaned; the handler must force-retire it. - let yaml_one = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp2 = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .expect("POST failed"); - let body2: serde_json::Value = resp2.json().await.unwrap(); - let retired = body2["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(retired, vec![2u64]); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Retired - ); - } - - #[tokio::test] - async fn test_streaming_config_swap_response_shape_with_empty_catalog() { - // With no registered sids, the swap still works — it just - // produces an empty `sids_retired` array. The `agg_ids_added` - // / `agg_ids_removed` / `new_aggregation_count` fields are - // driven purely by the diff of the two configs and are - // independent of the sid catalog. - // - // PR 5: the YAML's `aggregationId: 42` is silently dropped at - // parse time — the backend identity is content-addressed via - // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 - // in the HTTP response is the fingerprint's `as_u64()` form, - // NOT the literal `42` the YAML once spelled out. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; - let client = Client::new(); - - let yaml = r#" -aggregations: - - aggregationType: Sum - aggregationSubType: '' - metric: m - labels: - grouping: [] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - assert_eq!(body["new_aggregation_count"], 1); - let added = body["agg_ids_added"].as_array().expect("array"); - assert_eq!(added.len(), 1, "exactly one agg was added"); - assert_ne!( - added[0].as_u64().unwrap(), - 0, - "agg id is not the 0 sentinel" - ); - assert_eq!(body["agg_ids_removed"], serde_json::json!([])); - // No pre-registered sids → nothing to retire. - assert_eq!(body["sids_retired"].as_array().unwrap().len(), 0); - } - #[tokio::test] async fn test_get_schemas_returns_active_and_retired_sids_with_status_filter() { // Schema retirement final cut: `/api/v1/db/schemas` now @@ -3255,29 +2987,12 @@ aggregations: .await; let client = Client::new(); - // Retire sid 2 by pushing a config covering only `m1`. - let yaml_one = r#" -aggregations: - - aggregationId: 1 - aggregationType: Sum - aggregationSubType: '' - metric: m1 - labels: { grouping: [], rollup: [], aggregated: [] } - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .unwrap(); - assert!(resp.status().is_success()); + assert!(summary_store + .force_retire( + 2, + crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION + ) + .is_some()); // GET /api/v1/db/schemas (no filter = all). let resp = client @@ -3514,7 +3229,7 @@ aggregations: registry: Arc, active_agg_ids: &[u64], ) -> (u16, std::collections::HashMap) { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -3536,7 +3251,7 @@ aggregations: let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -5621,96 +5336,11 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res (StatusCode::OK, axum::Json(body)).into_response() } -async fn handle_post_streaming_config( - State(state): State, - body: axum::body::Bytes, -) -> axum::response::Response { - use axum::http::StatusCode; +async fn handle_post_streaming_config() -> axum::response::Response { use axum::response::IntoResponse; - use std::collections::HashSet; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - - let yaml_text = match std::str::from_utf8(&body) { - Ok(s) => s, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("request body is not valid UTF-8: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let yaml_value: serde_yaml::Value = match serde_yaml::from_str(yaml_text) { - Ok(v) => v, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("YAML parse error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let new_config = - match crate::storage_engines::types::StreamingConfig::from_yaml_data(&yaml_value) { - Ok(c) => c, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("StreamingConfig build error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - - let new_ids: HashSet = new_config - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let added: Vec = new_ids.difference(&old_ids).copied().collect(); - let removed: Vec = old_ids.difference(&new_ids).copied().collect(); - - if !removed.is_empty() { - warn!( - "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ - precompute worker groups for these ids will continue with their \ - construction-time config until they close naturally (phase 1 \ - limitation; see StreamingConfigHandle module doc)", - removed - ); - } - - // Schema retirement final cut: the sid catalog is the only - // lifecycle registry. The legacy per-`agg_id` `SchemaRegistry` is - // gone, so the swap handler now drives a single sid-level - // reconcile (`reconcile_from_streaming_config`) which force-retires - // any sid whose content signature no longer appears in the new - // config. There is no "added" set: sids are minted lazily at the - // first ingest write under the new config (see - // `SketchStore::ingest_precompute_for_agg_config`). - let snap = handle.snapshot(); - let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.summary_store.as_ref(), - snap.as_ref(), - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - - let body = serde_json::json!({ - "status": "success", - "agg_ids_added": added, - "agg_ids_removed": removed, - "new_aggregation_count": new_ids.len(), - "sids_retired": sid_summary.retired}); - (StatusCode::OK, axum::Json(body)).into_response() + (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ + "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" + }))).into_response() } pub use asap_types::plan_publication::PhysicalPlanInstallRequest; @@ -5739,11 +5369,48 @@ pub fn validate_and_build_runtime_plan( .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; } + for entry in request.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let materialization = request + .precompute_plan + .materializations + .iter() + .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) + .ok_or_else(|| "query binding has no precompute definition".to_string())?; + if binding.window_ms != materialization.stored_window_ms() { + return Err( + "query physical pane duration differs from installed precompute definition" + .into(), + ); + } + if binding.pane_origin_ms != materialization.pane_origin_ms { + return Err( + "query physical pane origin differs from installed precompute definition" + .into(), + ); + } + // `full_window_slide_ms` is `#[serde(default)]`, so a publication from an + // older controller -- or one replayed from a stored artifact -- arrives as + // `None` on a FullWindow materialization. Without this gate the readout + // silently takes the overlap-merging path and counts observations twice, + // which is exactly what the full-window binding exists to prevent. + let full_window_slide_ms = matches!( + materialization.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) + .then_some(materialization.slide_interval.saturating_mul(1_000)); + if binding.full_window_slide_ms != full_window_slide_ms { + return Err( + "query window layout differs from installed precompute definition".into(), + ); + } + } + } asap_types::plan_publication::validate_stored_output_references( &request.precompute_plan, &request.query_plan, )?; - let runtime_materializations = request + let _runtime_materializations = request .precompute_plan .runtime_materializations() .map_err(|error| format!("PrecomputePlan validation error: {error}"))?; @@ -5758,8 +5425,10 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = - crate::storage_engines::types::StreamingConfig::new(runtime_materializations); + let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; let typed_fps: BTreeSet<_> = streaming_config .materializations_by_policy_fingerprint .keys() diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2721bc68c..8c3ac483c 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -37,13 +37,13 @@ pub mod utils; // Re-export commonly used types to avoid glob import conflicts pub use storage_engines::types::{ - AggregateCore, AggregationConfig, KeyByLabelValues, Measurement, MergeableAccumulator, - MultipleSubpopulationAggregate, PrecomputedOutput, SerializableToSink, - SingleSubpopulationAggregate, + AggregateCore, KeyByLabelValues, Measurement, MergeableAccumulator, + MultipleSubpopulationAggregate, PrecomputeMaterialization, PrecomputedOutput, + SerializableToSink, SingleSubpopulationAggregate, }; pub use precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; pub use storage_engines::StoreResult; diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 43a779fc6..9f94c8c93 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,30 +1,19 @@ use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, - HydraKllSketchAccumulator, IncreaseAccumulator, MaxAccumulator, MinAccumulator, - MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, - MultipleSumAccumulator, SumAccumulator, + HydraKllSketchAccumulator, IncreaseAccumulator, KeyedCounterState, KeyedMaxState, + KeyedMinState, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::{ AggregateCore, AggregationType, KeyByLabelValues, Measurement, }; -use asap_types::aggregation_config::AggregationConfig; -// Step 5 (sketch-identity unification, see -// scratchpad/artifacts/enum-unification-plan.md): dispatch below is -// driven by `AccumulatorSpec` (SummaryFamilyType + typed family parameters + -// keyed-axis grouping) instead of raw `AggregationType` + -// `aggregation_sub_type` string matching. Numeric params come straight -// off the committed family's typed params (no HashMap lookups) except -// `cms_params`, kept as a raw-`parameters` read for the one case Planner's -// family parameters have no field for: HydraKLL's `(row, col)` tiling grid (see -// `asap_types::accumulator_spec`'s module doc for why). `cms_params` -// now lives there — the only place that still needs the other three -// former local helpers (`kll_k_param`, `heap_size_param`, -// `ddsketch_alpha_param`) is that module's own `AccumulatorSpec` -// construction, so they aren't re-imported here. +use asap_types::aggregation_config::PrecomputeMaterialization; +// Production dispatch consumes Planner SummaryAgg payloads directly. The +// config adapter below is compiled only for isolated historical kernel tests. use super::operators::hll_sketch_accumulator::HllSketchAccumulator; use super::operators::univmon_accumulator::UnivMonAccumulator; -use asap_types::accumulator_spec::{cms_params, AccumulatorSpecError}; +#[cfg(test)] +use asap_types::accumulator_spec::cms_params; use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; /// Generate the two boilerplate clone-based `AccumulatorUpdater` methods @@ -379,28 +368,32 @@ impl AccumulatorUpdater for DDSketchAccumulatorUpdater { } // --------------------------------------------------------------------------- -// MultipleSumAccumulatorUpdater +// KeyedSumCountAccumulatorUpdater // --------------------------------------------------------------------------- -pub struct MultipleSumAccumulatorUpdater { - acc: MultipleSumAccumulator, +pub struct KeyedSumCountAccumulatorUpdater { + acc: KeyedSumCountAccumulator, } -impl MultipleSumAccumulatorUpdater { +impl KeyedSumCountAccumulatorUpdater { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { Self { - acc: MultipleSumAccumulator::new(), + acc: KeyedSumCountAccumulator::for_family(family), } } } -impl Default for MultipleSumAccumulatorUpdater { +impl Default for KeyedSumCountAccumulatorUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { +impl AccumulatorUpdater for KeyedSumCountAccumulatorUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -415,7 +408,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleSumAccumulator::new(); + self.acc = KeyedSumCountAccumulator::for_family(self.acc.family.clone()); } fn is_keyed(&self) -> bool { @@ -423,13 +416,13 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() - + self.acc.sums.len() * (std::mem::size_of::() + 8) + std::mem::size_of::() + + self.acc.sums.len() * (std::mem::size_of::() + 16) } } // --------------------------------------------------------------------------- -// MultipleMinAccumulatorUpdater / MultipleMaxAccumulatorUpdater +// KeyedMinStateUpdater / KeyedMaxStateUpdater // --------------------------------------------------------------------------- macro_rules! multiple_extremum_updater { @@ -475,32 +468,32 @@ macro_rules! multiple_extremum_updater { }; } -multiple_extremum_updater!(MultipleMinAccumulatorUpdater, MultipleMinAccumulator); -multiple_extremum_updater!(MultipleMaxAccumulatorUpdater, MultipleMaxAccumulator); +multiple_extremum_updater!(KeyedMinStateUpdater, KeyedMinState); +multiple_extremum_updater!(KeyedMaxStateUpdater, KeyedMaxState); // --------------------------------------------------------------------------- -// MultipleIncreaseAccumulatorUpdater +// KeyedCounterStateUpdater // --------------------------------------------------------------------------- -pub struct MultipleIncreaseAccumulatorUpdater { - acc: MultipleIncreaseAccumulator, +pub struct KeyedCounterStateUpdater { + acc: KeyedCounterState, } -impl MultipleIncreaseAccumulatorUpdater { +impl KeyedCounterStateUpdater { pub fn new() -> Self { Self { - acc: MultipleIncreaseAccumulator::new(), + acc: KeyedCounterState::new(), } } } -impl Default for MultipleIncreaseAccumulatorUpdater { +impl Default for KeyedCounterStateUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { +impl AccumulatorUpdater for KeyedCounterStateUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -528,7 +521,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleIncreaseAccumulator::new(); + self.acc = KeyedCounterState::new(); } fn is_keyed(&self) -> bool { @@ -536,7 +529,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() + std::mem::size_of::() + self.acc.increases.len() * (std::mem::size_of::() + std::mem::size_of::()) @@ -899,27 +892,20 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &AggregationConfig) -> bool { - matches!( - config.aggregation_type, - AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax - | AggregationType::CountMinSketch - | AggregationType::CountMinSketchWithHeap - | AggregationType::CountSketch - | AggregationType::CountSketchWithHeap - | AggregationType::HydraKLL - ) +pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { + config + .accumulator_spec() + .expect("valid fixture") + .grouping + .is_some() } /// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. /// /// * `value` / `sum`: sum values per key (default). /// * `count` / `frequency` / `freq`: count occurrences per key. -fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { +#[cfg(test)] +fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { match config.sample_update_rule() { asap_types::SampleUpdateRule::Count => TopkWeight::Count, asap_types::SampleUpdateRule::Value { .. } @@ -927,7 +913,8 @@ fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { } } -fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { +#[cfg(test)] +fn topk_weight_scale_param(config: &PrecomputeMaterialization) -> f64 { match config.sample_update_rule() { asap_types::SampleUpdateRule::Value { scale } => scale, asap_types::SampleUpdateRule::CounterDelta { scale } => scale, @@ -943,6 +930,7 @@ fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { /// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with /// `SketchParams::Kll`, so the /// other arm is unreachable from a `spec` this module builds itself. +#[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { // Lossless: `accumulator_spec()` only ever stores a value that @@ -955,12 +943,12 @@ fn kll_k(params: &SketchParams) -> u16 { } } -/// Read `(width, depth)` out of `SketchParams::Cms` or `::CountSketch` +/// Read `(rows = depth, columns = width)` out of `SketchParams::Cms` or `::CountSketch` /// — same shape, different variant per bare-sketch identity. fn cms_dims(params: &SketchParams) -> (usize, usize) { match params { SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { - (*width as usize, *depth as usize) + (*depth as usize, *width as usize) } other => unreachable!( "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" @@ -968,7 +956,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { } } -/// Read `(width, depth, heap_size)` out of `SketchParams::CmsWithHeap` +/// Read `(rows = depth, columns = width, heap_size)` out of `SketchParams::CmsWithHeap` /// or `::CountSketchWithHeap`. fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { match params { @@ -981,7 +969,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { width, depth, heap_size, - } => (*width as usize, *depth as usize, *heap_size as usize), + } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), @@ -989,6 +977,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { } /// Read the DDSketch relative-accuracy `alpha` out of `SketchParams::DDSketch`. +#[cfg(test)] fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, @@ -998,54 +987,28 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { } } -/// Create an appropriate `AccumulatorUpdater` from an `AggregationConfig`. -/// -/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryFamilyType` identity -/// plus the keyed/unkeyed `grouping` axis — instead of the pre-Step-5 -/// `AggregationType` + `aggregation_sub_type` string combo. See -/// `asap_types::accumulator_spec`'s module doc for why min/max direction, -/// HydraKLL's `(row, col)` tiling, and top-k `weight_mode` still read -/// `config` directly rather than going through Planner family parameters — -/// none of those three have a field in the Planner-owned types. -pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let spec = match config.accumulator_spec() { - Ok(spec) => spec, - // Three fallback paths, preserved verbatim from the pre-Step-5 - // dispatch: same warning text, same default updater per case - // (Single- and MultipleSubpopulation default to *different* - // updaters — see `AccumulatorSpecError`'s doc). - Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown SingleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(SumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown MultipleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(MultipleSumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnmappedAggregationType(other)) => { - tracing::warn!( - "Unknown aggregation_type '{:?}', defaulting to SingleSubpopulation Sum", - other - ); - return Box::new(SumAccumulatorUpdater::new()); - } - }; +/// Construct isolated payload fixtures for kernel/storage unit tests. +/// Production execution requires a validated Planner DAG program. +#[cfg(test)] +pub fn create_fixture_accumulator( + config: &PrecomputeMaterialization, +) -> Box { + let spec = config + .accumulator_spec() + .expect("invalid isolated kernel fixture"); let keyed = spec.grouping.is_some(); match (&spec.family, keyed) { - (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), false) => { Box::new(SumAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { - Box::new(MultipleSumAccumulatorUpdater::new()) + Box::new(KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Sum)) } + (SummaryFamilyType::ExactAggregate(ExactKind::Count, _), true) => Box::new( + KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Count), + ), // Direction comes off the family itself now. It used to be read // back out of `aggregation_sub_type` because Planner had one @@ -1056,20 +1019,20 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - Box::new(MultipleMinAccumulatorUpdater::new()) + Box::new(KeyedMinStateUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), false) => { Box::new(MaxAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), true) => { - Box::new(MultipleMaxAccumulatorUpdater::new()) + Box::new(KeyedMaxStateUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), false) => { Box::new(IncreaseAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { - Box::new(MultipleIncreaseAccumulatorUpdater::new()) + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), true) => { + Box::new(KeyedCounterStateUpdater::new()) } (SummaryFamilyType::Sketch(kind, _), false) @@ -1187,14 +1150,8 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - tracing::warn!( - "SummaryFamilyType {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", - other_family, - keyed - ); - Box::new(SumAccumulatorUpdater::new()) + panic!("unsupported isolated kernel fixture {other_family:?}, keyed={keyed}") } } } @@ -1271,7 +1228,7 @@ mod tests { #[test] fn hll_and_univmon_raw_updates_share_value_identity() { for family in [AggregationType::HLL, AggregationType::UnivMon] { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( family, String::new(), Default::default(), @@ -1288,7 +1245,7 @@ mod tests { None, None, ); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); for value in [0.0, -0.0, 2.0, 2.0, f64::NAN] { updater.update_single(value, 1000); } @@ -1362,7 +1319,7 @@ mod tests { #[test] fn test_multiple_sum_updater() { - let mut updater = MultipleSumAccumulatorUpdater::new(); + let mut updater = KeyedSumCountAccumulatorUpdater::new(); assert!(updater.is_keyed()); let key_a = KeyByLabelValues::new_with_labels(vec!["a".to_string()]); @@ -1372,7 +1329,7 @@ mod tests { updater.update_keyed(&key_b, 2.0, 2000); let acc = updater.take_accumulator(); - assert_eq!(acc.type_name(), "MultipleSumAccumulator"); + assert_eq!(acc.type_name(), "KeyedSumCountAccumulator"); } #[test] @@ -1424,7 +1381,7 @@ mod tests { use std::collections::HashMap; let make_config = |agg_type: AggregationType, sub_type: &str| { - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), HashMap::new(), @@ -1463,18 +1420,15 @@ mod tests { AggregationType::MultipleSubpopulation, "Sum" ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleSum, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleIncrease, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleMax, - "" - ))); + let mut keyed = make_config(AggregationType::Sum, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Increase, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Max, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); assert!(config_is_keyed(&make_config( AggregationType::CountMinSketch, "" @@ -1497,12 +1451,12 @@ mod tests { for (agg_type, sub_type) in &[ (AggregationType::SingleSubpopulation, "Sum"), (AggregationType::MultipleSubpopulation, "Sum"), - (AggregationType::MultipleSum, ""), + (AggregationType::Sum, ""), (AggregationType::DatasketchesKLL, ""), (AggregationType::CountMinSketch, ""), ] { let config = make_config(*agg_type, sub_type); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); assert_eq!( config_is_keyed(&config), updater.is_keyed(), @@ -1518,7 +1472,7 @@ mod tests { use std::collections::HashMap; let mut params = HashMap::new(); params.insert("K".to_string(), serde_json::Value::from(50_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "DatasketchesKLL".to_string(), params, @@ -1535,7 +1489,7 @@ mod tests { None, None, ); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); let acc = updater.snapshot_accumulator(); let kll = acc .as_any() @@ -1555,7 +1509,7 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), serde_json::Value::from(7_u64)); params.insert("w".to_string(), serde_json::Value::from(2048_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), params, @@ -1575,7 +1529,7 @@ mod tests { assert_eq!(super::cms_params(&config), (7, 2048)); // Empty params — defaults `(4, 1000)`. - let empty_config = AggregationConfig::new( + let empty_config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), HashMap::new(), @@ -1601,7 +1555,10 @@ mod tests { /// Build a `*WithHeap` config keyed by group-by label `host`, with the /// given `weight_mode` param (None → default = value-weighted). - fn topk_config(agg_type: AggregationType, weight_mode: Option<&str>) -> AggregationConfig { + fn topk_config( + agg_type: AggregationType, + weight_mode: Option<&str>, + ) -> PrecomputeMaterialization { use std::collections::HashMap; let mut params = HashMap::new(); // Small, deterministic geometry; heap big enough to hold all hosts. @@ -1611,7 +1568,7 @@ mod tests { if let Some(m) = weight_mode { params.insert("weight_mode".to_string(), serde_json::Value::from(m)); } - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -1699,7 +1656,7 @@ mod tests { fn value_weighted_topk_ranks_hosts_by_sum_of_value() { // DEFAULT mode (no weight_mode param) must be value-weighted. let config = topk_config(AggregationType::CountMinSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); assert!(updater.is_keyed()); feed_stream(&mut *updater); @@ -1725,14 +1682,24 @@ mod tests { #[test] fn counter_delta_scale_preserves_sub_unit_membership_weights() { - let mut config = topk_config( - AggregationType::CountMinSketchWithHeap, - Some("counter_delta"), - ); - config - .parameters - .insert("weight_scale".into(), serde_json::json!(1_000_000)); - let mut updater = create_accumulator_updater(&config); + use planner_types::post_asap::{ + EntityIdentity, NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, + }; + let config = topk_config(AggregationType::CountMinSketchWithHeap, None); + let family = config.accumulator_spec().unwrap().family; + let input = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::ResetAwareCounterDelta { + value: planner_types::pre_asap::ColumnRef::SampleValue, + series: EntityIdentity::PromqlLabelSet { excluding: vec![] }, + }, + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::ResetAwareCounterDerivative, + }, + }; + let mut updater = create_planner_accumulator(&family, &input, &Default::default()).unwrap(); updater.update_keyed(&host_key("payment"), 0.004, 1_000); updater.update_keyed(&host_key("order"), 0.002, 1_000); let ranked = ranked_topk(&*updater.take_accumulator()); @@ -1744,7 +1711,7 @@ mod tests { fn count_weighted_topk_still_ranks_by_occurrence_frequency() { // Opt-in frequency-top-k: weight_mode=count must rank by event count. let config = topk_config(AggregationType::CountMinSketchWithHeap, Some("count")); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); @@ -1764,7 +1731,7 @@ mod tests { // (real median-of-signed-rows math) — same value-weighted default // as the CMS-family heap path, but no longer conflated with it. let config = topk_config(AggregationType::CountSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); assert_eq!(acc.type_name(), "CountSketchWithHeapAccumulator"); @@ -1800,3 +1767,278 @@ mod tests { } } } + +#[cfg(test)] +mod planner_family_regression { + use super::*; + use asap_types::{enums::WindowKind, KeyByLabelNames}; + + // Every installed exact producer must retain its family in runtime state. + #[test] + fn exact_state_identity_survives_factory_and_reset() { + for kind in [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Rate, + AggregationType::Increase, + AggregationType::Min, + AggregationType::Max, + ] { + let config = PrecomputeMaterialization::new( + kind, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "metric".into(), + None, + None, + None, + ); + let mut updater = create_planner_accumulator( + &config.accumulator_spec().unwrap().family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &Default::default(), + ) + .unwrap(); + updater.update_single(4.0, 1000); + updater.update_single(7.0, 2000); + assert_eq!(updater.take_accumulator().get_accumulator_type(), kind); + assert_eq!(updater.snapshot_accumulator().get_accumulator_type(), kind); + } + } +} + +/// Construct the kernel declared by a Planner SummaryAgg. No backend config +/// tags participate in this dispatch and unsupported payloads are errors. +pub fn create_planner_accumulator( + family: &SummaryFamilyType, + input: &planner_types::post_asap::SummaryUpdate, + grouping: &planner_types::post_asap::GroupingStrategy, +) -> Result, String> { + use planner_types::post_asap::GroupingStrategy; + if grouping != &GroupingStrategy::PerSubpopulationInstance { + return Err("shared summary grouping requires a supported Planner Hydra kernel".into()); + } + if matches!(family, SummaryFamilyType::ExactAggregate(..)) { + return Ok(Box::new(PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator::new( + family.clone(), + input.item.is_some(), + )?, + })); + } + let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + return Err(format!("unsupported Planner summary family {family:?}")); + }; + if family_grouping != grouping { + return Err("Planner family and operator grouping disagree".into()); + } + // Heap counters use fixed-point storage for fractional counter deltas. + // This encodes the selected update; it does not choose another family. + let weight_scale = if matches!( + input.weight, + planner_types::post_asap::SummaryInputExpr::ResetAwareCounterDelta { .. } + ) { + 1_000_000.0 + } else { + 1.0 + }; + let updater: Box = match (kind.algorithm(), kind.params()) { + (SketchAlgorithm::Kll, SketchParams::Kll { k }) => Box::new(KllAccumulatorUpdater::new( + u16::try_from(*k).map_err(|_| "KLL k exceeds runtime bound")?, + )), + (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Box::new(DDSketchAccumulatorUpdater::new(*alpha)) + } + (SketchAlgorithm::Cms, params @ SketchParams::Cms { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CmsAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CountSketch, params @ SketchParams::CountSketch { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CountSketchAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CmsWithHeap, params @ SketchParams::CmsWithHeap { .. }) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CmsHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + ( + SketchAlgorithm::CountSketchWithHeap, + params @ SketchParams::CountSketchWithHeap { .. }, + ) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CountSketchWithHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => Box::new(HllUpdater { + acc: HllSketchAccumulator::new( + asap_sketchlib::HllVariant::Regular, + u32::from(*precision), + ), + }), + ( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + }, + ) => Box::new(UnivMonUpdater { + acc: UnivMonAccumulator::new( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + *layers as usize, + ) + .map_err(|e| e.to_string())?, + }), + _ => { + return Err(format!( + "unsupported Planner algorithm/parameters: {kind:?}" + )) + } + }; + if updater.is_keyed() != input.item.is_some() + && !asap_types::accumulator_spec::is_unit_sample_frequency(input) + { + return Err("Planner item expression does not match the selected kernel layout".into()); + } + Ok(updater) +} + +struct PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator, +} +impl AccumulatorUpdater for PlannerExactUpdater { + fn update_single(&mut self, value: f64, timestamp: i64) { + self.acc.update(None, value, timestamp); + } + fn update_keyed(&mut self, key: &KeyByLabelValues, value: f64, timestamp: i64) { + self.acc.update(Some(key), value, timestamp); + } + impl_clone_accumulator_methods!(acc); + fn reset(&mut self) { + self.acc = super::operators::exact_accumulator::ExactAccumulator::new( + self.acc.family().clone(), + self.acc.is_keyed(), + ) + .expect("installed exact family"); + } + fn is_keyed(&self) -> bool { + self.acc.is_keyed() + } + fn memory_usage_bytes(&self) -> usize { + self.acc.approx_memory_bytes() + } +} + +#[cfg(test)] +mod planner_parameter_regression { + use super::*; + use planner_types::post_asap::{SketchKind, SummaryInputExpr, SummaryUpdate}; + + // Planner width is the bucket count; depth is the independent hash-row count. + #[test] + fn planner_sketch_dimensions_are_not_transposed() { + for (algorithm, params) in [ + ( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ] { + let family = SummaryFamilyType::Sketch( + SketchKind::new(algorithm.clone(), params), + Default::default(), + ); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + }; + let state = create_planner_accumulator(&family, &update, &Default::default()) + .unwrap() + .snapshot_accumulator(); + let dims = match algorithm { + SketchAlgorithm::Cms => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketch => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows, s.inner.cols) + } + SketchAlgorithm::CmsWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketchWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + _ => unreachable!(), + }; + assert_eq!(dims, (3, 128), "{algorithm:?}"); + } + } +} diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 38c43b07d..e97045fe5 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -56,7 +56,7 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::AggregationConfig, + config: &asap_types::PrecomputeMaterialization, timestamp_ms: i64, value: f64, ) { @@ -264,8 +264,8 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { - let config = asap_types::AggregationConfig::new( + fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { + let config = asap_types::PrecomputeMaterialization::new( asap_types::AggregationType::HLL, String::new(), Default::default(), diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index d975eb6a9..67940478b 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,7 +1,7 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::StreamingConfigHandle; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,11 +28,11 @@ pub struct IngestObservability { /// A full / delta frame failed to decode (or a delta failed to /// apply) and was dropped. pub dropped_decode_fail: AtomicU64, - /// A decoded sketch matched no `AggregationConfig` in the running + /// A decoded sketch matched no `PrecomputeMaterialization` in the running /// streaming config (legacy routing-side bucketing miss). pub dropped_unconfigured: AtomicU64, /// The output sink could not resolve a `policy_fp` to an - /// `AggregationConfig` (registry miss) and skipped the write. + /// `PrecomputeMaterialization` (registry miss) and skipped the write. pub dropped_policy_miss: AtomicU64, /// RES-1 — max number of distinct tumbling windows a per-series /// snapshot base may lag behind the newest observed `window_start` @@ -120,7 +120,7 @@ pub struct IngestState { pub samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64, /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. - /// This replaces the old frozen `Vec>`. + /// This replaces the old frozen `Vec>`. pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, @@ -157,7 +157,7 @@ impl IngestState { /// visible immediately without restart. /// /// Returns the shared `Arc` — no cloning of - /// individual AggregationConfig objects, just an atomic refcount + /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() @@ -247,7 +247,7 @@ impl IngestState { /// ingest sources (e.g. OTLP) can reuse it. pub fn extract_group_key_for( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { extract_group_key(series_key, config) } @@ -261,7 +261,7 @@ impl IngestState { /// [`Self::extract_group_key_for`] does after the round-trip. pub fn extract_group_key_from_labels( labels: &std::collections::HashMap, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map( |name| { @@ -278,7 +278,7 @@ impl IngestState { /// for a given series key and aggregation config. fn extract_group_key( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { let labels = parse_labels_from_series_key(series_key); crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map(|name| { @@ -294,18 +294,18 @@ mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Kept as a parameter to // avoid churning the call sites below. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), std::collections::HashMap::new(), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 08de6788e..d0a7a1394 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -122,7 +122,7 @@ fn frozen_population_value( struct OperatorAdapter<'a> { binding: &'a BackendExecutableBinding, inputs: MaintenanceInputs<'a>, - configs: &'a [asap_types::aggregation_config::AggregationConfig], + configs: &'a [asap_types::aggregation_config::PrecomputeMaterialization], } impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { @@ -195,7 +195,12 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } finalize_exact(node, inputs) } - ExecutableOperatorPayload::SummaryAgg { family, input, .. } => { + ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } => { let [value] = inputs else { return Err("maintenance SummaryAgg requires exactly one row input".into()); }; @@ -251,7 +256,9 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = super::accumulator_factory::create_accumulator_updater(config); + let mut updater = super::accumulator_factory::create_planner_accumulator( + family, input, grouping, + )?; if updater.is_keyed() { return Err("keyed maintenance accumulator requires an item expression".into()); } diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 068ac13b7..3f744870a 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod metrics; pub mod multisource_coordinator; pub mod operators; pub mod output_sink; +pub mod raw_dag; pub mod series_buffer; pub mod series_router; pub mod subdag_scheduler; diff --git a/data_plane/src/precompute_engine/operators/exact_accumulator.rs b/data_plane/src/precompute_engine/operators/exact_accumulator.rs new file mode 100644 index 000000000..b7fcead12 --- /dev/null +++ b/data_plane/src/precompute_engine/operators/exact_accumulator.rs @@ -0,0 +1,327 @@ +//! Exact summary state identified by Planner family, independent of keyed layout. +use super::increase_accumulator::IncreaseAccumulator; +use crate::storage_engines::types::{ + AggregateCore, AggregationType, AuxStats, KeyByLabelValues, Measurement, SerializableToSink, +}; +use asap_types::Statistic; +use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +type Error = Box; + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum ScalarState { + Sum(f64), + Count(u64), + Min(Option), + Max(Option), + Counter(Option), +} + +/// Both the family and population layout survive persistence. Sharing counter +/// arithmetic never authorizes a Rate state to answer an Increase readout. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExactAccumulator { + family: SummaryFamilyType, + scalar: ScalarState, + keyed: Option>, +} + +impl ExactAccumulator { + pub fn new(family: SummaryFamilyType, keyed: bool) -> Result { + use ExactKind as K; + use ExactParams as P; + let scalar = match &family { + SummaryFamilyType::ExactAggregate(K::Sum, P::Sum) => ScalarState::Sum(0.0), + SummaryFamilyType::ExactAggregate(K::Count, P::Count) => ScalarState::Count(0), + SummaryFamilyType::ExactAggregate(K::Min, P::Min) => ScalarState::Min(None), + SummaryFamilyType::ExactAggregate(K::Max, P::Max) => ScalarState::Max(None), + SummaryFamilyType::ExactAggregate(K::Rate, P::Rate) + | SummaryFamilyType::ExactAggregate(K::Increase, P::Increase) => { + ScalarState::Counter(None) + } + _ => return Err(format!("unsupported exact Planner family: {family:?}")), + }; + Ok(Self { + family, + scalar, + keyed: keyed.then(HashMap::new), + }) + } + + pub fn family(&self) -> &SummaryFamilyType { + &self.family + } + pub fn is_keyed(&self) -> bool { + self.keyed.is_some() + } + + pub fn update(&mut self, key: Option<&KeyByLabelValues>, value: f64, timestamp: i64) { + let state = match (&mut self.keyed, key) { + (Some(states), Some(key)) => states + .entry(key.clone()) + .or_insert_with(|| self.scalar.clone()), + (None, None) => &mut self.scalar, + _ => panic!("exact update population layout differs from installed DAG"), + }; + match state { + ScalarState::Sum(sum) => *sum += value, + ScalarState::Count(count) => { + *count = count.checked_add(1).expect("exact count overflow") + } + ScalarState::Min(current) => { + *current = Some(current.map_or(value, |old| old.min(value))) + } + ScalarState::Max(current) => { + *current = Some(current.map_or(value, |old| old.max(value))) + } + ScalarState::Counter(current) => match current { + Some(counter) => counter.update(Measurement::new(value), timestamp), + None => { + *current = Some(IncreaseAccumulator::new( + Measurement::new(value), + timestamp, + Measurement::new(value), + timestamp, + )) + } + }, + } + } + + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let state: Self = rmp_serde::from_slice(bytes)?; + let expected = Self::new(state.family.clone(), state.is_keyed())?; + let same_variant = |value: &ScalarState| { + std::mem::discriminant(value) == std::mem::discriminant(&expected.scalar) + }; + if !same_variant(&state.scalar) + || state + .keyed + .as_ref() + .is_some_and(|states| states.values().any(|s| !same_variant(s))) + { + return Err("exact payload differs from declared Planner family".into()); + } + Ok(state) + } + + fn statistic(&self) -> Statistic { + match self.family { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Min, _) => Statistic::Min, + SummaryFamilyType::ExactAggregate(ExactKind::Max, _) => Statistic::Max, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => Statistic::Rate, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => Statistic::Increase, + _ => unreachable!("validated exact family"), + } + } +} + +fn merge_scalar(left: &ScalarState, right: &ScalarState) -> Result { + Ok(match (left, right) { + (ScalarState::Sum(a), ScalarState::Sum(b)) => ScalarState::Sum(a + b), + (ScalarState::Count(a), ScalarState::Count(b)) => { + ScalarState::Count(a.checked_add(*b).ok_or("exact count overflow")?) + } + (ScalarState::Min(a), ScalarState::Min(b)) => { + ScalarState::Min(a.iter().chain(b).copied().reduce(f64::min)) + } + (ScalarState::Max(a), ScalarState::Max(b)) => { + ScalarState::Max(a.iter().chain(b).copied().reduce(f64::max)) + } + (ScalarState::Counter(a), ScalarState::Counter(b)) => { + ScalarState::Counter(match (a, b) { + (Some(a), Some(b)) => Some( + >::merge_accumulators(vec![a.clone(), b.clone()])?, + ), + (a, b) => a.clone().or_else(|| b.clone()), + }) + } + _ => return Err("exact scalar state families differ".into()), + }) +} + +impl SerializableToSink for ExactAccumulator { + fn serialize_to_json(&self) -> serde_json::Value { + serde_json::json!({"family": self.family, "scalar": self.scalar, "keyed": self.keyed.as_ref().map(|m|m.iter().collect::>())}) + } + fn serialize_to_bytes(&self) -> Vec { + rmp_serde::to_vec_named(self).expect("exact state encoding") + } +} + +impl AggregateCore for ExactAccumulator { + fn clone_boxed_core(&self) -> Box { + Box::new(self.clone()) + } + fn type_name(&self) -> &'static str { + "PlannerExactAccumulatorV1" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn merge_with(&self, other: &dyn AggregateCore) -> Result, Error> { + let other = other + .as_any() + .downcast_ref::() + .ok_or("merge requires Planner exact state")?; + if self.family != other.family || self.is_keyed() != other.is_keyed() { + return Err("cannot merge different Planner families or layouts".into()); + } + let mut merged = self.clone(); + if let (Some(target), Some(source)) = (&mut merged.keyed, &other.keyed) { + for (key, state) in source { + let combined = match target.get(key) { + Some(old) => merge_scalar(old, state)?, + None => state.clone(), + }; + target.insert(key.clone(), combined); + } + } else { + merged.scalar = merge_scalar(&self.scalar, &other.scalar)?; + } + Ok(Box::new(merged)) + } + fn get_accumulator_type(&self) -> AggregationType { + match self.statistic() { + Statistic::Sum => AggregationType::Sum, + Statistic::Count => AggregationType::Count, + Statistic::Min => AggregationType::Min, + Statistic::Max => AggregationType::Max, + Statistic::Rate => AggregationType::Rate, + Statistic::Increase => AggregationType::Increase, + _ => unreachable!(), + } + } + fn approx_memory_bytes(&self) -> usize { + std::mem::size_of::() + + self.keyed.as_ref().map_or(0, |m| { + m.keys() + .map(|k| { + std::mem::size_of::() + + k.labels.iter().map(String::len).sum::() + }) + .sum::() + }) + } + fn aux_stats(&self) -> AuxStats { + if self.is_keyed() { + return AuxStats::empty(); + } + match self.scalar { + ScalarState::Sum(value) => AuxStats { + sum: Some(value), + ..AuxStats::empty() + }, + ScalarState::Count(value) => AuxStats { + count: Some(value), + ..AuxStats::empty() + }, + ScalarState::Min(value) => AuxStats { + min: value, + ..AuxStats::empty() + }, + ScalarState::Max(value) => AuxStats { + max: value, + ..AuxStats::empty() + }, + ScalarState::Counter(_) => AuxStats::empty(), + } + } + fn get_keys(&self) -> Option> { + self.keyed.as_ref().map(|m| m.keys().cloned().collect()) + } + fn query_statistic( + &self, + statistic: Statistic, + key: &Option, + kwargs: &HashMap, + ) -> Result { + if statistic != self.statistic() { + return Err("readout differs from Planner exact family".into()); + } + let state = match (&self.keyed, key) { + (Some(states), Some(key)) => states.get(key).ok_or("unknown exact population")?, + (None, None) => &self.scalar, + _ => return Err("readout population differs from installed layout".into()), + }; + match state { + ScalarState::Sum(sum) => Ok(*sum), + ScalarState::Count(count) => Ok(*count as f64), + ScalarState::Min(value) | ScalarState::Max(value) => { + value.ok_or_else(|| "empty exact population".into()) + } + ScalarState::Counter(Some(counter)) => { + counter.query_statistic(statistic, &None, kwargs) + } + ScalarState::Counter(None) => Err("empty counter population".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Identity, population isolation, and readout survive the persisted format. + #[test] + fn exact_families_roundtrip_and_reject_cross_family_operations() { + let families = [ + (ExactKind::Sum, ExactParams::Sum, Statistic::Sum, 16.0), + (ExactKind::Count, ExactParams::Count, Statistic::Count, 3.0), + (ExactKind::Min, ExactParams::Min, Statistic::Min, 2.0), + (ExactKind::Max, ExactParams::Max, Statistic::Max, 8.0), + (ExactKind::Rate, ExactParams::Rate, Statistic::Rate, 3.0), + ( + ExactKind::Increase, + ExactParams::Increase, + Statistic::Increase, + 6.0, + ), + ]; + for keyed in [false, true] { + let key = keyed.then(|| KeyByLabelValues::new_with_labels(vec!["a".into()])); + let mut states = Vec::new(); + for (kind, params, stat, value) in &families { + let mut state = ExactAccumulator::new( + SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), + keyed, + ) + .unwrap(); + for (ts, v) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(key.as_ref(), v, ts); + } + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()).unwrap(); + assert_eq!(restored.family(), state.family()); + assert_eq!( + restored + .query_statistic(*stat, &key, &HashMap::new()) + .unwrap(), + *value + ); + for (_, _, wrong, _) in &families { + if wrong != stat { + assert!(restored + .query_statistic(*wrong, &key, &HashMap::new()) + .is_err()); + } + } + states.push(restored); + } + for (i, a) in states.iter().enumerate() { + for (j, b) in states.iter().enumerate() { + assert_eq!(a.merge_with(b).is_ok(), i == j); + } + } + } + } +} diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs similarity index 86% rename from data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_counter_state.rs index c1d59aa1a..b94d2faba 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs @@ -12,11 +12,11 @@ use asap_types::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys /// Allows tracking rate/increase for different label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleIncreaseAccumulator { +pub struct KeyedCounterState { pub increases: HashMap, } -impl MultipleIncreaseAccumulator { +impl KeyedCounterState { pub fn new() -> Self { Self { increases: HashMap::new(), @@ -91,13 +91,13 @@ impl MultipleIncreaseAccumulator { } } -impl Default for MultipleIncreaseAccumulator { +impl Default for KeyedCounterState { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleIncreaseAccumulator { +impl SerializableToSink for KeyedCounterState { fn serialize_to_json(&self) -> Value { let entries: Vec = self .increases @@ -135,13 +135,13 @@ impl SerializableToSink for MultipleIncreaseAccumulator { } } -impl AggregateCore for MultipleIncreaseAccumulator { +impl AggregateCore for KeyedCounterState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleIncreaseAccumulator" + "KeyedCounterState" } fn as_any(&self) -> &dyn std::any::Any { @@ -156,20 +156,20 @@ impl AggregateCore for MultipleIncreaseAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleIncreaseAccumulator + // Check if other is also a KeyedCounterState if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleIncreaseAccumulator with {}", + "Cannot merge KeyedCounterState with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleIncreaseAccumulator + // Downcast to KeyedCounterState let other_multiple_increase = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleIncreaseAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedCounterState")?; // Clone self once, then merge each matching counter with the same // reset-aware, boundary-aware implementation used by the unkeyed path. @@ -189,7 +189,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleIncrease + AggregationType::Increase } fn approx_memory_bytes(&self) -> usize { @@ -210,14 +210,12 @@ impl AggregateCore for MultipleIncreaseAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleIncreaseAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedCounterState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { +impl MultipleSubpopulationAggregate for KeyedCounterState { fn query( &self, statistic: Statistic, @@ -227,7 +225,7 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { let data = self .increases .get(key) - .ok_or_else(|| format!("Key {key} not found in MultipleIncreaseAccumulator"))?; + .ok_or_else(|| format!("Key {key} not found in KeyedCounterState"))?; data.query(statistic, query_kwargs) } @@ -237,15 +235,15 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { } } -impl MergeableAccumulator for MultipleIncreaseAccumulator { +impl MergeableAccumulator for KeyedCounterState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleIncreaseAccumulator::new(); + let mut result = KeyedCounterState::new(); for accumulator in accumulators { for (key, data) in accumulator.increases { @@ -291,14 +289,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_creation() { - let acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_creation() { + let acc = KeyedCounterState::new(); assert!(acc.increases.is_empty()); } #[test] - fn test_multiple_increase_accumulator_update() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_update() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -316,8 +314,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_query() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_query() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -344,14 +342,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_sum_per_key() { - // `sum by (zone) (counter)` reaches MultipleIncreaseAccumulator + fn test_keyed_counter_state_sum_per_key() { + // `sum by (zone) (counter)` reaches KeyedCounterState // only when the ASAP-tier ingest groups multiple series under // a single accumulator (the `Multiple*` variant). In that case // each per-key Sum should be the series' latest cumulative // value; the engine's outer `by` aggregation does the cross-key // grouping. (Issue ProjectASAP/ASAPCollector#46.) - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let east = KeyByLabelValues::new_with_labels(vec!["us-east-1".to_string()]); let west = KeyByLabelValues::new_with_labels(vec!["us-west-2".to_string()]); @@ -369,9 +367,9 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_merge() { - let mut acc1 = MultipleIncreaseAccumulator::new(); - let mut acc2 = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_merge() { + let mut acc1 = KeyedCounterState::new(); + let mut acc2 = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -387,7 +385,7 @@ mod tests { create_test_increase_accumulator_with_time(15.0, 2000, 30.0, 3000), ); // Later time range - let merged = MultipleIncreaseAccumulator::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = KeyedCounterState::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.increases.len(), 2); assert!(merged.increases.contains_key(&key1)); @@ -400,8 +398,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_serialization() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_serialization() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let second_key = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -415,7 +413,7 @@ mod tests { // Test JSON serialization let json_value = acc.serialize_to_json(); - let deserialized = MultipleIncreaseAccumulator::deserialize_from_json(&json_value).unwrap(); + let deserialized = KeyedCounterState::deserialize_from_json(&json_value).unwrap(); assert_eq!(deserialized.increases.len(), 2); let deserialized_acc = deserialized.increases.get(&key).unwrap(); @@ -425,8 +423,7 @@ mod tests { // Test binary serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = - MultipleIncreaseAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedCounterState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.increases.len(), 2); let deserialized_acc_bytes = deserialized_bytes.increases.get(&key).unwrap(); @@ -445,8 +442,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_get_keys() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_get_keys() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let key2 = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -462,7 +459,7 @@ mod tests { #[test] fn test_trait_object() { - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new(); acc.update(key.clone(), create_test_increase_accumulator(10.0, 25.0)); @@ -477,7 +474,7 @@ mod tests { } // #[test] - // fn test_multiple_increase_accumulator_arroyo_deserialization() { + // fn test_keyed_counter_state_arroyo_deserialization() { // // Create test data in Arroyo MessagePack format // // Format: {key: [starting_value, starting_timestamp, last_seen_value, last_seen_timestamp]} // let mut test_data = std::collections::HashMap::new(); @@ -489,7 +486,7 @@ mod tests { // // Test Arroyo deserialization // let deserialized_acc = - // MultipleIncreaseAccumulator::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); + // KeyedCounterState::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); // // Verify the deserialized accumulator has the correct data // assert_eq!(deserialized_acc.increases.len(), 2); diff --git a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_max_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_max_state.rs index 1865d2686..4309c04a0 100644 --- a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_max_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key maximum over many populations, mergeable by comparison. /// /// The minimum direction is -/// [`MultipleMinAccumulator`](super::multiple_min_accumulator::MultipleMinAccumulator), +/// [`KeyedMinState`](super::keyed_min_state::KeyedMinState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMaxAccumulator { +pub struct KeyedMaxState { pub values: HashMap, } -impl MultipleMaxAccumulator { +impl KeyedMaxState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMaxAccumulator { } } -impl SerializableToSink for MultipleMaxAccumulator { +impl SerializableToSink for KeyedMaxState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMaxAccumulator { } } -impl AggregateCore for MultipleMaxAccumulator { +impl AggregateCore for KeyedMaxState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMaxAccumulator" + "KeyedMaxState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMaxAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMaxAccumulator with {}", + "Cannot merge KeyedMaxState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMaxAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMaxAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMaxState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMaxAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMax + AggregationType::Max } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMaxAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMaxAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMaxState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { +impl MultipleSubpopulationAggregate for KeyedMaxState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMaxAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMaxAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMaxState").into()), + other => Err(format!("Unsupported statistic in KeyedMaxState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { } } -impl MergeableAccumulator for MultipleMaxAccumulator { +impl MergeableAccumulator for KeyedMaxState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMaxAccumulator::new(); + let mut result = KeyedMaxState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_largest_per_key() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Min, &key("a"), None).is_err()); assert!(acc.query(Statistic::Max, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMaxAccumulator::new(); + let mut left = KeyedMaxState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMaxAccumulator::new(); + let mut right = KeyedMaxState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Max, &key("a"), None).unwrap(), 10.0); assert_eq!(merged.query(Statistic::Max, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_min_accumulator::MultipleMinAccumulator; - let mine = MultipleMaxAccumulator::new(); - let theirs = MultipleMinAccumulator::new(); + use super::super::keyed_min_state::KeyedMinState; + let mine = KeyedMaxState::new(); + let theirs = KeyedMinState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMaxAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMaxState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMaxAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMaxState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_min_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_min_state.rs index 00c250402..5be698f50 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_min_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key minimum over many populations, mergeable by comparison. /// /// The maximum direction is -/// [`MultipleMaxAccumulator`](super::multiple_max_accumulator::MultipleMaxAccumulator), +/// [`KeyedMaxState`](super::keyed_max_state::KeyedMaxState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMinAccumulator { +pub struct KeyedMinState { pub values: HashMap, } -impl MultipleMinAccumulator { +impl KeyedMinState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMinAccumulator { } } -impl SerializableToSink for MultipleMinAccumulator { +impl SerializableToSink for KeyedMinState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMinAccumulator { } } -impl AggregateCore for MultipleMinAccumulator { +impl AggregateCore for KeyedMinState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMinAccumulator" + "KeyedMinState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMinAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMinAccumulator with {}", + "Cannot merge KeyedMinState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMinAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMinAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMinState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMinAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMin + AggregationType::Min } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMinAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMinAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMinState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMinAccumulator { +impl MultipleSubpopulationAggregate for KeyedMinState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMinAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMinAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMinState").into()), + other => Err(format!("Unsupported statistic in KeyedMinState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { } } -impl MergeableAccumulator for MultipleMinAccumulator { +impl MergeableAccumulator for KeyedMinState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMinAccumulator::new(); + let mut result = KeyedMinState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_smallest_per_key() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Max, &key("a"), None).is_err()); assert!(acc.query(Statistic::Min, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMinAccumulator::new(); + let mut left = KeyedMinState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMinAccumulator::new(); + let mut right = KeyedMinState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Min, &key("a"), None).unwrap(), 5.0); assert_eq!(merged.query(Statistic::Min, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_max_accumulator::MultipleMaxAccumulator; - let mine = MultipleMinAccumulator::new(); - let theirs = MultipleMaxAccumulator::new(); + use super::super::keyed_max_state::KeyedMaxState; + let mine = KeyedMinState::new(); + let theirs = KeyedMaxState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMinAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMinState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMinAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMinState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs similarity index 50% rename from data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs index 85a9982d8..c39d55831 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs @@ -7,26 +7,53 @@ use serde_json::Value; use std::collections::HashMap; use asap_types::Statistic; +use planner_types::post_asap::ExactKind; + +fn sum_family() -> ExactKind { + ExactKind::Sum +} /// Accumulator that maintains separate sum values for multiple keys /// Allows querying sums for specific label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleSumAccumulator { +pub struct KeyedSumCountAccumulator { + #[serde(default = "sum_family")] + pub family: ExactKind, pub sums: HashMap, + #[serde(default)] + pub counts: HashMap, } -impl MultipleSumAccumulator { +impl KeyedSumCountAccumulator { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { + assert!(matches!(family, ExactKind::Sum | ExactKind::Count)); Self { + family, sums: HashMap::new(), + counts: HashMap::new(), } } pub fn update(&mut self, key: KeyByLabelValues, value: f64) { - *self.sums.entry(key).or_insert(0.0) += value; + let is_new = !self.sums.contains_key(&key); + *self.sums.entry(key.clone()).or_insert(0.0) += value; + if let Some(count) = self.counts.get(&key).copied() { + if let Some(next) = count.checked_add(1).filter(|next| *next != u64::MAX) { + self.counts.insert(key, next); + } else { + self.counts.remove(&key); + } + } else if is_new { + self.counts.insert(key, 1); + } } pub fn add_sum(&mut self, key: KeyByLabelValues, sum: f64) { + self.counts.remove(&key); self.sums.insert(key, sum); } @@ -43,7 +70,28 @@ impl MultipleSumAccumulator { sums.insert(key, sum); } - Ok(Self { sums }) + let mut counts = HashMap::new(); + if let Some(counts_data) = data.get("counts").and_then(Value::as_object) { + for (key_str, value) in counts_data { + let key_json: Value = serde_json::from_str(key_str)?; + let key = KeyByLabelValues::deserialize_from_json(&key_json)?; + let count = value.as_u64().ok_or("Invalid count value")?; + if !sums.contains_key(&key) { + return Err("Count key missing from sums".into()); + } + counts.insert(key, count); + } + } + let family = match data.get("family").and_then(Value::as_str) { + None | Some("Sum") => ExactKind::Sum, + Some("Count") => ExactKind::Count, + _ => return Err("Invalid keyed additive family".into()), + }; + Ok(Self { + family, + sums, + counts, + }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -62,6 +110,7 @@ impl MultipleSumAccumulator { offset += 4; let mut sums = HashMap::new(); + let mut keys = Vec::new(); for _ in 0..num_entries { // Read key length and data @@ -99,20 +148,50 @@ impl MultipleSumAccumulator { ]); offset += 8; + keys.push(key.clone()); sums.insert(key, sum); } - - Ok(Self { sums }) + let remaining = buffer.len() - offset; + let count_bytes = num_entries + .checked_mul(8) + .ok_or("Count section too large")?; + if remaining != 0 && remaining != count_bytes && remaining != count_bytes + 1 { + return Err("Invalid count section length".into()); + } + let mut counts = HashMap::new(); + if count_bytes != 0 && remaining >= count_bytes { + for key in keys { + let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); + offset += 8; + if count != u64::MAX { + counts.insert(key, count); + } + } + } + let family = if remaining == count_bytes + 1 { + match buffer[offset] { + 0 => ExactKind::Sum, + 1 => ExactKind::Count, + _ => return Err("Invalid keyed additive family tag".into()), + } + } else { + ExactKind::Sum + }; + Ok(Self { + family, + sums, + counts, + }) } } -impl Default for MultipleSumAccumulator { +impl Default for KeyedSumCountAccumulator { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleSumAccumulator { +impl SerializableToSink for KeyedSumCountAccumulator { fn serialize_to_json(&self) -> Value { let mut sums_obj = serde_json::Map::new(); for (key, sum) in &self.sums { @@ -124,8 +203,16 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + let mut counts_obj = serde_json::Map::new(); + for (key, count) in &self.counts { + let key_str = serde_json::to_string(&key.serialize_to_json()).unwrap(); + counts_obj.insert(key_str, Value::from(*count)); + } + serde_json::json!({ - "sums": sums_obj + "family": if self.family == ExactKind::Count { "Count" } else { "Sum" }, + "sums": sums_obj, + "counts": counts_obj }) } @@ -136,7 +223,9 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&(self.sums.len() as u32).to_le_bytes()); // Write each key-value pair + let mut ordered_keys = Vec::with_capacity(self.sums.len()); for (key, sum) in &self.sums { + ordered_keys.push(key); let key_bytes = key.serialize_to_bytes(); // Write key length and data @@ -147,17 +236,34 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&sum.to_le_bytes()); } + for key in ordered_keys { + buffer.extend_from_slice( + &self + .counts + .get(key) + .copied() + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + } + + buffer.push(if self.family == ExactKind::Count { + 1 + } else { + 0 + }); + buffer } } -impl AggregateCore for MultipleSumAccumulator { +impl AggregateCore for KeyedSumCountAccumulator { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleSumAccumulator" + "KeyedSumCountAccumulator" } fn as_any(&self) -> &dyn std::any::Any { @@ -172,20 +278,20 @@ impl AggregateCore for MultipleSumAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleSumAccumulator + // Check if other is also a KeyedSumCountAccumulator if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleSumAccumulator with {}", + "Cannot merge KeyedSumCountAccumulator with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleSumAccumulator + // Downcast to KeyedSumCountAccumulator let other_multiple_sum = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleSumAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedSumCountAccumulator")?; // Use the existing merge_accumulators method let merged = Self::merge_accumulators(vec![self.clone(), other_multiple_sum.clone()])?; @@ -194,13 +300,17 @@ impl AggregateCore for MultipleSumAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleSum + if self.family == ExactKind::Count { + AggregationType::Count + } else { + AggregationType::Sum + } } fn approx_memory_bytes(&self) -> usize { // HashMap. Label strings dominate; use a // conservative per-entry estimate plus HashMap overhead. - const BYTES_PER_ENTRY: usize = 96; + const BYTES_PER_ENTRY: usize = 112; std::mem::size_of::() + self.sums.len() * BYTES_PER_ENTRY } @@ -217,26 +327,35 @@ impl AggregateCore for MultipleSumAccumulator { use crate::storage_engines::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() - .ok_or("Key required for MultipleSumAccumulator")?; + .ok_or("Key required for KeyedSumCountAccumulator")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleSumAccumulator { +impl MultipleSubpopulationAggregate for KeyedSumCountAccumulator { fn query( &self, statistic: Statistic, key: &KeyByLabelValues, _query_kwargs: Option<&HashMap>, ) -> Result> { - match statistic { - Statistic::Sum | Statistic::Count => self - .sums + match (&self.family, statistic) { + (ExactKind::Sum, Statistic::Sum) => self.sums.get(key).copied().ok_or_else(|| { + "Key not found in KeyedSumCountAccumulator" + .to_string() + .into() + }), + (ExactKind::Count, Statistic::Count) => self + .counts .get(key) - .copied() - .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), + .map(|count| *count as f64) + .ok_or_else(|| { + "Sample count unavailable in KeyedSumCountAccumulator" + .to_string() + .into() + }), _ => Err( - format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), + format!("Unsupported statistic in KeyedSumCountAccumulator: {statistic:?}").into(), ), } } @@ -246,17 +365,41 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { } } -impl MergeableAccumulator for MultipleSumAccumulator { +impl MergeableAccumulator for KeyedSumCountAccumulator { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleSumAccumulator::new(); + let family = accumulators[0].family.clone(); + if accumulators.iter().any(|acc| acc.family != family) { + return Err("Cannot merge different keyed additive families".into()); + } + let mut result = KeyedSumCountAccumulator::for_family(family); for acc in accumulators { + for key in acc.sums.keys() { + match ( + result.counts.get(key).copied(), + acc.counts.get(key).copied(), + ) { + (None, Some(count)) if !result.sums.contains_key(key) => { + result.counts.insert(key.clone(), count); + } + (Some(existing), Some(count)) => { + if let Some(total) = existing.checked_add(count) { + result.counts.insert(key.clone(), total); + } else { + result.counts.remove(key); + } + } + _ => { + result.counts.remove(key); + } + } + } for (key, sum) in acc.sums { *result.sums.entry(key).or_insert(0.0) += sum; } @@ -273,14 +416,14 @@ mod tests { use super::*; #[test] - fn test_multiple_sum_accumulator_creation() { - let acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_creation() { + let acc = KeyedSumCountAccumulator::new(); assert!(acc.sums.is_empty()); } #[test] - fn test_multiple_sum_accumulator_update() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_update() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -295,8 +438,37 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_query() { - let mut acc = MultipleSumAccumulator::new(); + fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut first = KeyedSumCountAccumulator::for_family(ExactKind::Count); + first.update(key.clone(), 10.0); + first.update(key.clone(), 20.0); + let mut second = KeyedSumCountAccumulator::for_family(ExactKind::Count); + second.update(key.clone(), 7.0); + let merged = KeyedSumCountAccumulator::merge_accumulators(vec![first, second]).unwrap(); + for acc in [ + merged.clone(), + KeyedSumCountAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + ] { + assert_eq!(acc.family, ExactKind::Count); + assert!(acc.query(Statistic::Sum, &key, None).is_err()); + assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); + } + } + + #[test] + fn keyed_additive_merge_rejects_different_planner_families() { + assert!(KeyedSumCountAccumulator::merge_accumulators(vec![ + KeyedSumCountAccumulator::for_family(ExactKind::Sum), + KeyedSumCountAccumulator::for_family(ExactKind::Count), + ]) + .is_err()); + } + + #[test] + fn test_keyed_sum_count_accumulator_query() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -315,8 +487,8 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_get_keys() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_get_keys() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -332,9 +504,9 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_merge() { - let mut acc1 = MultipleSumAccumulator::new(); - let mut acc2 = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_merge() { + let mut acc1 = KeyedSumCountAccumulator::new(); + let mut acc2 = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -345,15 +517,15 @@ mod tests { acc2.add_sum(key1.clone(), 5.0); // Same key, different accumulator - let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.sums.get(&key1), Some(&15.0)); // Should be merged assert_eq!(merged.sums.get(&key2), Some(&20.0)); // Should be preserved } #[test] - fn test_multiple_sum_accumulator_serialization() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_serialization() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -361,18 +533,18 @@ mod tests { // Test JSON serialization let json = acc.serialize_to_json(); - let deserialized = MultipleSumAccumulator::deserialize_from_json(&json).unwrap(); + let deserialized = KeyedSumCountAccumulator::deserialize_from_json(&json).unwrap(); assert_eq!(deserialized.sums.get(&key), Some(&42.5)); // Test byte serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = MultipleSumAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedSumCountAccumulator::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.sums.get(&key), Some(&42.5)); } #[test] fn test_trait_object() { - let mut acc = MultipleSumAccumulator::new(); + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -381,6 +553,6 @@ mod tests { let trait_obj: Box = Box::new(acc); // Test type name through trait object - assert_eq!(trait_obj.type_name(), "MultipleSumAccumulator"); + assert_eq!(trait_obj.type_name(), "KeyedSumCountAccumulator"); } } diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index 9df95ba19..073db6e82 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -4,15 +4,16 @@ pub mod count_sketch_accumulator; pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; +pub mod exact_accumulator; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; +pub mod keyed_counter_state; +pub mod keyed_max_state; +pub mod keyed_min_state; +pub mod keyed_sum_count_accumulator; pub mod max_accumulator; pub mod min_accumulator; -pub mod multiple_increase_accumulator; -pub mod multiple_max_accumulator; -pub mod multiple_min_accumulator; -pub mod multiple_sum_accumulator; pub mod sketch_envelope_accumulator; pub mod sum_accumulator; pub mod univmon_accumulator; @@ -26,11 +27,11 @@ pub use dd_sketch_accumulator::*; pub use hll_sketch_accumulator::*; pub use hydra_kll_accumulator::*; pub use increase_accumulator::*; +pub use keyed_counter_state::*; +pub use keyed_max_state::*; +pub use keyed_min_state::*; +pub use keyed_sum_count_accumulator::*; pub use max_accumulator::*; pub use min_accumulator::*; -pub use multiple_increase_accumulator::*; -pub use multiple_max_accumulator::*; -pub use multiple_min_accumulator::*; -pub use multiple_sum_accumulator::*; pub use sketch_envelope_accumulator::*; pub use sum_accumulator::*; diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 2cbe66fe5..d5ff3b02c 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -197,7 +197,11 @@ impl SingleSubpopulationAggregate for SumAccumulator { } match statistic { - Statistic::Sum | Statistic::Count => Ok(self.sum), + Statistic::Sum => Ok(self.sum), + Statistic::Count => self + .observation_count + .map(|count| count as f64) + .ok_or_else(|| "sample count is unavailable for this Sum payload".into()), _ => Err(format!("Unsupported statistic in SumAccumulator: {statistic:?}").into()), } } @@ -308,10 +312,7 @@ mod tests { crate::SingleSubpopulationAggregate::query(&acc, Statistic::Sum, None).unwrap(), 42.0 ); - assert_eq!( - crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), - 42.0 - ); + assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).is_err()); assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Min, None).is_err()); // SumAccumulator is a single subpopulation accumulator, doesn't need key-based queries @@ -321,6 +322,17 @@ mod tests { ); } + #[test] + fn count_readout_uses_observation_count_not_sum() { + let mut acc = SumAccumulator::new(); + acc.update(10.0); + acc.update(20.0); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), + 2.0 + ); + } + #[test] fn test_sum_accumulator_merge() { let acc1 = SumAccumulator::with_sum(10.0); diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 820bb7a21..940df03e0 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -112,7 +112,7 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `AggregationConfig` via + /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via /// `PolicyRegistry::get(output.policy_fp)`. The legacy /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` /// is the only identity handle on `PrecomputedOutput`. Outputs @@ -338,7 +338,7 @@ mod tests { use crate::precompute_engine::operators::{DDSketchAccumulator, SumAccumulator}; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -384,11 +384,11 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> AggregationConfig { + fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers obtain the id // via `config.policy_fp_u64()`. - AggregationConfig { + PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs new file mode 100644 index 000000000..ee76c8b49 --- /dev/null +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -0,0 +1,295 @@ +//! Bind raw ingestion to a selected Planner producer and its raw dependency edge. +use super::accumulator_factory::{create_planner_accumulator, AccumulatorUpdater}; +use crate::storage_engines::types::KeyByLabelValues; +use asap_types::{executable_plan::BackendNodeBinding, PrecomputeMaterialization}; +use planner_types::post_asap::{ + EdgeRole, ExecutableOperatorPayload, GroupingStrategy, PostAsapNodeId, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; +use planner_types::pre_asap::{ColumnRef, QueryExpr, Source}; +use std::collections::HashMap; + +/// A validated executable projection; semantics come from the installed node. +/// The retained node ID makes failures attributable to the selected DAG. +#[derive(Debug, Clone)] +pub struct RawDagProgram { + pub node: PostAsapNodeId, + pub family: SummaryFamilyType, + pub input: SummaryUpdate, + pub grouping: GroupingStrategy, + pub reduction: planner_types::pre_asap::Reduction, + projected_column: Option, +} + +impl RawDagProgram { + pub fn from_plan( + plan: &asap_types::precompute_plan::PrecomputePlan, + config: &PrecomputeMaterialization, + ) -> Result { + let mut selected: Option = None; + for installed in plan.executable_dags.values() { + installed.validate()?; + let dag = installed.document.decode()?; + for node in &dag.nodes { + if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { summary_definition }) if summary_definition.fingerprint() == config.policy_fingerprint()) + { + continue; + } + let ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + reduction, + } = &node.payload + else { + return Err( + "raw materialization binding must identify a Planner SummaryAgg".into(), + ); + }; + if config.derived_input.is_some() { + return Err("derived producer must execute through maintenance DAG".into()); + } + let incoming: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node.id).collect(); + let [edge] = incoming.as_slice() else { + return Err("raw SummaryAgg must have exactly one DAG input".into()); + }; + if edge.role != EdgeRole::Input { + return Err("raw SummaryAgg input edge has wrong role".into()); + } + let source = dag + .nodes + .iter() + .find(|n| n.id == edge.producer) + .ok_or("missing raw DAG input")?; + let ExecutableOperatorPayload::Fallback { expression } = &source.payload else { + return Err("raw producer requires an executable source input; maintenance edges cannot be bypassed".into()); + }; + let scan = match expression { + QueryExpr::TimeRange { child, .. } => child.as_ref(), + source => source, + }; + match scan { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + .. + } if metric == &config.metric => { + let (metric, window, filter) = + control_plane::physical::compiler::raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + )?; + if metric != config.metric + || window.is_some_and(|seconds| seconds != config.window_size) + || asap_types::utils::normalize_spatial_filter(&filter) + != config.spatial_filter_normalized + { + return Err( + "raw DAG source filter/window differs from physical binding".into(), + ); + } + } + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if config.table_name.as_ref() == Some(table_ref) => { + return Err( + "raw table execution requires a validated table scan executor".into(), + ); + } + _ => return Err("raw DAG input does not match installed source routing".into()), + } + if let planner_types::pre_asap::Reduction::Reduce(keys) = reduction { + if keys.is_without() { + return Err( + "raw without reduction requires explicit dynamic population routing" + .into(), + ); + } + let names = keys + .keys() + .iter() + .map(|id| { + source + .output_schema + .fields + .get(*id) + .map(|f| f.name.clone()) + .ok_or("missing reduction column") + }) + .collect::, _>>()?; + if names != config.grouping_labels.names() { + return Err("DAG reduction differs from physical population binding".into()); + } + } + if let SummaryFamilyType::ExactAggregate(kind, _) = family { + if input.item.is_some() { + return Err( + "raw exact populations must follow Planner reduction, not an item map" + .into(), + ); + } + if !matches!(input.weight, SummaryInputExpr::Column(_)) + && !(matches!(kind, planner_types::post_asap::ExactKind::Count) + && input.weight == SummaryInputExpr::Constant(1.0)) + { + return Err("raw exact update differs from stored source projection".into()); + } + } + if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { + return Err( + "materialization storage family differs from selected Planner node".into(), + ); + } + // The stored descriptor must name the same update semantics; its + // content identity cannot be reused for an unrelated DAG program. + let update_matches = match (&input.weight, config.sample_update_rule()) { + ( + SummaryInputExpr::Column(_), + asap_types::SampleUpdateRule::Value { scale }, + ) => scale == 1.0, + (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { + *value == 1.0 + } + ( + SummaryInputExpr::ResetAwareCounterDelta { .. }, + asap_types::SampleUpdateRule::CounterDelta { scale }, + ) => scale == 1_000_000.0, + _ => { + asap_types::accumulator_spec::is_unit_sample_frequency(input) + || (matches!( + family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ) + ) && input.weight == SummaryInputExpr::Constant(1.0)) + } + }; + if !update_matches { + return Err("DAG update differs from stored summary identity".into()); + } + let program = Self { + node: node.id, + family: family.clone(), + input: input.clone(), + grouping: grouping.clone(), + reduction: reduction.clone(), + projected_column: config + .effective_value_projection() + .column() + .map(str::to_owned), + }; + program.validate()?; + if let Some(old) = &selected { + if old.family != program.family + || old.input != program.input + || old.grouping != program.grouping + || old.reduction != program.reduction + { + return Err( + "one stored definition is bound to incompatible Planner producers" + .into(), + ); + } + } else { + selected = Some(program); + } + } + } + selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) + } + + pub fn updater(&self) -> Result, String> { + create_planner_accumulator(&self.family, &self.input, &self.grouping) + } + + fn validate(&self) -> Result<(), String> { + match &self.input.weight { + SummaryInputExpr::Column(ColumnRef::SampleValue) | SummaryInputExpr::Constant(_) => {} + SummaryInputExpr::Column( + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }, + ) if self.projected_column.as_ref() == Some(name) => {} + SummaryInputExpr::ResetAwareCounterDelta { + value: ColumnRef::SampleValue, + series: planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + } if excluding.is_empty() => {} + _ => return Err("raw DAG weight expression is unsupported".into()), + } + fn item(expr: &SummaryInputExpr) -> bool { + match expr { + SummaryInputExpr::Column(ColumnRef::Named(_) | ColumnRef::SampleValue) => true, + SummaryInputExpr::Tuple(items) => items.iter().all(item), + SummaryInputExpr::EntityIdentity( + planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + ) => excluding.is_empty(), + _ => false, + } + } + if self.input.item.as_ref().is_some_and(|e| !item(e)) { + return Err("raw DAG item expression is unsupported".into()); + } + self.updater().map(|_| ()) + } + + pub fn uses_counter_delta(&self) -> bool { + matches!( + self.input.weight, + SummaryInputExpr::ResetAwareCounterDelta { .. } + ) + } + + pub fn apply( + &self, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + ) -> Result<(), String> { + let weight = match &self.input.weight { + SummaryInputExpr::Constant(c) => *c, + // The worker retains one previous value per series across pane rotation. + SummaryInputExpr::Column(_) | SummaryInputExpr::ResetAwareCounterDelta { .. } => value, + _ => return Err("unsupported raw weight expression".into()), + }; + let scalar_frequency = asap_types::accumulator_spec::is_unit_sample_frequency(&self.input) + && !updater.is_keyed(); + let weight = if scalar_frequency { value } else { weight }; + updater.validate_single_input(weight)?; + if updater.is_keyed() { + let labels = super::worker::parse_labels_from_series_key(series); + fn eval( + expr: &SummaryInputExpr, + series: &str, + value: f64, + labels: &HashMap<&str, &str>, + ) -> Result, String> { + Ok(match expr { + SummaryInputExpr::EntityIdentity(_) => vec![series.to_owned()], + SummaryInputExpr::Column(ColumnRef::SampleValue) => vec![value.to_string()], + SummaryInputExpr::Column(ColumnRef::Named(name)) => vec![labels + .get(name.as_str()) + .map(|s| super::worker::decode_label_value(s).into_owned()) + .ok_or_else(|| format!("missing DAG item column {name}"))?], + SummaryInputExpr::Tuple(items) => items + .iter() + .map(|i| eval(i, series, value, labels)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + _ => return Err("unsupported raw item expression".into()), + }) + } + let item = self + .input + .item + .as_ref() + .ok_or("keyed DAG kernel requires an explicit item")?; + let key = KeyByLabelValues::new_with_labels(eval(item, series, value, &labels)?); + updater.update_keyed(&key, weight, timestamp); + } else { + updater.update_single(weight, timestamp); + } + Ok(()) + } +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 751f47e5f..01af62314 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -20,7 +20,7 @@ use xxhash_rust::xxh64::xxh64; /// hashing or pane lookup. `group_key` and `policy_fp` still travel /// alongside the sid: `group_key` is consumed at emit-time to render the /// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `AggregationConfig` from the hot-reload snapshot (window +/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window /// shape, late-data policy, etc.). Together they let the worker key state /// by sid without losing the data the legacy `(agg_id, group_key)` shape /// carried. @@ -50,8 +50,8 @@ pub enum WorkerMessage { /// `(metric, attrs_fingerprint, agg_kind_canonical)` — see /// `SeriesIdResolver::resolve`. Worker keys `group_states` on this. sid: u64, - /// Source `AggregationConfig` fingerprint. Worker looks up its - /// `AggregationConfig` (window size, sketch kind/config, late + /// Source `PrecomputeMaterialization` fingerprint. Worker looks up its + /// `PrecomputeMaterialization` (window size, sketch kind/config, late /// data policy, etc.) via `snap.get_aggregation_config(policy_fp.as_u64())`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons (e.g. "constant"). @@ -78,7 +78,7 @@ pub enum WorkerMessage { AccumulatorInput { /// Registry-allocated bucket identity; see `GroupSamples::sid`. sid: u64, - /// Source `AggregationConfig` fingerprint; see + /// Source `PrecomputeMaterialization` fingerprint; see /// `GroupSamples::policy_fp`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons, matching the diff --git a/data_plane/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs index afccd0169..e1214fd9b 100644 --- a/data_plane/src/precompute_engine/window_manager.rs +++ b/data_plane/src/precompute_engine/window_manager.rs @@ -18,7 +18,7 @@ pub struct WindowManager { impl WindowManager { /// Create a new WindowManager. /// - /// `window_size_secs` and `slide_interval_secs` come from `AggregationConfig` + /// `window_size_secs` and `slide_interval_secs` come from `PrecomputeMaterialization` /// (which stores them in seconds). They are converted to milliseconds internally. pub fn new(window_size_secs: u64, slide_interval_secs: u64) -> Self { Self::with_origin(window_size_secs, slide_interval_secs, None) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 438dcdb2c..fe4e74e05 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,6 +1,6 @@ -use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, -}; +#[cfg(test)] +use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; +use crate::precompute_engine::accumulator_factory::AccumulatorUpdater; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::group_key::GroupKey; use crate::precompute_engine::metrics::record_late_input; @@ -11,7 +11,7 @@ use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; @@ -37,10 +37,11 @@ use tracing::{debug, debug_span, info, warn}; /// producing one output per (sid, window) — exactly like Arroyo's /// `GROUP BY window, key`. struct GroupState { + program: Option>, series_id: u64, catalog_generation: Option>, input_revisions: BTreeMap>, - config: Arc, + config: Arc, /// Source policy fingerprint that minted this sid. Held so /// `evict_orphaned_groups` can check liveness against the streaming /// config snapshot (a sid stays alive only while its source policy is @@ -411,7 +412,7 @@ impl Worker { /// /// B7.6 — buckets are now keyed by `sid` (a single u64) rather than /// `(agg_id, group_key)`. `policy_fp` is the source config's - /// fingerprint, used to fetch the `AggregationConfig` from the + /// fingerprint, used to fetch the `PrecomputeMaterialization` from the /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// @@ -425,12 +426,16 @@ impl Worker { sid: u64, policy_fp: PolicyFingerprint, group_key: &Arc, - ) -> Option<&mut GroupState> { + ) -> Result, String> { if !self.group_states.contains_key(&sid) { let snap = self.hot_reload.snapshot(); - let cfg = snap.get_aggregation_config(policy_fp.as_u64())?; + let Some(cfg) = snap.get_aggregation_config(policy_fp.as_u64()) else { + return Ok(None); + }; + let program = snap.raw_programs.get(&policy_fp.as_u64()).cloned(); let config = Arc::new(cfg.clone()); let gs = GroupState { + program, series_id: sid, catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), @@ -454,7 +459,7 @@ impl Worker { self.group_count .store(self.group_states.len(), Ordering::Relaxed); } - self.group_states.get_mut(&sid) + Ok(self.group_states.get_mut(&sid)) } /// Process a batch of samples for a specific sid bucket. @@ -462,7 +467,7 @@ impl Worker { /// /// This is the core of the Arroyo-equivalent GROUP BY logic. /// B7.6 — buckets are keyed by `sid`; `policy_fp` is the source - /// `AggregationConfig` fingerprint used to resolve the bucket's + /// `PrecomputeMaterialization` fingerprint used to resolve the bucket's /// config on first sight; `group_key` is held on the resulting /// `GroupState` for emit-time label rendering. pub fn process_group_samples( @@ -479,7 +484,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -489,6 +494,10 @@ impl Worker { return Ok(()); } let state = self.group_states.get_mut(&sid).unwrap(); + #[cfg(not(test))] + if state.program.is_none() { + return Err("raw precompute requires an installed post-ASAP DAG producer".into()); + } // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. @@ -537,12 +546,19 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = - if let SampleUpdateRule::CounterDelta { .. } = state.config.sample_update_rule() { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), + ) { + reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) + } else { + Some(*val) + }; for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -589,9 +605,14 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } + if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), ) { if let Some(input) = state.input_revisions.get_mut(&bucket_start) { Arc::make_mut(input).first_revision = 0; @@ -600,8 +621,16 @@ impl Worker { continue; } record_late_input("append_correction", "raw_sample"); - let mut updater = create_accumulator_updater(&state.config); - apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + let mut updater = + installed_updater(state.program.as_deref(), &state.config)?; + apply_installed_sample( + state.program.as_deref(), + &mut *updater, + series_key, + *val, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { @@ -646,12 +675,21 @@ impl Worker { // only closes an idle pane, not a long-running bulk ingest whose // records share one event timestamp. state.touch_pane(bucket_start, now_ms); - let updater = state - .active_panes - .entry(bucket_start) - .or_insert_with(|| create_accumulator_updater(&state.config)); + if let std::collections::btree_map::Entry::Vacant(entry) = + state.active_panes.entry(bucket_start) + { + entry.insert(installed_updater(state.program.as_deref(), &state.config)?); + } + let updater = state.active_panes.get_mut(&bucket_start).unwrap(); if let Some(value) = value { - apply_sample(&mut **updater, series_key, value, *ts, &state.config); + apply_installed_sample( + state.program.as_deref(), + &mut **updater, + series_key, + value, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { observer.observe( @@ -767,7 +805,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -1349,7 +1387,10 @@ pub fn extract_metric_name(series_key: &str) -> &str { /// aggregation config's `grouping_labels`. /// /// The series key format is: `metric_name{label1="val1",label2="val2",...}` -pub fn extract_key_from_series(series_key: &str, config: &AggregationConfig) -> KeyByLabelValues { +pub fn extract_key_from_series( + series_key: &str, + config: &PrecomputeMaterialization, +) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1492,19 +1533,61 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { std::borrow::Cow::Owned(out) } +fn installed_updater( + program: Option<&super::raw_dag::RawDagProgram>, + config: &PrecomputeMaterialization, +) -> Result, String> { + if let Some(program) = program { + return program.updater(); + } + #[cfg(test)] + { + Ok(create_fixture_accumulator(config)) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + +fn apply_installed_sample( + program: Option<&super::raw_dag::RawDagProgram>, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + config: &PrecomputeMaterialization, +) -> Result<(), String> { + if let Some(program) = program { + return program.apply(updater, series, value, timestamp); + } + #[cfg(test)] + { + apply_sample(updater, series, value, timestamp, config); + Ok(()) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + /// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. /// /// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted /// from the series' **aggregated_labels** — these are the labels that become /// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a MultipleSumAccumulator's HashMap). This matches the Arroyo SQL +/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL /// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +#[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) { if updater.is_keyed() { // Planner's PromQL Top-K item is the series identity. When no @@ -1529,7 +1612,7 @@ pub(crate) fn apply_sample( /// Convert a cumulative counter sample into a non-negative, reset-aware /// increment. Only the immediately preceding sample per series is retained; /// pane rotation therefore cannot lose the boundary increment. -fn reset_aware_counter_delta( +pub(crate) fn reset_aware_counter_delta( previous: &mut HashMap, series_key: &str, value: f64, @@ -1560,7 +1643,7 @@ fn reset_aware_counter_delta( /// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. fn extract_aggregated_key_from_series( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1792,7 +1875,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::storage_engines::types::StreamingConfig; @@ -1808,7 +1891,7 @@ mod tests { window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { make_agg_config_full( id, metric, @@ -1831,7 +1914,7 @@ mod tests { slide_secs: u64, grouping: Vec<&str>, aggregated: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers below build the // streaming-config map by reading `config.policy_fp_u64()` @@ -1841,7 +1924,7 @@ mod tests { } else { WindowKind::Sliding }; - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, agg_sub_type.to_string(), HashMap::new(), @@ -1861,7 +1944,7 @@ mod tests { } fn make_worker( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1871,7 +1954,7 @@ mod tests { } fn make_worker_with_lateness( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1900,12 +1983,12 @@ mod tests { } /// Build a fresh `StreamingConfigHandle` from a map of agg_id - /// → AggregationConfig. Worker::new takes this handle instead of - /// the old `HashMap>`. Tests use this + /// → PrecomputeMaterialization. Worker::new takes this handle instead of + /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( - configs: HashMap, + configs: HashMap, ) -> crate::storage_engines::types::StreamingConfigHandle { crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), @@ -1991,7 +2074,7 @@ mod tests { assert_eq!(output.start_timestamp as i64, *ts); assert_eq!(output.end_timestamp as i64, *ts); // Raw mode emits PolicyFingerprint::UNSET (no source - // AggregationConfig in the raw-mode fast path). The sink + // PrecomputeMaterialization in the raw-mode fast path). The sink // drops UNSET outputs with a warn — verified separately // via integration tests. assert!(output.policy_fp.is_unset()); @@ -2452,7 +2535,7 @@ mod tests { #[test] fn test_keyed_accumulator_aggregated_labels() { // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside MultipleSumAccumulator) + // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) let config = make_agg_config_full( 3, "cpu", @@ -2504,10 +2587,10 @@ mod tests { let (_output, acc) = &captured[0]; let ms_acc = acc .as_any() - .downcast_ref::() - .expect("should be MultipleSumAccumulator"); + .downcast_ref::() + .expect("should be KeyedSumCountAccumulator"); - // The MultipleSumAccumulator should have two internal keys: "A" and "B" + // The KeyedSumCountAccumulator should have two internal keys: "A" and "B" assert_eq!(ms_acc.sums.len(), 2, "two host keys inside one accumulator"); let mut found_a = false; @@ -2680,100 +2763,15 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn test_worker_from_streaming_config_yaml() { - let yaml = r#" -aggregations: -- aggregationType: SingleSubpopulation - aggregationSubType: Sum - labels: - grouping: [] - rollup: [] - aggregated: [] - metric: requests_total - parameters: {} - tumblingWindowSize: 10 - windowSize: 10 - windowType: tumbling - slideInterval: 0 - spatialFilter: '' -"#; - - let data: serde_yaml::Value = serde_yaml::from_str(yaml).expect("valid YAML"); - let streaming_config = - StreamingConfig::from_yaml_data(&data).expect("valid streaming config"); - - // PR 5: the streaming-config key is the policy fingerprint. - let agg_id = *streaming_config - .materializations() - .keys() - .next() - .expect("one agg"); - assert!(streaming_config.contains(agg_id)); - - let agg_configs = streaming_config.materializations().clone(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - let pf = PolicyFingerprint(agg_id); - let sid = 1_u64; - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(1_000, 3.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(5_000, 4.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(9_000, 5.0)]), - ) - .unwrap(); - assert_eq!(sink.len(), 0); - - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(10_000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - - let (output, acc) = &captured[0]; - let _ = agg_id; - assert!(!output.policy_fp.is_unset()); - assert_eq!(output.start_timestamp, 0); - assert_eq!(output.end_timestamp, 10_000); - - let sum_acc = acc - .as_any() - .downcast_ref::() - .expect("should be SumAccumulator"); - assert!( - (sum_acc.sum - 12.0).abs() < 1e-10, - "sum should be 3+4+5=12, got {}", - sum_acc.sum - ); + fn test_worker_rejects_flat_streaming_config_yaml() { + let data = + serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); + assert!(StreamingConfig::from_yaml_data(&data).is_err()); } #[test] fn test_extract_key_from_series() { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), @@ -3413,7 +3411,7 @@ aggregations: /// Build a worker with explicit wall-clock closure grace values. fn make_worker_with_wall_clock_policy( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, late_data_policy: LateDataPolicy, idle_grace_period_ms: i64, @@ -4278,3 +4276,179 @@ aggregations: ); } } + +#[cfg(test)] +mod dag_execution_tests { + use super::*; + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::precompute_engine::output_sink::CapturingOutputSink; + use crate::storage_engines::types::StreamingConfig; + use asap_types::query_plan::ExactReadout; + + fn plan(query: &str) -> control_plane::physical::compiler::CompiledPhysicalPlan { + let mut json: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut item = json["query_workload"]["repeating_queries"][0].clone(); + item["query"] = query.into(); + json["query_workload"]["repeating_queries"] = serde_json::json!([item]); + let snapshot = serde_json::from_value(json).unwrap(); + crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap() + } + + // A selected producer must govern updates, persisted family, and query readout. + #[test] + fn installed_dag_ingestion_persistence_and_readout() { + for (query, readout, answer) in [ + ( + "sum_over_time(asap_demo_gauge[5s])", + ExactReadout::Sum, + 54.0, + ), + ( + "count_over_time(asap_demo_gauge[5s])", + ExactReadout::Count, + 5.0, + ), + ("min_over_time(asap_demo_gauge[5s])", ExactReadout::Min, 3.0), + ( + "max_over_time(asap_demo_gauge[5s])", + ExactReadout::Max, + 20.0, + ), + ("rate(asap_demo_counter_total[5s])", ExactReadout::Rate, 5.5), + ( + "increase(asap_demo_counter_total[5s])", + ExactReadout::Increase, + 27.5, + ), + ] { + let plan = plan(query); + let config = plan + .precompute_plan + .materializations + .first() + .expect("ASAP producer required") + .clone(); + let fp = config.policy_fingerprint(); + let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); + let doc = serde_json::to_value(&streaming).unwrap(); + assert!(doc.get("aggregation_configs").is_none()); + let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let sink = Arc::new(CapturingOutputSink::new()); + let (_tx, rx) = mpsc::channel(8); + let mut worker = Worker::new( + 0, + rx, + sink.clone(), + StreamingConfigHandle::new(streaming), + WorkerRuntimeConfig { + max_buffer_per_series: 100, + allowed_lateness_ms: 10_000, + pass_raw_samples: false, + raw_mode_aggregation_id: 0, + late_data_policy: LateDataPolicy::Drop, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + }, + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicI64::new(0)), + ); + worker + .process_group_samples( + 1, + fp, + &Arc::new(GroupKey::new([])), + [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, v)| (config.metric.clone(), t, v)) + .collect(), + ) + .unwrap(); + worker.force_close_all().unwrap(); + let mut states = BTreeMap::new(); + for (output, state) in sink.drain() { + let select = if matches!( + config.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) { + output.start_timestamp == 0 && output.end_timestamp == 5000 + } else { + output.end_timestamp <= 5000 + }; + if select { + assert_eq!( + state.get_accumulator_type().planner_exact_family(), + Some(readout.planner_family()) + ); + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()) + .unwrap(); + states.insert( + output.end_timestamp as i64, + Arc::new(restored) as Arc, + ); + } + } + assert!(!states.is_empty(), "{query}: no stored states"); + let group = + crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { + entries: vec![std::rc::Rc::new(states)], + agg_type: config.aggregation_type, + }; + assert_eq!( + group.exact_value_for(readout, &None, 0, 5000), + Some(answer), + "{query}" + ); + } + } + + // A flat config and a DAG whose producer no longer matches its binding cannot install. + #[test] + fn execution_requires_matching_dag_producer() { + assert!(serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err()); + let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; + plan.executable_dags.clear(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("DAG producer")); + } + // Changing a raw update must not silently reuse the original summary identity. + #[test] + fn altered_dag_update_cannot_reuse_a_stored_definition() { + let mut plan = plan("sum_over_time(asap_demo_gauge[5s])").precompute_plan; + let installed = plan.executable_dags.values_mut().next().unwrap(); + let mut dag = installed.document.decode().unwrap(); + for node in &mut dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + input, .. + } = &mut node.payload + { + input.weight = planner_types::post_asap::SummaryInputExpr::Constant(99.0); + } + } + installed.document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( + installed.document.query_id.clone(), + &dag, + ) + .unwrap(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("update")); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 4e3bb7f31..c227e5eb0 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{StoredOutputId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; -use asap_types::AggregationType; +use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; use crate::query_engines::EngineError; @@ -51,64 +51,40 @@ impl ResolvedMaterialization<'_> { matches!( &self.summary.operator, SummaryOperator::Configured { - aggregation_type: AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease - | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleMin - | AggregationType::MultipleMax, + family: SummaryFamilyType::ExactAggregate(..), .. } ) } fn supports(&self, node: &QueryPlanNode) -> bool { - let SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - .. - } = &self.summary.operator - else { + let SummaryOperator::Configured { family, .. } = &self.summary.operator else { // Partial legacy descriptors cannot attest a configured capability. return false; }; - use AggregationType::*; match node { - QueryPlanNode::ExactReadout { readout, .. } => match readout { - ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Sum, - ExactReadout::Increase | ExactReadout::Rate => { - matches!(aggregation_type, Increase | MultipleIncrease) - } - // Direction is the family now -- no `aggregation_sub_type` - // cross-check, and a minimum summary can no longer be - // offered up for a maximum readout. - ExactReadout::Min => matches!(aggregation_type, Min | MultipleMin), - ExactReadout::Max => matches!(aggregation_type, Max | MultipleMax), - }, + QueryPlanNode::ExactReadout { readout, .. } => family == &readout.planner_family(), QueryPlanNode::SummaryEstimate { query, .. } => match query { QueryReadout::Quantile { q } => { q.is_finite() && (0.0..=1.0).contains(q) - && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) + } + QueryReadout::Cardinality => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon)) } - QueryReadout::Cardinality => matches!(aggregation_type, HLL | UnivMon), QueryReadout::FrequencyL2 | QueryReadout::FrequencyEntropy => { - *aggregation_type == UnivMon + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) } - QueryReadout::PointCount { value: None, .. } if *aggregation_type == UnivMon => { + QueryReadout::PointCount { value: None, .. } if matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) => { true } - QueryReadout::PointCount { .. } => matches!( - aggregation_type, - CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap - ), - QueryReadout::TopK { .. } => matches!( - aggregation_type, - CountMinSketchWithHeap | CountSketchWithHeap - ), + QueryReadout::PointCount { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap)) + } + QueryReadout::TopK { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + } }, _ => false, } diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 7b2350907..b24908b45 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -910,9 +910,9 @@ mod tests { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 0bf056e18..2ea8c6229 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1375,9 +1375,9 @@ mod tests { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 979f3c845..e25ea4fcd 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -65,8 +65,8 @@ use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryNode, + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, }; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; @@ -203,11 +203,13 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match agg_type { - AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease => asap_types::Statistic::Sum, + let stat = match agg_type.planner_exact_family()? { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => asap_types::Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => asap_types::Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { + asap_types::Statistic::Increase + } + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -224,9 +226,7 @@ impl GroupState { .ok() } - /// Finalize a compiler-declared exact readout. Rate and increase share - /// reset-aware Increase state physically, but remain distinct operations - /// in QueryPlan so serving never infers semantics from PromQL text. + /// Finalize the Planner-declared exact family with its matching readout. pub fn exact_value_for( &self, readout: asap_types::query_plan::ExactReadout, @@ -237,40 +237,32 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { - asap_types::Statistic::Count - } - ( - asap_types::query_plan::ExactReadout::Sum, - AggregationType::Sum | AggregationType::MultipleSum, - ) => asap_types::Statistic::Sum, - ( - asap_types::query_plan::ExactReadout::Increase, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Increase, - ( - asap_types::query_plan::ExactReadout::Rate, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Rate, - ( - asap_types::query_plan::ExactReadout::Min, - AggregationType::Min | AggregationType::MultipleMin, - ) => asap_types::Statistic::Min, - ( - asap_types::query_plan::ExactReadout::Max, - AggregationType::Max | AggregationType::MultipleMax, - ) => asap_types::Statistic::Max, - _ => return None, + if agg_type.planner_exact_family().as_ref() != Some(&readout.planner_family()) { + return None; + } + let stat = match readout { + asap_types::query_plan::ExactReadout::Count => asap_types::Statistic::Count, + asap_types::query_plan::ExactReadout::Sum => asap_types::Statistic::Sum, + asap_types::query_plan::ExactReadout::Increase => asap_types::Statistic::Increase, + asap_types::query_plan::ExactReadout::Rate => asap_types::Statistic::Rate, + asap_types::query_plan::ExactReadout::Min => asap_types::Statistic::Min, + asap_types::query_plan::ExactReadout::Max => asap_types::Statistic::Max, }; + let planner_state = entries.iter().flat_map(|w| w.values()).any(|a| { + a.as_any() + .is::() + }); // Temporal exact summaries are the hot path for long-window // dashboards. Merge their concrete, fixed-size states in one batch // instead of allocating a boxed trait object for every pane. - if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease - ) { + if !planner_state + && matches!( + readout, + asap_types::query_plan::ExactReadout::Increase + | asap_types::query_plan::ExactReadout::Rate + ) + { let accumulators = entries .iter() .flat_map(|windows| windows.values()) @@ -286,11 +278,7 @@ impl GroupState { ]); return merged.query_statistic(stat, key, &query_kwargs).ok(); } - if matches!( - agg_type, - AggregationType::Min | AggregationType::MultipleMin - ) && readout == asap_types::query_plan::ExactReadout::Min - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Min { return entries .iter() .flat_map(|windows| windows.values()) @@ -303,11 +291,7 @@ impl GroupState { .into_iter() .reduce(f64::min); } - if matches!( - agg_type, - AggregationType::Max | AggregationType::MultipleMax - ) && readout == asap_types::query_plan::ExactReadout::Max - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() .flat_map(|windows| windows.values()) @@ -334,9 +318,6 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == asap_types::query_plan::ExactReadout::Count { - return merged.aux_stats().count.map(|count| count as f64); - } merged.query_statistic(stat, key, &query_kwargs).ok() } @@ -613,9 +594,13 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + let exact_family = agg_type.planner_exact_family(); if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Increase | ExactKind::Rate, + _ + )) ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -631,12 +616,12 @@ impl QueryExecutionContext<'_> { )); } } - if let Some((reduction, is_min)) = match agg_type { - AggregationType::Min | AggregationType::MultipleMin => Some(( + if let Some((reduction, is_min)) = match exact_family.as_ref() { + Some(SummaryFamilyType::ExactAggregate(ExactKind::Min, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Min, true, )), - AggregationType::Max | AggregationType::MultipleMax => Some(( + Some(SummaryFamilyType::ExactAggregate(ExactKind::Max, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Max, false, )), @@ -688,8 +673,11 @@ impl QueryExecutionContext<'_> { // counter and extrema state. Additive pane summaries must // remain contiguous because a missing pane is not zero. if matches!( - agg_type, - AggregationType::Sum | AggregationType::MultipleSum + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count, + _ + )) ) { check_panes(windows.keys().copied().collect())?; } @@ -934,9 +922,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } ( - GroupState::ExactAgg { entries, .. }, - GroupState::ExactAgg { entries: more, .. }, + GroupState::ExactAgg { entries, agg_type }, + GroupState::ExactAgg { + entries: more, + agg_type: incoming, + }, ) => { + if agg_type.planner_exact_family() != incoming.planner_exact_family() { + return Err(SummaryExecutorError::UnsupportedFamily); + } entries.extend(more); } // `find_candidates`'s exact-match contract never produces a @@ -1278,29 +1272,19 @@ fn summary_family_matches_sketch( /// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> /// ExactKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease -/// -> ExactKind::Increase` — confirmed against that module's own -/// dispatch table rather than invented here). +/// side. Count and Rate remain distinct families even though their runtime +/// accumulators share implementations with Sum and Increase. /// -/// `ExactKind::Count`/`Rate`/`Min`/`Max` are not matched by this legacy -/// family-discovery path. For `Count`/`Rate` the final operation is ambiguous -/// from the stored accumulator alone. `Min`/`Max` were excluded for a reason -/// that no longer holds -- direction used to be unrecoverable once a summary -/// reached `AggKind::ExactAgg`, and is now the family itself -- but admitting -/// them here widens candidate discovery beyond the family split and is left -/// as follow-up. Installed QueryPlans carry an explicit `ExactReadout`, and +/// Installed QueryPlans carry an explicit `ExactReadout`, and /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (family, agg_type), - ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - AggregationType::Sum | AggregationType::MultipleSum, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - AggregationType::Increase | AggregationType::MultipleIncrease, + family, + SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count | ExactKind::Increase | ExactKind::Rate, + _ ) - ) + ) && agg_type.planner_exact_family().as_ref() == Some(family) } /// Project a full label-values map down to the requested `by` columns -- @@ -1470,6 +1454,58 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + #[test] + fn keyed_count_state_follows_planner_family_and_query_readout() { + use crate::precompute_engine::operators::KeyedSumCountAccumulator; + use asap_types::query_plan::ExactReadout; + + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut payload = KeyedSumCountAccumulator::for_family(ExactKind::Count); + payload.update(key.clone(), 10.0); + payload.update(key.clone(), 20.0); + let state = GroupState::ExactAgg { + entries: vec![Rc::new(BTreeMap::from([( + 60_000, + Arc::new(payload) as Arc, + )]))], + agg_type: AggregationType::Count, + }; + assert_eq!( + state.exact_value_for(ExactReadout::Count, &Some(key.clone()), 0, 60_000), + Some(2.0) + ); + assert_eq!( + state.exact_value_for(ExactReadout::Sum, &Some(key), 0, 60_000), + None + ); + } + + #[test] + fn state_merge_rejects_different_planner_families() { + let index = SketchStore::new(); + let context = QueryExecutionContext { + index: &index, + t0_ms: 0, + t1_ms: 60_000, + is_cumulative: true, + allowed_materializations: None, + }; + let states = vec![ + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Rate, + }, + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Increase, + }, + ]; + assert!(matches!( + context.merge_states(states), + Err(SummaryExecutorError::UnsupportedFamily) + )); + } + #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { let binding = asap_types::query_plan::MaterializationBinding { diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index 08eb75e00..0d46b2525 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -102,7 +102,7 @@ impl QueryResult { /// Attach an accuracy envelope. Chainable so engine paths /// can build the bare result first and decorate once the - /// `agg_id → AggregationConfig → AccuracyProfile` lookup + /// `agg_id → PrecomputeMaterialization → AccuracyProfile` lookup /// has resolved. pub fn with_accuracy(mut self, envelope: AccuracyEnvelope) -> Self { match &mut self { diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index f3780b044..084c87cc6 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -1,5 +1,5 @@ //! `AccuracyProfile` — derived error / confidence bound for each -//! `AggregationConfig`. +//! `PrecomputeMaterialization`. //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. @@ -12,7 +12,7 @@ //! //! ## Scope of this module //! -//! Pure derivation: `derive(&AggregationConfig)` +//! Pure derivation: `derive(&PrecomputeMaterialization)` //! looks at `aggregation_type` and the relevant entries in //! `config.parameters` and returns an `AccuracyProfile`. No //! runtime measurement, no sampling — just the textbook bound. @@ -31,14 +31,14 @@ use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::AggregationType; pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; use planner_types::post_asap::SketchParams as PlannerParams; /// Derive an [`AccuracyProfile`] from a pinned -/// [`AggregationConfig`]. Reads `aggregation_type` and any +/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any /// necessary entries in `parameters`; falls back to exact for /// unknown / legacy variants (harmless — the caller just gets /// "0 error" rather than a panic). @@ -52,7 +52,7 @@ use planner_types::post_asap::SketchParams as PlannerParams; /// ε_st`; the random parts compose in quadrature but the staleness part is /// adversarial, so linear addition is the honest envelope). δ is /// unchanged (staleness is not probabilistic). -pub fn derive(config: &AggregationConfig) -> AccuracyProfile { +pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { let mut profile = derive_sketch_only(config); let eps_st = config .parameters @@ -67,20 +67,20 @@ pub fn derive(config: &AggregationConfig) -> AccuracyProfile { /// Source adapter for installed aggregation configs. pub trait BackendAccuracyProfile { - fn derive(config: &AggregationConfig) -> Self; - fn derive_sketch_only(config: &AggregationConfig) -> Self; + fn derive(config: &PrecomputeMaterialization) -> Self; + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; } impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &AggregationConfig) -> Self { + fn derive(config: &PrecomputeMaterialization) -> Self { derive(config) } - fn derive_sketch_only(config: &AggregationConfig) -> Self { + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { derive_sketch_only(config) } } /// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { +fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { match config.aggregation_type { AggregationType::UnivMon => AccuracyProfile { epsilon: f64::MAX, @@ -91,13 +91,11 @@ fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { // `DeltaSetAggregator` exact-set-membership family lived // here too before its retirement.) AggregationType::Sum + | AggregationType::Count | AggregationType::Increase + | AggregationType::Rate | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax => AccuracyProfile::exact(), + | AggregationType::Max => AccuracyProfile::exact(), AggregationType::CountMinSketch => { let (rows, cols) = cms_params(config); @@ -220,7 +218,7 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { // authority on *accuracy*, not on *construction*. /// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &AggregationConfig) -> (u64, u64) { +fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { let rows = config .parameters .get("d") @@ -234,7 +232,7 @@ fn cms_params(config: &AggregationConfig) -> (u64, u64) { (rows, cols) } -fn hll_precision(config: &AggregationConfig) -> u32 { +fn hll_precision(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("precision") @@ -244,7 +242,7 @@ fn hll_precision(config: &AggregationConfig) -> u32 { .unwrap_or(14) } -fn kll_k(config: &AggregationConfig) -> u32 { +fn kll_k(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("K") @@ -254,7 +252,7 @@ fn kll_k(config: &AggregationConfig) -> u32 { .unwrap_or(200) } -fn ddsketch_alpha(config: &AggregationConfig) -> f64 { +fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { config .parameters .get("alpha") @@ -266,7 +264,7 @@ fn ddsketch_alpha(config: &AggregationConfig) -> f64 { /// from `parameters["heap_size"]` with a default of 100 — /// matches the default the control plane's planner uses when the /// caller didn't override. -fn cms_heap_size(config: &AggregationConfig) -> u64 { +fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { config .parameters .get("heap_size") @@ -373,8 +371,11 @@ mod tests { use serde_json::{json, Value}; use std::collections::HashMap; - fn base_config(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( + fn base_config( + agg_type: AggregationType, + params: HashMap, + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index 0e970b821..ce8e476d2 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use std::time::{SystemTime, UNIX_EPOCH}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; @@ -507,12 +507,12 @@ impl BackfillRegistry { /// control-plane-facing HTTP endpoint can return specific 404 / /// 409 / 400 statuses. `CreateError::UnknownAgg` is no longer /// returned from this method — the caller proves the agg - /// exists by holding the `AggregationConfig` — but the variant + /// exists by holding the `PrecomputeMaterialization` — but the variant /// is kept on the enum for HTTP error-mapping compatibility /// (the handler still produces it when its own lookup misses). pub fn create_checked( &self, - config: &AggregationConfig, + config: &PrecomputeMaterialization, created_at_ms: u64, time_range: (u64, u64), source: BackfillSource, @@ -883,7 +883,9 @@ pub use service::{ default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, BackfillServiceHandle, ReaderFactory, }; +#[cfg(test)] pub use window_builder::build_backfilled_accumulator; +pub use window_builder::build_dag_accumulator; pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; #[cfg(test)] diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 430aca740..8bfcf0cf0 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -22,10 +22,11 @@ use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use super::raw_sample_reader::RawSample; +#[cfg(test)] use super::window_builder::build_backfilled_accumulator; use super::worker::WindowProcessor; use super::BackfillRegistry; @@ -39,7 +40,7 @@ use super::BackfillRegistry; /// Kept local to the backfill module (not shared with live) per /// the §5e separation ask; the implementations must stay identical /// by convention. -fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { +fn extract_group_key(series_key: &str, config: &PrecomputeMaterialization) -> String { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); for label_name in &config.grouping_labels.names() { @@ -71,7 +72,7 @@ fn build_group_key_label_values(group_key: &str) -> KeyByLabelValues { /// live windows occupy the same storage row. fn resolve_backfill_bucket_sid( resolver: &SeriesIdResolver, - config: &AggregationConfig, + config: &PrecomputeMaterialization, series_key: &str, store: Option<&crate::storage_engines::sketch_db::index::SketchStore>, captured_generation: Option<&asap_types::sds::CatalogGeneration>, @@ -124,7 +125,7 @@ fn fallback_bucket_id(group_key: &str) -> u64 { pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest /// `StreamingConfig` at each window to find the - /// `AggregationConfig` for `agg_id`. The snapshot is cheap + /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. config: StreamingConfigHandle, /// Destination for rebuilt windows. Tests may omit it to record registry @@ -185,22 +186,6 @@ impl BackfillWindowProcessor { self.series_resolver = Some(series_resolver); self } - - /// Look up the `AggregationConfig` for `agg_id` in the current - /// `StreamingConfig` snapshot. Returns an error string if the - /// agg has been removed from the config since the job was - /// created — rare but worth handling (e.g. operator retired - /// the agg mid-backfill; the `BackfillWorker` will - /// `mark_failed` the job with this message). - fn config_for_agg( - &self, - agg_id: u64, - ) -> Result> { - let snap = self.config.snapshot(); - snap.get_aggregation_config(agg_id).cloned().ok_or_else(|| { - format!("agg_id {agg_id} not in current StreamingConfig — retired mid-backfill?").into() - }) - } } /// One per-sid bucket assembled by [`BackfillWindowProcessor::process_window`]. @@ -219,7 +204,16 @@ impl WindowProcessor for BackfillWindowProcessor { window_range: (u64, u64), samples: Vec, ) -> Result<(), Box> { - let config = self.config_for_agg(agg_id)?; + let snapshot = self.config.snapshot(); + let config = snapshot + .get_aggregation_config(agg_id) + .cloned() + .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + let program = snapshot.raw_programs.get(&agg_id).cloned(); + #[cfg(not(test))] + if program.is_none() { + return Err("backfill requires a post-ASAP DAG installation".into()); + } // B7.7 — sid-keyed bucketing. Per the schema-retirement #5 // step 6 plan, the backfill processor's per-window grouping is @@ -289,7 +283,18 @@ impl WindowProcessor for BackfillWindowProcessor { for (sid, bucket) in by_bucket { let SidBucket { group_key, samples } = bucket; - let accumulator = build_backfilled_accumulator(&config, &samples); + let accumulator = if let Some(program) = &program { + super::window_builder::build_dag_accumulator(program, &samples)? + } else { + #[cfg(test)] + { + build_backfilled_accumulator(&config, &samples) + } + #[cfg(not(test))] + { + return Err("missing backfill DAG producer".into()); + } + }; // Keyed accumulators (MultipleSubpopulation) carry their // subpopulation keys internally; the PrecomputedOutput's // `key` represents the *group* key (grouping_labels @@ -374,7 +379,7 @@ mod tests { use asap_types::KeyByLabelNames; use std::sync::Arc; - fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. let grouping_labels = if grouping.is_empty() { @@ -382,7 +387,7 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -401,7 +406,7 @@ mod tests { ) } - fn streaming_config_with(config: AggregationConfig) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); Arc::new(StreamingConfig::new(map)) @@ -567,7 +572,7 @@ mod tests { /// when given the same ordered samples. #[test] fn backfill_builds_bit_identical_sum_accumulator_to_live() { - use crate::precompute_engine::accumulator_factory::create_accumulator_updater; + use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; let cfg = sum_config(1, "m", vec![]); @@ -592,7 +597,7 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_accumulator_updater(&cfg); + let mut updater = create_fixture_accumulator(&cfg); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -612,7 +617,7 @@ mod tests { serialisations for SumAccumulator. \ If this test fails, something diverged — check:\n\ (1) Is `build_backfilled_accumulator` still calling \ - `create_accumulator_updater`?\n\ + `create_fixture_accumulator`?\n\ (2) Did a recent change to `SumAccumulator` introduce \ non-deterministic state (e.g. a seed)?\n\ (3) Does `serialize_to_bytes` include any timestamp \ diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index 7f9208a1e..79d9ccdb0 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -43,7 +43,7 @@ pub struct RawSample { /// honour. Exactly one metric name plus zero or more equality /// matchers on grouping labels — no regex, no negation, no /// lexicographic ranges. The control plane picks the subset of -/// `AggregationConfig.grouping_labels` that should gate the read. +/// `PrecomputeMaterialization.grouping_labels` that should gate the read. /// /// Rationale: every supported exact-DB backend (Prometheus, /// ClickHouse, S3+Gorilla) can evaluate this filter efficiently, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index c0b35971a..86d64fac3 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -342,15 +342,15 @@ mod tests { MockRawSampleReader, RawSample, }; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Mutex; - fn sum_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -369,7 +369,7 @@ mod tests { ) } - fn streaming_with(cfg: AggregationConfig) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); Arc::new(StreamingConfig::new(m)) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index 04a4eed39..6bf028c9c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -4,13 +4,16 @@ //! It shares the pure accumulator factory and update primitives with live ingest //! so both paths use the same sketch semantics. +#[cfg(test)] use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, + create_fixture_accumulator, AccumulatorUpdater, }; +#[cfg(test)] use crate::precompute_engine::worker::apply_sample; use crate::storage_engines::sketch_db::backfill::raw_sample_reader::RawSample; use crate::storage_engines::types::AggregateCore; -use asap_types::aggregation_config::AggregationConfig; +#[cfg(test)] +use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -25,11 +28,12 @@ use asap_types::aggregation_config::AggregationConfig; /// The function is synchronous + pure (no I/O, no async, no global /// state). Suitable to call from inside a `WindowProcessor` /// implementation without worrying about the async runtime. +#[cfg(test)] pub fn build_backfilled_accumulator( - config: &AggregationConfig, + config: &PrecomputeMaterialization, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_accumulator_updater(config); + let mut updater: Box = create_fixture_accumulator(config); for sample in samples { apply_sample( &mut *updater, @@ -45,14 +49,14 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_config() -> AggregationConfig { - AggregationConfig::new( + fn sum_config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), @@ -156,3 +160,28 @@ mod tests { assert!(aux.sum == Some(0.0) || aux.sum.is_none()); } } + +/// Backfill uses the same selected DAG producer and update expressions as live input. +pub fn build_dag_accumulator( + program: &crate::precompute_engine::raw_dag::RawDagProgram, + samples: &[RawSample], +) -> Result, String> { + let mut updater = program.updater()?; + let mut previous = std::collections::HashMap::new(); + for sample in samples { + let value = if program.uses_counter_delta() { + crate::precompute_engine::worker::reset_aware_counter_delta( + &mut previous, + &sample.labels, + sample.value, + sample.timestamp_ms, + ) + } else { + Some(sample.value) + }; + if let Some(value) = value { + program.apply(&mut *updater, &sample.labels, value, sample.timestamp_ms)?; + } + } + Ok(updater.take_accumulator()) +} diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index e8f3d873f..e696fe525 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -137,7 +137,7 @@ pub enum AggKind { /// Complete resolver identity for a configured materialization. All live and /// replay paths must include policy semantics, not just the sketch family. pub(crate) fn materialization_kind_for_config( - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, ) -> String { format!( "{}|{}", @@ -149,7 +149,9 @@ pub(crate) fn materialization_kind_for_config( /// Resolve the physical state family produced by a precompute policy. This is /// shared by SID minting and store registration so a sketch policy can never /// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config(config: &asap_types::aggregation_config::AggregationConfig) -> AggKind { +pub fn agg_kind_for_config( + config: &asap_types::aggregation_config::PrecomputeMaterialization, +) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; // HLL is intentionally absent from raw-value accumulator dispatch because @@ -592,7 +594,7 @@ mod tests { #[test] fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::AggregationConfig::new( + let config = asap_types::aggregation_config::PrecomputeMaterialization::new( AggregationType::HLL, String::new(), HashMap::from([("precision".to_string(), serde_json::json!(12))]), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e099f0a21..c3a0283ed 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -92,11 +92,12 @@ fn reconstruct_exact_agg( bytes: &[u8], ) -> Option> { use crate::precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleIncreaseAccumulator, - MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedCounterState, KeyedSumCountAccumulator, MaxAccumulator, + MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::AggregateCore; match type_name { + "PlannerExactAccumulatorV1" => crate::precompute_engine::operators::exact_accumulator::ExactAccumulator::deserialize_from_bytes(bytes).ok().map(|a|Box::new(a) as Box), "SumAccumulator" => SumAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), @@ -109,10 +110,12 @@ fn reconstruct_exact_agg( "MaxAccumulator" => MaxAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), - "MultipleSumAccumulator" => MultipleSumAccumulator::deserialize_from_bytes(bytes) - .ok() - .map(|a| Box::new(a) as Box), - "MultipleIncreaseAccumulator" => MultipleIncreaseAccumulator::deserialize_from_bytes(bytes) + "KeyedSumCountAccumulator" => { + KeyedSumCountAccumulator::deserialize_from_bytes(bytes) + .ok() + .map(|a| Box::new(a) as Box) + } + "KeyedCounterState" => KeyedCounterState::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), // The keyed `MultipleMin`/`MultipleMax` forms and the @@ -138,7 +141,7 @@ fn reconstruct_exact_agg( /// so the mint-driven path (B7.6) and the sid-direct path (B7.7) stay /// byte-identical on the values they hand to the index. fn build_attrs_fp_and_label_map( - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Result<(String, BTreeMap), String> { if let Some(labels) = &output.population_labels { @@ -232,7 +235,7 @@ pub struct SummarySeriesMetadata { /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source - /// `AggregationConfig` (test fixtures + the early-Phase-5 sketch + /// `PrecomputeMaterialization` (test fixtures + the early-Phase-5 sketch /// ingest path that didn't thread the config through); the index /// skips those entries — they're reachable through the legacy /// `instances_matching(metric, gbk)` walk if a query needs them. @@ -2253,7 +2256,7 @@ impl SketchStore { } /// Phase 5 M2.3.5 — query the precompute payloads across every sid - /// belonging to one `AggregationConfig` (identified by `metric` + + /// belonging to one `PrecomputeMaterialization` (identified by `metric` + /// `agg_cfg.aggregation_type`), shaped as the legacy `Store` /// trait's `TimestampedBucketsMap`. Lets the query engine swap /// `Store::query_precomputed_output` for `SketchStore` without @@ -2956,7 +2959,7 @@ impl SketchStore { } /// Phase 5 M2.3.6e — write-side helper. Given an - /// `AggregationConfig` and one `(PrecomputedOutput, AggregateCore)` + /// `PrecomputeMaterialization` and one `(PrecomputedOutput, AggregateCore)` /// pair (the shape both the live worker AND the backfill processor /// emit), compute the precompute sid, register a metadata entry on /// first sight, and append the payload window. Used by @@ -2976,7 +2979,7 @@ impl SketchStore { pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3108,7 +3111,7 @@ impl SketchStore { pub fn ingest_precompute_with_series_id( &self, sid: u64, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3126,6 +3129,18 @@ impl SketchStore { output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + let expected = agg_cfg.accumulator_spec().ok()?.family; + if matches!( + expected, + planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) + ) && accumulator + .get_accumulator_type() + .planner_exact_family() + .as_ref() + != Some(&expected) + { + return None; + } let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; // Keep the physical lifetime alive through publication. Removal takes @@ -6315,4 +6330,91 @@ mod tests { ); assert_eq!(idx.series.len(), 2); } + // Flush and reopen must preserve Planner family rather than reconstructing Rate as Increase. + #[test] + fn planner_exact_families_survive_disk_eviction_and_restart() { + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::storage_engines::types::{AggregateCore, AggregationType}; + let kinds = [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Min, + AggregationType::Max, + AggregationType::Rate, + AggregationType::Increase, + ]; + let stats = [ + asap_types::Statistic::Sum, + asap_types::Statistic::Count, + asap_types::Statistic::Min, + asap_types::Statistic::Max, + asap_types::Statistic::Rate, + asap_types::Statistic::Increase, + ]; + let expected = [16.0, 3.0, 2.0, 8.0, 3.0, 6.0]; + let temp = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + for (i, kind) in kinds.iter().enumerate() { + let mut metadata = meta(9000 + i as u64); + metadata.agg_kind = AggKind::ExactAgg { + agg_type: *kind, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }; + metadata.capability = Some(Capability::ExactAgg(*kind)); + metadata.accuracy = None; + store.register(metadata); + } + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + for window in 0..10u64 { + let mut state = + ExactAccumulator::new(kind.planner_exact_family().unwrap(), false).unwrap(); + for (time, value) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(None, value, time); + } + store.append_precompute( + 9000 + i as u64, + BTreeMap::new(), + (window * 30000, (window + 1) * 30000), + Box::new(state), + ); + } + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.approx_memory_bytes() == 0 + && store.list_sealed_epochs_len() == 0, + std::time::Duration::from_secs(5) + )); + persistence.shutdown(); + } + let store = Arc::new(SketchStore::new()); + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + let series = store.query_exact_agg_range(9000 + i as u64, 0, 30001); + assert_eq!(series.len(), 1, "{kind:?}"); + let state = &series[0].1[&30000]; + assert_eq!(state.get_accumulator_type(), *kind); + assert_eq!( + state + .query_statistic(stats[i], &None, &HashMap::new()) + .unwrap(), + expected[i] + ); + for (j, stat) in stats.iter().enumerate() { + if i != j { + assert!(state + .query_statistic(*stat, &None, &HashMap::new()) + .is_err()); + } + } + } + persistence.shutdown(); + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 668aa99e9..f82f06a17 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -195,13 +195,13 @@ mod tests { use super::*; use crate::precompute_engine::operators::SumAccumulator; use crate::storage_engines::types::{AggregationType, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> AggregationConfig { - AggregationConfig { + fn sum_agg_config(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index f9a834934..0b712f456 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::Duration; use crate::storage_engines::types::StreamingConfig; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; use crate::storage_engines::sketch_db::index::SketchStore; @@ -87,7 +87,7 @@ pub fn reconcile_from_streaming_config( } // `live_signatures` is built exclusively from // `signature_from_agg_config`, which canonicalizes every - // streaming-config `AggregationConfig` to an `AggKind::ExactAgg` + // streaming-config `PrecomputeMaterialization` to an `AggKind::ExactAgg` // signature (`P`-prefixed). An `AggKind::Sketch` sid (OTLP // modified-sketch ingest path: KLL / HLL / DDSketch / CMS / // CountSketch) always produces an `S`-prefixed signature, so it @@ -166,7 +166,7 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { +fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { let agg_kind = AggKind::ExactAgg { agg_type: cfg.aggregation_type, parameters_canonical: canonical_parameters(&cfg.parameters), @@ -264,7 +264,7 @@ mod tests { use super::*; use std::collections::HashMap; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -276,8 +276,8 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), HashMap::new(), @@ -321,7 +321,7 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> StreamingConfig { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); diff --git a/data_plane/src/storage_engines/sketch_db/mod.rs b/data_plane/src/storage_engines/sketch_db/mod.rs index e2aff1959..0ef39870c 100644 --- a/data_plane/src/storage_engines/sketch_db/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/mod.rs @@ -16,12 +16,12 @@ pub mod sds; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; pub use backfill::{ - build_backfilled_accumulator, clickhouse_reader_factory, default_reader_factory, - noop_reader_factory, BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, - BackfillWorkerError, ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, - MockRawSampleReader, PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, - ReaderFactory, WindowProcessor, + build_dag_accumulator, clickhouse_reader_factory, default_reader_factory, noop_reader_factory, + BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, BackfillServiceHandle, + BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, BackfillWorkerError, + ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, MockRawSampleReader, + PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, ReaderFactory, + WindowProcessor, }; pub use lifecycle::{ warn_if_retention_inverted, AggStatus, SchemaEvictionConfig, SchemaEvictionHandle, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index d1d15c6e0..fc9062f1e 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -680,7 +680,7 @@ impl ActivePhysicalPlanHandle { #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::AggregationConfig; + use crate::storage_engines::types::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -756,8 +756,8 @@ mod tests { } } - fn dummy_agg(id: u64) -> AggregationConfig { - AggregationConfig::new( + fn dummy_agg(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 57d47470b..a91e83148 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -25,7 +25,7 @@ pub use streaming_config::*; pub use traits::*; // Cross-module re-export of asap_types data types so callers can -// write `crate::storage_engines::types::AggregationConfig` instead of +// write `crate::storage_engines::types::PrecomputeMaterialization` instead of // reaching across crates. pub use asap_types::aggregation_config::*; diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 4d268dc41..28e71b4c3 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -62,7 +62,7 @@ pub struct PrecomputedOutput { #[serde(default)] pub origin: Origin, /// Content-addressed policy identity. The data plane's only handle - /// on which source `AggregationConfig` produced this output. + /// on which source `PrecomputeMaterialization` produced this output. /// `#[serde(default)]` on read preserves forward-compat with /// PR-3 / PR-4-era records that may not have carried the field; /// sinks treat `PolicyFingerprint::UNSET` as "skip this output" @@ -75,7 +75,7 @@ impl PrecomputedOutput { /// Construct a `Native` precompute. /// /// `policy_fp` is the content-addressed handle on the source - /// [`asap_types::AggregationConfig`]; sinks use it to look up the + /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the /// config via `PolicyRegistry::get(policy_fp)`. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 430dd10bf..95055c2ea 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -6,31 +6,22 @@ use std::fs::File; use std::io::BufReader; use std::ops::Index; -use asap_types::enums::QueryLanguage; -use asap_types::{AggregationConfig, MonitorSpec, PolicyRegistry}; +use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; use super::storage_backend::StorageBackend; -/// The backend's active streaming policy config: every `AggregationConfig` -/// currently pushed by the controller, plus the storage-backend pin and CDM -/// monitor specs. -/// -/// Formerly `asap_types::streaming_config::StreamingConfig` — moved here -/// (see `scratchpad/artifacts/enum-unification-plan.md`) because -/// `control_plane` never actually depended on this type: its own -/// `StreamingConfigEmitter` hand-builds wire-compatible JSON independently, -/// and `PolicyRegistry::from_streaming_config` (the only thing that made -/// `asap_types::PolicyRegistry` -- genuinely shared -- look coupled to this -/// type) had exactly one real caller, this struct's own `policy_registry()` -/// method below. `asap_types` keeps the lower-level `PolicyRegistry:: -/// from_configs` primitive this method now calls directly. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// DAG installation plus a derived in-memory routing index. The flat index is +/// never serialized as executable configuration. Raw programs are validated +/// and shared once per installed producer across all of its population states. +#[derive(Debug, Clone, Serialize)] pub struct StreamingConfig { - #[serde( - rename = "aggregation_configs", - alias = "materializations_by_policy_fingerprint" - )] - pub materializations_by_policy_fingerprint: HashMap, + #[serde(skip)] + pub(crate) raw_programs: + HashMap>, + /// Authoritative execution configuration: Planner DAGs and physical bindings. + pub precompute_plan: Option, + #[serde(skip)] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -45,15 +36,60 @@ pub struct StreamingConfig { pub monitors: Vec, } +// Flat aggregation lists are deliberately not an accepted execution document. +impl<'de> Deserialize<'de> for StreamingConfig { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + precompute_plan: asap_types::precompute_plan::PrecomputePlan, + #[serde(default)] + storage_backend: StorageBackend, + #[serde(default)] + monitors: Vec, + } + let doc = Document::deserialize(deserializer)?; + let mut config = + Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; + config.storage_backend = doc.storage_backend; + config.monitors = doc.monitors; + Ok(config) + } +} + impl StreamingConfig { - pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { + pub fn new( + materializations_by_policy_fingerprint: HashMap, + ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } } + /// Build the routing projection only after validating the DAG installation. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|c| { + c.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::new(materializations); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + /// CDM monitor specs the data-plane coordinator should serve (may be empty). pub fn monitors(&self) -> &[MonitorSpec] { &self.monitors @@ -63,10 +99,12 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), @@ -80,12 +118,15 @@ impl StreamingConfig { self.storage_backend } - pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { + pub fn get_aggregation_config( + &self, + aggregation_id: u64, + ) -> Option<&PrecomputeMaterialization> { self.materializations_by_policy_fingerprint .get(&aggregation_id) } - pub fn materializations(&self) -> &HashMap { + pub fn materializations(&self) -> &HashMap { &self.materializations_by_policy_fingerprint } @@ -126,56 +167,12 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut materializations_by_policy_fingerprint: HashMap = - HashMap::new(); - - if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { - for aggregation_data in aggregations { - // Retention comes from each aggregation entry; identity is derived from - // its configuration content. - let num_aggregates_to_retain = aggregation_data - .get("numAggregatesToRetain") - .and_then(|v| v.as_u64()); - let config = AggregationConfig::from_yaml_data( - aggregation_data, - num_aggregates_to_retain, - QueryLanguage::PromQl, - )?; - if !config.population_key_encoding.is_legacy() { - anyhow::bail!( - "legacy streaming input does not support this population key encoding" - ); - } - if config.derived_input.is_some() { - anyhow::bail!( - "legacy streaming input cannot execute a derived summary program" - ); - } - // PR 5: the map key IS the policy-fingerprint u64. - // `AggregationConfig::policy_fp_u64()` is the canonical - // accessor for this value. - materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); - } - } - - let mut config = Self::new(materializations_by_policy_fingerprint); - // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each - // entry deserializing into a MonitorSpec. Absent → empty (the common - // case). The data-plane monitor coordinator reads these. - if let Some(monitors) = data.get("monitors").and_then(|v| v.as_sequence()) { - for m in monitors { - let spec: MonitorSpec = serde_yaml::from_value(m.clone()).map_err(|e| { - anyhow::anyhow!("invalid monitor spec in streaming-config: {e}") - })?; - config.monitors.push(spec); - } - } - Ok(config) + serde_yaml::from_value(data.clone()).map_err(Into::into) } } impl Index for StreamingConfig { - type Output = AggregationConfig; + type Output = PrecomputeMaterialization; fn index(&self, aggregation_id: u64) -> &Self::Output { &self.materializations_by_policy_fingerprint[&aggregation_id] @@ -190,7 +187,7 @@ impl Default for StreamingConfig { impl StreamingConfig { #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { + pub fn get_all_aggregation_configs(&self) -> &HashMap { self.materializations() } } @@ -199,153 +196,16 @@ impl StreamingConfig { mod tests { use super::*; - /// Pre-Phase-5 deploys serialize `StreamingConfig` without the - /// `storage_backend` field; deserialize must default to `SketchStore` - /// so the router keeps dispatching to `ASAPQueryEngine` unchanged. - #[test] - fn deserialize_legacy_yaml_defaults_to_asap_tier() { - let yaml = "{\"aggregation_configs\":{}}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("legacy decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::SketchStore); - } - - #[test] - fn deserialize_with_explicit_double_write_pin() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"double_write\"}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("Phase-5 decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::DoubleWrite); - } - - /// #746 deleted the archive tier; its storage-axis spelling is no longer - /// a known variant, so a stale config naming it fails to decode rather - /// than silently pinning some other tier. + // Old flat lists cannot become execution authority through JSON or YAML. #[test] - fn deserialize_rejects_the_removed_archive_axis() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_object_store\"}"; - assert!(serde_json::from_str::(yaml).is_err()); - } - - #[test] - fn legacy_yaml_rejects_derived_summary_input() { - let data = serde_yaml::from_str::(&format!( - "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) - )).unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!(error - .to_string() - .contains("legacy streaming input cannot execute")); - } - - #[test] - fn legacy_yaml_rejects_canonical_population_key_encoding() { - let data: Value = serde_yaml::from_str( - r#" -aggregations: -- aggregationType: Sum - aggregationSubType: '' - metric: m - population_key_encoding: canonical_labels_v1 - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#, - ) - .unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!( - error.to_string().contains("population key encoding"), - "{error}" - ); - } - - /// PR 5: a streaming-config YAML that omits `aggregationId` - /// parses correctly — the backend derives identity from content - /// via `PolicyFingerprint::from_config`. The map key is the - /// fingerprint's u64 form. - #[test] - fn from_yaml_data_accepts_entry_without_aggregation_id() { - let yaml = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); - let (k, v) = cfg - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_ne!(*k, 0, "derived id is not the 0 sentinel"); - assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); - assert_eq!(v.metric, "cpu_seconds"); - } - - /// PR 5: a streaming-config YAML that still spells out - /// `aggregationId: N` parses the SAME as one without — the field - /// is silently dropped. - #[test] - fn from_yaml_data_ignores_explicit_aggregation_id() { - let with = "\ -aggregations:\n\ -- aggregationId: 42\n aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let without = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let w: Value = serde_yaml::from_str(with).expect("with yaml ok"); - let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); - let cw = StreamingConfig::from_yaml_data(&w).expect("with"); - let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - let (kwo, _) = cwo - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_eq!( - kw, kwo, - "explicit aggregationId in YAML must not change identity" - ); - assert_ne!( - *kw, 42, - "the explicit value must NOT leak through as the map key" - ); - } - - #[test] - fn from_yaml_data_parses_monitors_section() { - // CDM monitor specs: a top-level `monitors:` array must populate - // StreamingConfig.monitors (the data-plane coordinator reads these). - let yaml = "\ -aggregations: []\n\ -monitors:\n\ -- agg_id: 16346598078036168951\n key: \"\"\n tau: 5000.0\n epsilon: 0.05\n window_ms: 10000\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode monitors"); - assert_eq!(cfg.monitors().len(), 1, "monitors: section must be parsed"); - let m = &cfg.monitors()[0]; - assert_eq!(m.agg_id, 16346598078036168951); - assert_eq!(m.tau, 5000.0); - assert_eq!(m.window_ms, 10000); - assert_eq!(m.epsilon, 0.05); - } - - #[test] - fn from_yaml_data_absent_monitors_is_empty() { - let yaml = "aggregations: []\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode"); - assert!( - cfg.monitors().is_empty(), - "no monitors: → empty (byte-compat)" - ); + fn rejects_flat_aggregation_documents() { + for text in [ + r#"{"aggregation_configs":{}}"#, + "aggregations: []", + "aggregations: [{aggregationType: Sum, metric: m}]", + ] { + let yaml = serde_yaml::from_str(text).unwrap(); + assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); + } } } diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index ed88c51d9..21cd2b115 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,7 +27,7 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -35,8 +35,8 @@ use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( +fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 3769f23af..974673941 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -2,14 +2,14 @@ //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore //! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build AggregationConfig with +//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with //! the correct aggregation_type string. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationConfig, AggregationType, KeyByLabelValues, PrecomputedOutput, QueryLanguage, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, StreamingConfig, WindowKind, }; use crate::AggregateCore; @@ -23,7 +23,7 @@ use std::collections::HashMap; fn ingest_with_fresh_resolver( summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, - agg_cfg: &AggregationConfig, + agg_cfg: &PrecomputeMaterialization, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { @@ -89,7 +89,7 @@ pub fn create_engine_single_pop_with_aggregated( .collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -121,6 +121,8 @@ pub fn create_engine_single_pop_with_aggregated( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -177,7 +179,7 @@ pub fn create_engine_dual_input( let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation - let value_agg_config = AggregationConfig { + let value_agg_config = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -209,7 +211,7 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation - let keys_agg_config = AggregationConfig { + let keys_agg_config = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -241,6 +243,8 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -306,7 +310,7 @@ pub fn create_engine_two_metrics( let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config_a = AggregationConfig { + let agg_config_a = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -337,7 +341,7 @@ pub fn create_engine_two_metrics( let id_a = agg_config_a.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_a, agg_config_a); - let agg_config_b = AggregationConfig { + let agg_config_b = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -369,6 +373,8 @@ pub fn create_engine_two_metrics( materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -444,7 +450,7 @@ pub fn create_engine_three_metrics( (aggregation_type_b, &labels_b, metric_b), (aggregation_type_c, &labels_c, metric_c), ] { - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -478,6 +484,8 @@ pub fn create_engine_three_metrics( } let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -528,7 +536,7 @@ pub fn create_engine_multi_timestamp( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -560,6 +568,8 @@ pub fn create_engine_multi_timestamp( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -588,7 +598,7 @@ pub fn create_engine_multi_timestamp( /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the AggregationConfig (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -604,7 +614,7 @@ pub fn create_engine_multi_timestamp_with_window( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { stored_output_id: None, semantic_fragment: None, population_key_encoding: Default::default(), @@ -636,6 +646,8 @@ pub fn create_engine_multi_timestamp_with_window( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b56408a24..a10cd3d14 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,4 +1,4 @@ -use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{KeyedSumCountAccumulator, SumAccumulator}; #[cfg(test)] use crate::storage_engines::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, @@ -18,7 +18,7 @@ fn test_single_subpopulation_interface() { #[test] fn test_multiple_subpopulation_interface() { // Multiple accumulator - matches Python behavior exactly - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let mut key = KeyByLabelValues::new(); key.insert("web".to_string()); @@ -43,7 +43,7 @@ fn test_interface_prevents_misuse() { let single_acc: Box = Box::new(SumAccumulator::with_sum(42.0)); let multi_acc: Box = - Box::new(MultipleSumAccumulator::new()); + Box::new(KeyedSumCountAccumulator::new()); // ✅ These work - correct usage let _result1 = single_acc.query(Statistic::Sum, None); @@ -68,7 +68,7 @@ fn test_python_alignment() { // Python: multiple_accumulator.query(Statistic.SUM, key) // Rust: multiple_accumulator.query(Statistic::Sum, &key) - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new(); multi_acc.add_sum(key.clone(), 100.0); let multi_trait: Box = Box::new(multi_acc); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 5150ddf0e..6f33ca87b 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -20,7 +20,7 @@ mod tests { use tempfile::NamedTempFile; #[test] - fn test_read_streaming_config() { + fn flat_streaming_file_is_rejected() { // PR 5: `aggregationId: 1` is silently dropped on read — the // streaming-config map key is the policy fingerprint derived // from content. The legacy field stays in this fixture to @@ -47,9 +47,6 @@ aggregations: let mut streaming_temp_file = NamedTempFile::new().unwrap(); write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.materializations_by_policy_fingerprint.is_empty()); - let agg = config.materializations().values().next().expect("one agg"); - assert_eq!(agg.num_aggregates_to_retain, Some(6)); + assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index daa4d60f0..2afb897c1 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -957,7 +957,14 @@ async fn run_shared_dashboard(multi_pane: bool) { snapshot = serde_json::to_value(&typed).unwrap(); let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); - assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!(plan.precompute_plan.materializations.len(), 2); + let families = plan + .precompute_plan + .materializations + .iter() + .map(|m| m.aggregation_type.as_str()) + .collect::>(); + assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); assert!(plan .precompute_plan diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 1a291c496..b44c3ebed 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -37,7 +37,7 @@ //! the GET endpoint reflects the registered aggregation. //! * Test 2 — same shape with `group_by_labels: ["zone"]`; verifies //! #245's grouping plumb survives the round-trip into the backend's -//! `AggregationConfig.grouping_labels`. +//! `PrecomputeMaterialization.grouping_labels`. //! * Test 3 — full controller-to-query roundtrip: harness simulates //! the agent (builds DDSketch state with `asap_sketchlib`, encodes //! as a modified-OTLP `DdSketchDataPoint`), POSTs sketches to the @@ -45,7 +45,7 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use asap_types::AggregationConfig; +use asap_types::PrecomputeMaterialization; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -76,7 +76,7 @@ fn phase_aligned_now_ns() -> u64 { async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[AggregationConfig], + materializations: &[PrecomputeMaterialization], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the @@ -171,7 +171,7 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( @@ -641,7 +641,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { // Verifies #245's grouping plumb survives the controller → backend // round-trip. The workload carries `group_by_labels: ["zone"]`; the // emitted JSON must surface `["zone"]` in `labels.grouping`, the -// backend's parser must materialise it into `AggregationConfig. +// backend's parser must materialise it into `PrecomputeMaterialization. // grouping_labels`, and the active-config snapshot must reflect that. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1261,7 +1261,7 @@ async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &AggregationConfig) -> (u32, u32) { +fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { let w = agg.parameters["w"] .as_u64() .expect("materialization must carry parameters.w") as u32; diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md new file mode 100644 index 000000000..7504c948b --- /dev/null +++ b/docs/design_docs/precompute-dag-execution.md @@ -0,0 +1,24 @@ +# Precompute execution from post-ASAP IR + +Audience: backend developers and reviewers of issue #762. + +The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. + +```mermaid +flowchart LR + P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] + I --> R[Raw source → SummaryAgg streaming kernel] + I --> M[Maintenance dependency scheduler] + R --> S[Stored summary frontier] + S --> M + S --> Q[Query projection and readout] + M --> S +``` + +For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. + +`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. + +Catalog schema version 3 carries Planner family in SDS. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. + +The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats. From 0f058996ad83a7ba79a332e3ccc559b40c417041 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 17:08:46 +0000 Subject: [PATCH 09/25] fix: validate final SDS semantics across Planner adapters and recovery --- .../examples/calibration_candidates.rs | 3 +- control_plane/src/physical/compiler.rs | 239 ++++++------------ control_plane/src/query_plan.rs | 136 ++++++++-- control_plane/src/query_plan/residual.rs | 23 +- control_plane/tests/offline_evidence.rs | 35 ++- crates/asap_types/src/precompute_plan.rs | 46 ++-- data_plane/src/drivers/query/servers/http.rs | 6 +- .../precompute_engine/accumulator_factory.rs | 37 +-- .../precompute_engine/maintenance_runtime.rs | 13 +- data_plane/src/precompute_engine/raw_dag.rs | 17 +- data_plane/src/precompute_engine/worker.rs | 11 +- .../relational_adapter.rs | 16 +- .../asap_query_engine/logical_dag.rs | 10 +- .../storage_engines/sketch_db/index/mod.rs | 20 +- .../sketch_db/persistence/metadata.rs | 4 +- ...e2e_controller_plans_and_backend_serves.rs | 128 +++------- data_plane/tests/support/physical_fixture.rs | 9 + .../pr749-sds-foundation-2026-09-28/README.md | 6 +- 18 files changed, 374 insertions(+), 385 deletions(-) diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 5b3e3c36d..5c95608bc 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -72,7 +72,8 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation left, right, kind, - pred, pruning, + pred, + pruning, } => ( "RelationalJoin", vec![left, right], diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 77b74d16b..6e76b2fc1 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2416,9 +2416,49 @@ pub fn select_post_asap( delete: false, }, ); + // These fixtures exercise collector heap transport. Collector fixtures do + // not offer backend-only temporal value readouts as a physical capability. + struct CollectorFixtureModel(ControlPlaneCostModel); + impl asap_aware_mapping::CostModel for CollectorFixtureModel { + fn rank_candidates( + &self, + intent: &planner_types::pre_asap::AggIntent, + candidates: &[planner_types::post_asap::SketchAlgorithm], + ) -> Vec { + self.0.rank_candidates(intent, candidates) + } + fn size_params( + &self, + kind: planner_types::post_asap::SketchAlgorithm, + intent: &planner_types::pre_asap::AggIntent, + eps: f64, + delta: f64, + ) -> planner_types::post_asap::SketchParams { + self.0.size_params(kind, intent, eps, delta) + } + fn candidate_cost( + &self, + candidate: &asap_aware_mapping::ReplacementSubDAG, + target: &asap_aware_mapping::TargetSubDAG<'_>, + ) -> Option { + self.0.candidate_cost(candidate, target) + } + fn summary_support_evidence(&self, summary: &SummaryNode) -> Option { + if matches!( + summary.expr, + SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::Limit { .. }, + .. + } + ) { + return Some(false); + } + self.0.summary_support_evidence(summary) + } + } crate::planner_selection::select_query_with_models( expr, - &model, + &CollectorFixtureModel(model), &DefaultAccuracyModel, &QueryEvidence(evidence), ) @@ -3561,9 +3601,11 @@ fn collect_selected_materializations( } match &node.expr { SummaryExpr::RelationalJoin { - right: candidates, left: values, + right: candidates, + left: values, kind: planner_types::pre_asap::JoinKind::Semi, - pruning: Some(_), .. + pruning: Some(_), + .. } => { walk(candidates, readout, composable, grouping.clone(), selected)?; // In a hybrid TopK, the sketch is only a candidate-membership @@ -3660,12 +3702,6 @@ fn collect_selected_materializations( _ => return Err("unsupported TopK SummaryUpdate weight".into()), }; parameters["weight_mode"] = mode.into(); - if mode == "counter_delta" { - // CMS/CountSketch heap implementations quantize - // weights to integer counters. Preserve sub-unit - // counter increments used by CPU metrics. - parameters["weight_scale"] = 1_000_000.into(); - } } let (metric, window_secs, spatial_filter) = match selected_input_contract(node) { @@ -3821,7 +3857,7 @@ pub(crate) mod tests { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - snapshot.schema_version = 2; + snapshot.schema_version = 3; snapshot.data_workload.data_ingestion_interval.value = Some(DurationMs(1_000)); let template = snapshot.query_workload.repeating_queries.as_ref().unwrap()[0].clone(); let queries = [ @@ -4444,7 +4480,7 @@ pub(crate) mod tests { } #[test] - fn weighted_counter_topk_keeps_heap_membership_separate_from_exact_values() { + fn unsupported_rate_heap_uses_explicit_exact_route() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -4457,89 +4493,21 @@ pub(crate) mod tests { let plan = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .unwrap(); - let entry = plan.query_plan.entries.values().next().unwrap(); - let crate::query_plan::QueryPlanNode::Logical { - operator: asap_types::query_plan::residual::ResidualQueryOperator::TopKSelection { .. }, - inputs, - } = &entry.nodes[&entry.root] - else { - panic!("expected ordinary TopK root") - }; - let crate::query_plan::QueryPlanNode::MembershipFilter { inputs, .. } = - &entry.nodes[&inputs[0]] - else { - panic!("Planner weighted TopK must lower to MembershipFilter: {entry:#?}"); - }; - assert!(matches!( - entry.nodes[&inputs[0]], - crate::query_plan::QueryPlanNode::SummaryEstimate { - query: crate::query_plan::QueryReadout::TopK { .. }, - .. - } - )); - let candidate_read = match &entry.nodes[&inputs[0]] { - crate::query_plan::QueryPlanNode::SummaryEstimate { input, .. } => *input, - _ => unreachable!(), - }; - assert!(matches!( - entry.nodes[&candidate_read], - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - )); - assert!(!entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. }))); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactReadout { - readout: crate::query_plan::ExactReadout::Rate, - .. - } - ))); - let heaps = plan - .precompute_plan - .materializations - .iter() - .filter(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap - && materialization.parameters["weight_mode"] == "counter_delta" - }) - .collect::>(); - assert_eq!(heaps.len(), 1, "unpartitioned TopK owns one global CMS"); - assert!(heaps[0].grouping_labels.names().is_empty()); - assert_eq!(heaps[0].aggregated_labels.labels, vec!["job"]); - assert_eq!(heaps[0].parameters["weight_scale"], 1_000_000); - assert_eq!(retained_partition_count(heaps[0], Some(5)), 1); - let crate::query_plan::QueryPlanNode::ReadMaterialization { binding } = - &entry.nodes[&candidate_read] - else { - unreachable!() - }; + let entry = plan.query_plan.lookup(query).unwrap(); assert!(matches!( - binding.output_grouping, - crate::query_plan::PhysicalGrouping::Reduce(ref labels) if labels.is_empty() + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } )); - assert_eq!(binding.item_labels, vec!["job"]); - let counter = plan - .precompute_plan - .materializations - .iter() - .find(|materialization| { - matches!( - materialization.aggregation_type, - asap_types::AggregationType::Rate - ) - }) - .expect("reset-aware exact counter"); - assert_eq!(retained_partition_count(counter, Some(5)), 5); + assert_eq!( + entry.fallback, + crate::query_plan::FallbackPolicy::ExactBackend + ); + assert_eq!(entry.canonical_query, query); + assert!(plan.precompute_plan.materializations.is_empty()); } #[test] - fn hybrid_weighted_topk_installs_only_candidates_and_delegates_filtered_exact_values() { - use crate::query_plan::{ - residual::ResidualQueryOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, - }; + fn hybrid_rate_topk_preserves_the_original_exact_subquery() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -4557,82 +4525,31 @@ pub(crate) mod tests { let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); - - assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - asap_types::AggregationType::CountMinSketchWithHeap - ); let entry = plan.query_plan.lookup(query).unwrap(); - let QueryPlanNode::Logical { - operator: ResidualQueryOperator::TopKSelection { .. }, - inputs, - } = &entry.nodes[&entry.root] - else { - panic!("expected ordinary TopK root") - }; - let QueryPlanNode::MembershipFilter { inputs, .. } = &entry.nodes[&inputs[0]] else { - panic!("expected candidate TopK: {entry:#?}"); - }; + use crate::query_plan::{residual::ResidualQueryOperator, QueryPlanNode}; assert!(matches!( - &entry.nodes[&inputs[1]], - QueryPlanNode::ExternalExact { - request, - inputs: exact_inputs, - } if request.language == crate::query_plan::QueryLanguage::PromQl - && request.expression == "sum by (job) (rate(m[1m]))" - && request.output == ExternalExactOutput::InstantVector - && request.input_contracts == vec![ExternalExactInput::CandidateMembership { - item_label: "job".into(), - }] - && exact_inputs == &vec![inputs[0]] + &entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: 2, .. }, + .. + } )); - assert!(entry.nodes.values().all(|node| !matches!( - node, - QueryPlanNode::ExactReadout { .. } - | QueryPlanNode::Logical { - operator: ResidualQueryOperator::Scan { .. }, - .. - } - ))); - let installed = plan - .precompute_plan - .executable_dags - .get(&entry.query_id) - .expect("compiled query retains its maintenance projection"); - installed - .validate() - .expect("typed maintenance DAG document"); assert_eq!( - installed.document.schema_version, - asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION + entry + .nodes + .values() + .filter(|node| matches!(node, QueryPlanNode::Logical { + operator: ResidualQueryOperator::ExactSubquery { query }, .. + } if query == "sum by (job) (rate(m[1m]))")) + .count(), + 1 ); - assert!(installed - .document - .nodes - .iter() - .all(|node| node.output_state.timing - == planner_types::post_asap::ExecutionTiming::IngestionTime)); - assert_eq!(installed.binding.query_plan_sink, entry.root); - let mut mismatched = plan.to_publication_artifact().unwrap(); - let projected = mismatched - .precompute_plan - .executable_dags - .get_mut(&entry.query_id) - .unwrap(); - projected.binding.query_plan_sink = asap_types::executable_plan::QueryNodeId(u64::MAX); - assert!(mismatched.validate().is_err()); - assert!(installed.binding.nodes.values().any(|placement| matches!( - placement, - crate::physical::executable_binding::BackendNodeBinding::Materialization { .. } - ))); - let encoded = serde_json::to_value(installed).unwrap(); - let decoded: crate::physical::executable_binding::InstalledPostAsapDag = - serde_json::from_value(encoded).unwrap(); - assert_eq!(&decoded, installed); - decoded - .validate() - .expect("round-tripped typed DAG document"); + assert!(entry.materialization_bindings().is_empty()); + assert!(plan.precompute_plan.materializations.is_empty()); + let artifact = plan.to_publication_artifact().unwrap(); + artifact.validate().unwrap(); + let encoded = serde_json::to_value(&artifact).unwrap(); + assert!(!encoded.to_string().contains("counter_delta")); } #[test] @@ -4648,7 +4565,7 @@ pub(crate) mod tests { .compile_promql( request_with_evidence( "topk-rate", - "topk(2, sum by (job) (rate(m[1m])))", + "topk(2, count_over_time(m[1m]))", Some(evidence), ) .unwrap(), diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 84916c422..838966c05 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -365,7 +365,12 @@ where } SummaryExpr::ValueOperation { child: sort, - operation: planner_types::post_asap::ValueOperation::Limit { n, offset: 0, partition_by: limit_partition }, + operation: + planner_types::post_asap::ValueOperation::Limit { + n, + offset: 0, + partition_by: limit_partition, + }, timing: planner_types::post_asap::ExecutionTiming::QueryTime, } => { let SummaryExpr::ValueOperation { @@ -379,7 +384,9 @@ where )); }; if limit_partition != partition_by { - return Err(QueryPlanError::Invalid("TopK Limit and Sort grouping differ".into())); + return Err(QueryPlanError::Invalid( + "TopK Limit and Sort grouping differ".into(), + )); } if keys.len() != 1 || keys[0].ascending { return Err(QueryPlanError::Invalid( @@ -446,9 +453,11 @@ where reason: "unsupported post-ASAP value operation".into(), }, SummaryExpr::RelationalJoin { - right: candidates, left: values, + right: candidates, + left: values, kind: planner_types::pre_asap::JoinKind::Semi, - pruning: Some(completeness), pred, + pruning: Some(completeness), + pred, } if !self.preserve_relational => { validate_membership_join(pred, &values.schema, &candidates.schema)?; let candidate_input = self.lower(candidates)?; @@ -1410,35 +1419,122 @@ fn validate_membership_join( left: &planner_types::post_asap::SummarySchema, right: &planner_types::post_asap::SummarySchema, ) -> Result<(), QueryPlanError> { - use std::collections::BTreeSet; - use planner_types::pre_asap::{QueryExpr, CompareOpKind, DataType}; use planner_types::post_asap::SummaryFamilyType; - fn collect(expr: &QueryExpr, width: usize, keys: &mut Vec<(usize, usize)>) -> Result<(), QueryPlanError> { + use planner_types::pre_asap::{CompareOpKind, DataType, QueryExpr}; + use std::collections::BTreeSet; + fn collect( + expr: &QueryExpr, + width: usize, + keys: &mut Vec<(usize, usize)>, + ) -> Result<(), QueryPlanError> { match expr { - QueryExpr::BoolAnd(parts) => { for part in parts { collect(part, width, keys)?; } } - QueryExpr::Compare { left, op: CompareOpKind::Eq, right } => { - let (QueryExpr::Column(a), QueryExpr::Column(b)) = (left.as_ref(), right.as_ref()) else { - return Err(QueryPlanError::Invalid("membership join requires column equality".into())); + QueryExpr::BoolAnd(parts) => { + for part in parts { + collect(part, width, keys)?; + } + } + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => { + let (QueryExpr::Column(a), QueryExpr::Column(b)) = (left.as_ref(), right.as_ref()) + else { + return Err(QueryPlanError::Invalid( + "membership join requires column equality".into(), + )); }; - let (a,b) = if a < b { (*a,*b) } else { (*b,*a) }; - if a >= width || b < width { return Err(QueryPlanError::Invalid("membership join requires cross-input keys".into())); } - keys.push((a,b-width)); + let (a, b) = if a < b { (*a, *b) } else { (*b, *a) }; + if a >= width || b < width { + return Err(QueryPlanError::Invalid( + "membership join requires cross-input keys".into(), + )); + } + keys.push((a, b - width)); + } + _ => { + return Err(QueryPlanError::Invalid( + "unsupported membership join predicate".into(), + )) } - _ => return Err(QueryPlanError::Invalid("unsupported membership join predicate".into())), } Ok(()) } - let labels = |schema: &planner_types::post_asap::SummarySchema| schema.fields.iter().filter(|f| f.name != "__name__" && matches!(f.dtype, SummaryFamilyType::Plain(DataType::Utf8))).map(|f| f.name.clone()).collect::>(); + let labels = |schema: &planner_types::post_asap::SummarySchema| { + schema + .fields + .iter() + .filter(|f| { + f.name != "__name__" && matches!(f.dtype, SummaryFamilyType::Plain(DataType::Utf8)) + }) + .map(|f| f.name.clone()) + .collect::>() + }; let mut keys = Vec::new(); collect(&pred.0, left.fields.len(), &mut keys)?; let mut matched = BTreeSet::new(); - for (a,b) in keys { - let Some((a,b)) = left.fields.get(a).zip(right.fields.get(b)) else { return Err(QueryPlanError::Invalid("membership join key out of bounds".into())); }; - if a.name != b.name { return Err(QueryPlanError::Invalid("membership join requires matching label names".into())); } + for (a, b) in keys { + let Some((a, b)) = left.fields.get(a).zip(right.fields.get(b)) else { + return Err(QueryPlanError::Invalid( + "membership join key out of bounds".into(), + )); + }; + if a.name != b.name { + return Err(QueryPlanError::Invalid( + "membership join requires matching label names".into(), + )); + } matched.insert(a.name.clone()); } if matched.is_empty() || matched != labels(left) || matched != labels(right) { - return Err(QueryPlanError::Invalid("membership join must match the complete label identity".into())); + return Err(QueryPlanError::Invalid( + "membership join must match the complete label identity".into(), + )); } Ok(()) } + +#[cfg(test)] +mod membership_binding_tests { + use super::*; + use planner_types::{ + post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, + pre_asap::{CompareOpKind, DataType, Predicate, QueryExpr}, + }; + + // This adapter can implement full-label equality only; reject narrower joins. + #[test] + fn membership_binding_rejects_other_join_semantics() { + let schema = |names: &[&str]| SummarySchema { + fields: names + .iter() + .map(|name| SummaryField { + name: (*name).into(), + dtype: SummaryFamilyType::Plain(DataType::Utf8), + nullable: false, + }) + .collect(), + time_index: None, + }; + let eq = |a, b| QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(a)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(b)), + }; + let one = schema(&["job"]); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 1))), &one, &one).is_ok()); + assert!( + validate_membership_join(&Predicate(Rc::new(eq(0, 1))), &one, &schema(&["host"])) + .is_err() + ); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 2))), &one, &one).is_err()); + assert!(validate_membership_join( + &Predicate(Rc::new(QueryExpr::BoolAnd(vec![]))), + &one, + &one + ) + .is_err()); + let two = schema(&["job", "host"]); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 2))), &two, &two).is_err()); + } +} diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index e8e1df437..3c7e51f05 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -361,11 +361,32 @@ pub(super) fn residual_nodes( horizons(residual, &mut intervals); intervals.sort_unstable(); intervals.dedup(); + // Accuracy annotations select a candidate, but exact execution still + // implements that candidate's computation. Reconstruct the same typed IR + // before comparing it; do not erase operators or source predicates. + let accuracy = match residual { + planner_types::pre_asap::QueryExpr::Aggregate { measures, .. } => measures + .iter() + .find_map(|intent| { + use planner_types::pre_asap::AggIntent; + match intent { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } + | AggIntent::FrequencyL2 { accuracy, .. } + | AggIntent::FrequencyEntropy { accuracy, .. } => Some(accuracy.clone()), + _ => None, + } + }) + .unwrap_or(planner_types::types::AccuracyTarget::Exact), + _ => planner_types::types::AccuracyTarget::Exact, + }; for expression in expressions { for interval in &intervals { if let Ok(candidate) = crate::query_parser::parse_query_expr_with_interval( &expression.to_string(), - planner_types::types::AccuracyTarget::Exact, + accuracy.clone(), *interval, ) { if &candidate == residual { diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index a07a9a163..ded77eed3 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -274,10 +274,25 @@ fn sketch(node: &SummaryNode) -> (&SketchAlgorithm, &SketchParams) { } } -/// The actual control-plane parser/binder selects the lower measured update -/// cost while preserving the selected algorithm's normal parameter sizing. +fn assert_exact_count(node: &SummaryNode) { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => assert_exact_count(summary_input), + SummaryExpr::SummaryAgg { + family: + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + planner_types::post_asap::ExactParams::Count, + ), + .. + } => {} + other => panic!("expected exact total-count state, got {other:?}"), + } +} + +/// Update evidence ranks frequency sketches but does not replace an exact +/// total-count accumulator with a point-frequency sketch. #[test] -fn offline_update_evidence_changes_typed_binding() { +fn offline_update_evidence_preserves_exact_count_binding() { let default = model(); let (artifact, context) = fixture(&default, &intent()); let empirical = @@ -292,12 +307,8 @@ fn offline_update_evidence_changes_typed_binding() { ); let default_bound = bound(&default); let measured_bound = bound(&empirical); - assert_eq!(sketch(&default_bound).0, &SketchAlgorithm::Cms); - assert_eq!(sketch(&measured_bound).0, &SketchAlgorithm::CountSketch); - assert_eq!( - sketch(&measured_bound).1, - &default.size_params(SketchAlgorithm::CountSketch, &intent(), 0.01, 0.01) - ); + assert_exact_count(&default_bound); + assert_exact_count(&measured_bound); assert!(default_bound.guarantee.is_some()); assert!(measured_bound.guarantee.is_some()); // Observed zero point-frequency error has no effect on formal sizing. @@ -334,11 +345,7 @@ fn incompatible_evidence_preserves_deployment_behavior() { candidates(), "{scenario}" ); - assert_eq!( - sketch(&bound(&empirical)).0, - &SketchAlgorithm::Cms, - "{scenario}" - ); + assert_exact_count(&bound(&empirical)); } } diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index de0641754..ded2fd6f4 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -660,28 +660,27 @@ impl PrecomputePlan { operation: ValueOperation::FinalizeExactAccumulator, } if children.len() == 1 && frontiers.contains_key(&children[0].producer) => {} - Payload::Binary { - operator, - } if children.len() == 2 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Left - }) - .count() - == 1 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Right - }) - .count() - == 1 - && operator.vector_match.is_none() - && matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) => + Payload::Binary { operator } + if children.len() == 2 + && children + .iter() + .filter(|edge| { + edge.role == planner_types::post_asap::EdgeRole::Left + }) + .count() + == 1 + && children + .iter() + .filter(|edge| { + edge.role == planner_types::post_asap::EdgeRole::Right + }) + .count() + == 1 + && operator.vector_match.is_none() + && matches!( + operator.kind, + planner_types::pre_asap::BinaryOpKind::Arithmetic(_) + ) => { pending.extend(children.iter().map(|edge| edge.producer)); } @@ -1099,8 +1098,7 @@ mod source_window_cohort_tests { assert!(validate_maintenance_reduction(&config, &node).is_err()); config.partitioning = None; assert!(validate_maintenance_reduction(&config, &node).is_err()); - node.payload = ExecutableOperatorPayload::SummaryMerge { - }; + node.payload = ExecutableOperatorPayload::SummaryMerge; assert!(validate_maintenance_reduction(&config, &node).is_err()); } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 2eba6811a..8c03df23f 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -6653,7 +6653,11 @@ mod catalog_install_tests { }) .expect("demo has maintained summaries"); binding.window_ms += 1; - assert!(install(request).unwrap_err().contains("query pane differs")); + let error = install(request).unwrap_err(); + assert!( + error.contains("query") && error.contains("pane") && error.contains("differs"), + "{error}" + ); } #[test] diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 9f94c8c93..3346f6059 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1682,24 +1682,14 @@ mod tests { #[test] fn counter_delta_scale_preserves_sub_unit_membership_weights() { - use planner_types::post_asap::{ - EntityIdentity, NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, - }; - let config = topk_config(AggregationType::CountMinSketchWithHeap, None); - let family = config.accumulator_spec().unwrap().family; - let input = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::ResetAwareCounterDelta { - value: planner_types::pre_asap::ColumnRef::SampleValue, - series: EntityIdentity::PromqlLabelSet { excluding: vec![] }, - }, - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - }, - }; - let mut updater = create_planner_accumulator(&family, &input, &Default::default()).unwrap(); + let mut config = topk_config( + AggregationType::CountMinSketchWithHeap, + Some("counter_delta"), + ); + config + .parameters + .insert("weight_scale".into(), serde_json::json!(1_000_000)); + let mut updater = create_fixture_accumulator(&config); updater.update_keyed(&host_key("payment"), 0.004, 1_000); updater.update_keyed(&host_key("order"), 0.002, 1_000); let ranked = ranked_topk(&*updater.take_accumulator()); @@ -1842,16 +1832,7 @@ pub fn create_planner_accumulator( if family_grouping != grouping { return Err("Planner family and operator grouping disagree".into()); } - // Heap counters use fixed-point storage for fractional counter deltas. - // This encodes the selected update; it does not choose another family. - let weight_scale = if matches!( - input.weight, - planner_types::post_asap::SummaryInputExpr::ResetAwareCounterDelta { .. } - ) { - 1_000_000.0 - } else { - 1.0 - }; + let weight_scale = 1.0; let updater: Box = match (kind.algorithm(), kind.params()) { (SketchAlgorithm::Kll, SketchParams::Kll { k }) => Box::new(KllAccumulatorUpdater::new( u16::try_from(*k).map_err(|_| "KLL k exceeds runtime bound")?, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index d0a7a1394..683ed5c06 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -170,11 +170,8 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { return Err("maintenance runtime requires ingestion-time nodes".into()); } match &node.payload { - ExecutableOperatorPayload::SummaryMerge { - } => merge_inputs(inputs), - ExecutableOperatorPayload::Binary { - operator, - } => { + ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs), + ExecutableOperatorPayload::Binary { operator } => { if !self.inputs.frozen_inputs().is_some() || node.output_state != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS @@ -2246,8 +2243,7 @@ mod tests { fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), - payload: ExecutableOperatorPayload::SummaryMerge { - }, + payload: ExecutableOperatorPayload::SummaryMerge, output_state: planner_types::post_asap::ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: vec![], @@ -2904,8 +2900,7 @@ mod tests { second_node.id = PostAsapNodeId(5); let mut merge = second_node.clone(); merge.id = PostAsapNodeId(6); - merge.payload = ExecutableOperatorPayload::SummaryMerge { - }; + merge.payload = ExecutableOperatorPayload::SummaryMerge; dag.nodes.extend([second_node, merge]); let original = dag .edges diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs index ee76c8b49..75bdad529 100644 --- a/data_plane/src/precompute_engine/raw_dag.rs +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -31,7 +31,7 @@ impl RawDagProgram { installed.validate()?; let dag = installed.document.decode()?; for node in &dag.nodes { - if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { summary_definition }) if summary_definition.fingerprint() == config.policy_fingerprint()) + if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { stored_output }) if stored_output.fingerprint() == config.policy_fingerprint()) { continue; } @@ -150,10 +150,6 @@ impl RawDagProgram { (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { *value == 1.0 } - ( - SummaryInputExpr::ResetAwareCounterDelta { .. }, - asap_types::SampleUpdateRule::CounterDelta { scale }, - ) => scale == 1_000_000.0, _ => { asap_types::accumulator_spec::is_unit_sample_frequency(input) || (matches!( @@ -209,10 +205,6 @@ impl RawDagProgram { SummaryInputExpr::Column( ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }, ) if self.projected_column.as_ref() == Some(name) => {} - SummaryInputExpr::ResetAwareCounterDelta { - value: ColumnRef::SampleValue, - series: planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, - } if excluding.is_empty() => {} _ => return Err("raw DAG weight expression is unsupported".into()), } fn item(expr: &SummaryInputExpr) -> bool { @@ -232,10 +224,7 @@ impl RawDagProgram { } pub fn uses_counter_delta(&self) -> bool { - matches!( - self.input.weight, - SummaryInputExpr::ResetAwareCounterDelta { .. } - ) + false } pub fn apply( @@ -248,7 +237,7 @@ impl RawDagProgram { let weight = match &self.input.weight { SummaryInputExpr::Constant(c) => *c, // The worker retains one previous value per series across pane rotation. - SummaryInputExpr::Column(_) | SummaryInputExpr::ResetAwareCounterDelta { .. } => value, + SummaryInputExpr::Column(_) => value, _ => return Err("unsupported raw weight expression".into()), }; let scalar_frequency = asap_types::accumulator_spec::is_unit_sample_frequency(&self.input) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index fe4e74e05..847245e13 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -4446,9 +4446,14 @@ mod dag_execution_tests { &dag, ) .unwrap(); - assert!(StreamingConfig::from_precompute_plan(plan) + installed.document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; + let error = StreamingConfig::from_precompute_plan(plan) .unwrap_err() - .to_string() - .contains("update")); + .to_string(); + assert!( + error.contains("update") || error.contains("semantic definition differs"), + "{error}" + ); } } diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index 27abba680..8d673fff3 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -484,9 +484,15 @@ impl ClickHouseRelationalAdapter { .rows .sort_by(|left, right| compare_sort_keys(left, right, keys, &schema)); } - ValueOperation::Limit { n, offset, partition_by } => { + ValueOperation::Limit { + n, + offset, + partition_by, + } => { if !partition_by.keys().is_empty() || partition_by.is_without() { - return Err(ClickHouseRelationalError::Unsupported("partitioned Limit".into())); + return Err(ClickHouseRelationalError::Unsupported( + "partitioned Limit".into(), + )); } input.rows = input.rows.into_iter().skip(*offset).take(*n).collect(); } @@ -1607,7 +1613,11 @@ mod tests { }], partition_by: GroupKeys::none(), }, - ValueOperation::Limit { n: 1, offset: 0, partition_by: planner_types::pre_asap::GroupKeys::by(vec![]) }, + ValueOperation::Limit { + n: 1, + offset: 0, + partition_by: planner_types::pre_asap::GroupKeys::by(vec![]), + }, ] { relation = adapter .apply_operation(&operation, &projected_schema, relation) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 2bf916356..31ef9f7c8 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -430,10 +430,16 @@ fn membership_filter( labels.remove("__name__"); labels }; - let candidate_ids: BTreeSet<_> = candidates.iter().map(|(labels, _)| identity(labels)).collect(); + let candidate_ids: BTreeSet<_> = candidates + .iter() + .map(|(labels, _)| identity(labels)) + .collect(); let value_ids: BTreeSet<_> = values.iter().map(|(labels, _)| identity(labels)).collect(); let missing: BTreeSet<_> = candidate_ids.difference(&value_ids).collect(); - let selected = values.into_iter().filter(|(labels, _)| candidate_ids.contains(&identity(labels))).collect(); + let selected = values + .into_iter() + .filter(|(labels, _)| candidate_ids.contains(&identity(labels))) + .collect(); if !missing.is_empty() && matches!(completeness, CandidateCompleteness::Certified { .. }) { return Err(miss("certified membership key has no authoritative value")); } diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index c3a0283ed..becf81bb2 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -5162,8 +5162,8 @@ mod tests { .is_some(), "live version change must allocate a fresh physical series" ); - let output = asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) - .stored_output_id; + let output = + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()).stored_output_id; let inventory = store .observed_summary_inventory( "backend-a", @@ -5245,7 +5245,6 @@ mod tests { || !persistence.manifest.live_parts().is_empty(), Duration::from_secs(5) )); - let old_parts = persistence.manifest.live_parts().len(); store.remove_instance(old_sid).unwrap(); assert!(resolver .resolve_with_reactivation("metric", "group", "family", |sid| store @@ -5270,7 +5269,20 @@ mod tests { ); } assert!(wait_until( - || persistence.manifest.live_parts().len() > old_parts, + || { + // Old-series epochs may still publish after reactivation. Wait + // for this series, not an unrelated increase in part count. + persistence.manifest.live_parts().iter().any(|part| { + let path = + persistence::part::part_dir_path(&persistence.parts_root, part.part_id); + persistence::part::PartReader::open(&path).is_ok_and(|reader| { + reader + .index_records() + .iter() + .any(|row| row.agg_id == new_sid && row.start_ts < 90_000) + }) + }) + }, Duration::from_secs(5) )); persistence.shutdown(); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 9360d4eb1..96a86d8d5 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -715,7 +715,7 @@ impl SidMetadataStore { }; if matches!( value.get("schema_version").and_then(|v| v.as_u64()), - Some(2 | 3 | 4) + Some(2..=4) ) { let sidecar: SdsSidecar = match serde_json::from_value(value) { Ok(sidecar) => sidecar, @@ -807,7 +807,7 @@ impl SidMetadataStore { let value: serde_json::Value = serde_json::from_slice(&bytes) .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; if let Some(version) = value.get("schema_version") { - if !matches!(version.as_u64(), Some(2 | 3 | 4)) { + if !matches!(version.as_u64(), Some(2..=4)) { return Err(PersistError::Format( "unsupported SID metadata version".into(), )); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index b44c3ebed..15a5d3e0d 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -14,11 +14,10 @@ //! projected into a physical-plan artifact with QueryPlan/SummaryCatalog //! bindings, then staged and activated before ingest. //! -//! Planner owns the summary choice. These tests declare an accuracy target and -//! build their payloads from whichever family and parameters it committed to — -//! `materializations[0].aggregation_type` and `.parameters` — rather than -//! pinning a family. Family selection itself is covered by the control-plane -//! compiler tests. +//! Quantile fixtures use Planner-selected materializations. CMS wire fixtures +//! explicitly declare the imported payload family; they do not assert that a +//! total-count query selects CMS. HLL and CountSketch production oracle tests +//! live in `all_sketches_process_oracle_e2e`. //! //! Queries are registered with the grouping the producer's attribute set //! carries (`sum by (service) (...)`), because the population key the backend @@ -208,6 +207,19 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec Vec { + vec![physical_fixture::materialization( + metric, + asap_types::AggregationType::CountMinSketch, + std::collections::HashMap::from([ + ("w".into(), serde_json::json!(512)), + ("d".into(), serde_json::json!(5)), + ]), + )] +} + /// Epsilon-delta accuracy target in the shape `QueryRequirements` expects. fn epsilon_delta(epsilon: f64, delta: f64) -> JsonValue { serde_json::json!({ "explicit": { "EpsilonDelta": { "epsilon": epsilon, "delta": delta } } }) @@ -878,8 +890,6 @@ async fn controller_plan_to_query_full_roundtrip_kll() { "sum by (service) (quantile_over_time(0.5, request_size_bytes[1s]))", epsilon_delta(0.05, 0.05), ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. post_full_config(&client, &stack, &materializations).await; let alpha = materializations[0].parameters["alpha"] @@ -936,42 +946,16 @@ async fn controller_plan_to_query_full_roundtrip_kll() { ); } -// ── Test 5 — full roundtrip with HLL (cardinality) ────────────────────────── -// -// HLL backs the cardinality readout. The workload pins HLL via -// `sketch_type_override: Some(SketchType::HLL)`. The OTLP DP carries -// a `HllSketchDataPoint` with `HyperLogLogState`. PromQL's -// `count(metric)` is the spec's distinct-counting idiom — returns -// the number of distinct label sets in the result vector — which -// the analyzer routes to `Capability::CardinalityApprox` and the -// reducer dispatches to the HLL cardinality readout. -// -// Closed by a chain of fixes: -// * `count(metric)` analyzer fix (PR #255) -// * `count` reducer alias (PR #255) -// * Vector-vs-Matrix instant-query response shape fix (this PR) +// An imported CMS state remains readable through its installed count binding. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_hll() { +async fn imported_cms_state_serves_count_query() { let stack = start_full_stack(19_565, 19_566).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "sum by (service) (count_over_time(unique_users_per_min[1s]))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("unique_users_per_min"); post_full_config(&client, &stack, &materializations).await; - // Precision must match what the controller plans for this - // workload (`HLLDefaults` in `control_plane::types`). The - // accuracy_sla=0.05 above is > the precision_threshold (0.02), - // so the planner picks `precision_coarse = 10`. If the OTLP DP - // were sent with a different precision, the backend would - // register two separate sids for the same metric — one with - // policy_fp=UNSET (no matching policy params) — and the query - // wouldn't find the policy-tagged one. let (w, d) = extract_w_d(&materializations[0]); let cells = (w as usize) * (d as usize); let mut counts = vec![0i64; cells]; @@ -1031,42 +1015,19 @@ async fn controller_plan_to_query_full_roundtrip_hll() { assert_eq!( status, "success", - "HLL cardinality query did not succeed:\n{}", + "Imported CMS count query did not succeed:\n{}", serde_json::to_string_pretty(&response).unwrap_or_default() ); } -// ── Test 6 — wire-format roundtrip with CountSketch (frequency) ───────────── -// -// CountSketch backs FREQUENCY estimation — signed-counter matrix -// producing approximate point-frequency answers. `top_endpoint_qps` -// is the canonical TopK metric, so the planner pins -// `with_heap: true` and the controller emits `CountSketchWithHeap` -// (regardless of override). To match, the wire DP carries a -// msgpack-encoded heap envelope, but the query -// uses `count_over_time(...)` instead of `topk(...)` — the -// reducer's `decode_frequency_total` reads row-0 of the underlying -// matrix for heap-bearing variants too, so FrequencyEstimate works -// on a heap-bearing SID. -// -// **Strict-success: `count_over_time(top_endpoint_qps[1s])`** binds -// to `Capability::FrequencyEstimate(Any)`, which -// `is_satisfied_by` accepts against -// `FrequencyTopk(CountSketchWithHeap)` (heap is additional info -// layered over the matrix — the matrix is a fully valid frequency -// sketch on its own). +// CMS transport binds the configured dimensions for the endpoint metric. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_count_sketch() { +async fn imported_cms_top_endpoint_wire_roundtrip() { let stack = start_full_stack(19_567, 19_568).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(top_endpoint_qps[1s])))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("top_endpoint_qps"); post_full_config(&client, &stack, &materializations).await; // Use the planner-picked `(w, d)` so the OTLP DP's wire-level @@ -1158,16 +1119,11 @@ async fn controller_plan_to_query_full_roundtrip_count_sketch() { // matrix and returns the per-window total count. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { +async fn imported_cms_frequency_wire_roundtrip() { let stack = start_full_stack(19_569, 19_570).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(endpoint_request_freq[1s])))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("endpoint_request_freq"); post_full_config(&client, &stack, &materializations).await; // Use planner-picked `(w, d)` so the wire DP's `rows`/`cols` @@ -1292,14 +1248,11 @@ fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { // (Prometheus spec for range queries). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_range_query_count_over_time_cms() { +async fn imported_cms_state_serves_range_query() { let stack = start_full_stack(19_575, 19_576).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(endpoint_request_freq[1s])))", - epsilon_delta(0.05, 0.05), - ); + let materializations = imported_cms_materializations("endpoint_request_freq"); post_full_config(&client, &stack, &materializations).await; let (w, d) = extract_w_d(&materializations[0]); @@ -1900,34 +1853,15 @@ async fn live_serve_actually_answers_ddsketch_quantile() { ); } -// ── Test — the live serving cutover MERGES the global-merge shape ───────── -// correctly, end to end (ASAPController#163/#165) -// -// `count(hll_metric)` with NO `by (...)` and MULTIPLE distinct-service HLL -// sids used to be the ambiguous shape the design doc's "Grouping -// semantics" section described: `SummaryAgg{by: []}` couldn't tell "no -// grouping concept" from "reduce everything," so `live_serve.rs`'s -// `ambiguous_merge_risk` gate DECLINED to serve it from the new path and -// fell back to the legacy `evaluate_cardinality_global` special case. -// -// `Reduction` (ASAPController#165) resolves that: `count(...)` is a -// genuine aggregation operator, so it lowers to `Reduce([])` and -// `resolve_group_key` gives both sids the same group key -- the new path -// merges them itself. The gate is gone; this SHOULD exercise the new -// path serving the shape directly, not a fallback. -// -// The installed cardinality readout merges all bound series and windows. +// Live serving keeps the two imported CMS series available. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn live_serve_hll_global_count_merges_across_sids() { +async fn live_serve_cms_reads_independent_series() { let _live = LiveServeEnvGuard::enable(); let stack = start_full_stack(19_595, 19_596).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "sum by (service) (count_over_time(unique_users_per_min[1s]))", - epsilon_delta(0.05, 0.05), - ); + let materializations = imported_cms_materializations("unique_users_per_min"); post_full_config(&client, &stack, &materializations).await; let (w, d) = extract_w_d(&materializations[0]); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index f4858dc03..609176084 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -38,6 +38,15 @@ pub fn artifact_from_materializations( // identities and bindings. for config in &mut configs { config.pane_origin_ms.get_or_insert(0); + // These fixtures import synthetic states and replace the producer's + // window layout. They do not install the original Planner DAG, so + // describe the supplied source/configuration instead of claiming its + // persisted-output closure. Derived inputs require the real DAG. + assert!( + config.derived_input.is_none(), + "use the Planner plan for derived inputs" + ); + config.semantic_fragment = None; } let catalog = control_plane::physical::summary_catalog::SummaryCatalog::from_materializations( 1, 1, &configs, diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md index f3d8413f4..3f3705539 100644 --- a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md +++ b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md @@ -30,7 +30,11 @@ On the corrected #749 code: The data-plane suite and process test were rerun after the final allocation fix. No production-cost or independent human approval claim is made. -## Scope +## Historical scope + +This report records the earlier foundation validation. The final SDS identity +contract and its new test results supersede the scope below; see +[final identity validation](../pr749-final-sds-2026-09-28/README.md). This is the plan/schema foundation described in #737's staged migration, not the completed SDS implementation. Its policy-fingerprint-based schema remains From c556130e90c762b257a942c821aa8bff491497fe Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 17:11:49 +0000 Subject: [PATCH 10/25] test: verify final SDS identity installation, serving and restart --- ...e2e_controller_plans_and_backend_serves.rs | 16 ++++- .../pr749-final-sds-2026-09-28/README.md | 55 ++++++++++++++++++ .../pr749-final-sds-2026-09-28/SHA256SUMS | 6 ++ .../pr749-final-sds-2026-09-28/clippy.log.gz | Bin 0 -> 207 bytes .../identity-integration-before-fix.log.gz | Bin 0 -> 11720 bytes .../libraries.log.gz | Bin 0 -> 30489 bytes .../offline-evidence.log.gz | Bin 0 -> 494 bytes .../restart-process.log.gz | Bin 0 -> 404 bytes .../pr749-final-sds-2026-09-28/serving.log.gz | Bin 0 -> 542 bytes 9 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/README.md create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/libraries.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/offline-evidence.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 15a5d3e0d..8c078ec49 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -210,13 +210,25 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec Vec { - vec![physical_fixture::materialization( - metric, + vec![PrecomputeMaterialization::new( asap_types::AggregationType::CountMinSketch, + String::new(), std::collections::HashMap::from([ ("w".into(), serde_json::json!(512)), ("d".into(), serde_json::json!(5)), ]), + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 5, + 5, + asap_types::enums::WindowKind::Tumbling, + String::new(), + metric.into(), + Some(12), + None, + None, )] } diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/README.md b/docs/evaluation/pr749-final-sds-2026-09-28/README.md new file mode 100644 index 000000000..8d215597a --- /dev/null +++ b/docs/evaluation/pr749-final-sds-2026-09-28/README.md @@ -0,0 +1,55 @@ +# #749 final SDS contract validation + +Audience: implementation reviewers. This supersedes the earlier +[foundation report](../pr749-sds-foundation-2026-09-28/README.md). + +## Contract established in this PR + +#749 consumes Planner's canonical semantic export and binds it to an explicit +logical dataset identity. Catalog schema 6 separates semantic definitions from +deployed outputs; planning snapshot schema 3 requires dataset identity. +Hot and rebuild outputs can share meaning without sharing read authorization. +The typed Count/Rate support previously staged in #771 is included here because +collapsing those families produces conflicting semantic identities. + +An installed query resolves only its selected output within its plan version, +then checks definition, revision, format and coverage. Recovery accepts the same +installed generation; a new version must populate fresh state before serving it. +Ad-hoc semantic discovery and cross-version state adoption are not implemented. +Restricted native configuration helpers remain for explicit imported-state +fixtures; they do not establish a multi-dataset Planner binding. + +## Problems found before the fixes + +- Policy fingerprints coupled semantic identity to deployment routing and could + not express separate hot/rebuild outputs sharing one definition. +- Metric/table names alone could not distinguish equal expressions over + different logical datasets. +- Importing semantic definitions without typed Count/Rate support failed the + workload tests: one output claimed different definitions. +- A Planner API adapter could discard a join pruning contract. Unsupported + pruned relational joins now take the explicit fallback path; the vector + adapter validates the complete label-equality predicate. +- Old tests assumed Planner always selected a heap. Collector fixtures now + advertise the intended capabilities; exact backend candidates remain legal. +- A persistence test waited for any new disk part, which could belong to the + old series. It now waits for the newly allocated series' actual record. + +## Validation + +Passed locally on the final implementation: + +- Type, control-plane and data-plane libraries: **117 + 431 + 1,164 tests**. +- Installed-plan serving/transport integration: **13 tests**. +- Offline evidence integration: **6 tests**. +- Production-process restart/new-version warm-up: **1 test**. +- All-target strict Clippy for all three packages and workspace formatting. + +Compressed logs are verified by `SHA256SUMS`. The before-fix log records the +identity collisions exposed while extracting the contract ahead of typed +Count/Rate support. +The production-process test covers same-version restart without re-ingestion, +new-version cold state, and fresh input becoming queryable in that version. +No production workload, production-cost, or independent human approval claim is +made. #728/#742/#775 retain their structural, synthetic-selection, and deployment +execution acceptance roles. diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS b/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS new file mode 100644 index 000000000..350b890c2 --- /dev/null +++ b/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS @@ -0,0 +1,6 @@ +46e0a7b5a3c3282188347874fb2f16640eb6ca2d35ed9855d6ad03f0abd7cbb6 clippy.log.gz +d15dc24dab9b42725dcccb3c27cbb84c91be5e45c5cf4015d9b1492b3ec2150d identity-integration-before-fix.log.gz +539c519be4bd3b50ae7fab2e0bad8c7a349bdde64d541ae79244f4110141c376 libraries.log.gz +f6756607f21bb4fcd7688059d4b55d3ecd36f88968757c30a9374f0b5d8e6735 offline-evidence.log.gz +235768d931779103e1f4a6c9a998a236708d09650df196d582dbb41f8cef8fb3 restart-process.log.gz +89acece7344c0ff746e93e7178771e5770381f9a6b88b295a2a7f2a99fc1b437 serving.log.gz diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..9037562b0c3431fe4970e66669ef272f6c065090 GIT binary patch literal 207 zcmV;=05Ja_iwFP!00002|E-TPZvrt4g?E0%8)3K$sHlh$kUAH2tf~?&KCmtqM@|lO z{Cb))RNa|8`1$*u0buK;={THlwVcarMbWY+lm#TWT6+&A|z J)x0|a000NQYCQk| literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..96ceb7e1df76a54f88b920838620f771c7741347 GIT binary patch literal 11720 zcmV;(EjQ91iwFP!00002|J8llZsf?7;Jdzp(=Ss6OjSv0)lCgQ^m;tNU=}+b+xuh~ zEG8M5$rh8rW{{Gy2K(=OZgC-#nG8~DV{E{7NyyGy*@N$Ro1&CGg*^% z$3~gt@%sJE`|IS*hr_YRy6nTxfByN8|JAGZ_->Qs52`LcynoW|L)R*0J~Zv;yRYxI zWt~;;OkqCcZPvlBJ-D}5Nw`j~>#o&R+EiJsUbs{UYP1YOdyO?b99kgxx3VtHUKPnV zIA!-OX=1xS zOq+kG%FX-M_#MK+?@R}qd&hu26l&A&;D4JCvG1}Uv(KBmPuHJsKi_`&oNwP=U9|%L z=H@OT_*eM9NtWmRp|7$|x2b7Vet%DYyuZ(NUw7%DtW)!#x_qBLmEAtwD_G-@H74zJ znjcK?+a_zXgGt`Mf1l`wDBuDZj$zUg{;Qwr^r5Vav@BHJmECa(uwbX*{+8@&{D-vA z%A~dK(nhsi-)_=IS7m-ofe3c0ZQ8Q#;vnPQ@#X;FLsg}%$_n@wZ!ZppYFqeKQ5vAR zBb&d{ie|kex%ej-ECHtS>)C3k42ji=1V}k zI;sMHc+n0#$OZcdU>*Ec;D@-co9uvJ?i(N|m9{eoK2Ovude@3BDDa|2Mz zE)73Z0`IQVV|Ji-R_kuSn?=oSjG)sKJ8a?G zt5nCogex+zr-h&$8v$}Opj@WrSm%TjxJW#x!dn1-Gl`EKAhO|G*SICZ39|m&;sbC{ z0?EL4`Sk!xtWe`6xvb$v%szwXm=Yuq#_(lS2$wCwCu9)(2aBN^hO$970g%#@lpzulJN(P7wfNhZAvbIzaiv?lB9iUd6Hvf4Dd9ZJws3(1=sg7Hqy z2-`oFsshWy|K*%z6$Mtr^XmdN4WJA;T3O-yd|lvz@7Dp!h9Qc?jAYDSJ+qcNp(k7b zRHNQ6w4%t4X4c`Y;S9E!uiB)uPS8dd=Jpz3_eVqgcu2n`ng1PXO(21~GlF$mWwLgx z<&uZepf%yNrb0C!!hwC1R!s*b(dfR-$zIsPUON-L|J4fm^yZB^WHsu!22}8@>rtm5 zT^tczFk|@@{ah$NH;Tn3AdIwHQU_yq4hE?bZeppJezoEfM-R;j+44K{tfD1eGy@sP z7F7K97`}@Rm}|$XIo=iujnw@bDB?s{h$9K3sxAD$PZoC$Ht!*8k5Sv&w!lysab}<+ zS6V;xO%x0!XQX=z2(^c-F=RoZ{KdnK_R}g*)D1_g>7vi!HU}zk3pzSzh1OUqQH45`6>Aa)B+z`X@`sT+HGL@+YE7`g zN@YkLsCa<7yNAzNg9hvWGo)l7r{EqCmDaNzt zDnh#il=xnhBh>lvS(7^ucjH8P=qnS$W3W2Ee_q&GCMPxve`?ZlIS;LGS#m0TSfO`S zHDq_@71<$hyz)D2k7N{S?eNQLWM#tBpt}Q{sAmUj(*t<`EKng>=&wiY78ERUx8N%B zCx4|Q&4^k3idbewpFS4s}9IZ$B%Iu zG}w!LC^+n)>BO_VWOL4G{S~9+j$2W2xuaGTicz& zt--A#4-)A-0B8zx!n${=25v!*!jhvk@=A0`pE~gf3ThEUSB0~=`6oo+uF{(!r`!We z@`7fPY6hER7en}B)4&B}S5}S2`#~=XTSR58vYk2$c3Yx#*&PE{2CFB;yqrQtuHF$R z9HP+5f*O8-Hk$V|TfSXc&j>T(*%PxJAT}<*aHc452~8Kv-lBO(;w($uVM`Z`jEP$g}rz7FggvOs!6ALbHHQI7?ja()j6l zvEHI*t3?$f&)S;SIsbra^%U|S_Zoo`jD0s2w`o>j%y?wMhtPz`5W9fwK6JGZqL-Zz z%zKo`aJvm^OvM?cW*$@1YU1Q(SngZ)tF{@4k;n~tnzUFHFtZ!vpXpCvWq*E&Y@D##7!K;fKnVm4EY$5wJ2O- zyuzLR(xWUTkug{Zl!t2M0;|yzLh+=bX?Z~aGxt|s#S0z+B{^`=m*hdeaZWn*Eb_yj zD(X`0b$VKGf{%zQeQL=m%>IlhkbB*UdbsJ!s$d-iO~D}ELe%i2S?3nIHE|n8ag2`| zq*F_A*FXzE#*JJtOOe%C5Z>7twSx0X!$K8n$y?)TmmKcl+nf-{=RiBJNEmuDHY$=a zBQ8`^=_9E(7{gkEn?S7^TbgIJk#k$!lg#tDGj@NW9Rl`}SC~di#yB+WZ$*}1l+-G# zq|ka{g-SX5yShkJTM-$&bBbqJx|!zGXTXiQh#9HYV{{X80$DTm@0{Isoizqu&`H3U zo|N~}EQ997)g)XAM5ASGN;K&#bTN#S0m*m|zj#l{TTNz+Dt=0?6AV1@za-L*|2%;9 z)XTq_-vP}59siVUZGu`^C)e+`g_3pZlCtRhJmRxK=44a#JXm3H!vlsxw&(xQrjx~MT3FGU%vcZ zOfO>gh;}gr<wZ2xNB8f7J1YfHNqcw9yva)93yQD1QR3rwh z{}#YiS&nb&NLiT32GS>0?)KdrR8an01fH!B2_d7kjQig9p*sxy6X>sC&t=-$ED`yB3`{^Zh@`$iS26kWfX0=o z#J7hTa04j1flUP54E95@Brs3LHqqsers%iY)g&L#J^hr8!V?lyWyd6F8vfRY9{%*# z9`))(qv?qWA-KHukdqh$*cnT!H_XiE{6BM_hAd&Vmef3dCF zfhJ)10UNS*X^mKLTht?%W2?uKXaEE`GHO7)O{(Et!p1r5z~ZY=ndkC87o+R~kjA1S4Z%TL#Be7=$Zb2n2~0v=Pi6 zf|sPdkrn+2N->UI;vgf4op4m`#csq7<8jrTB7e}2$a2vrc8reTP3O_f>I_Ca;5lVh z<2JOPnX@t}XH zM@a>W!p^1aOk{q|G5|LEm8kHJ>WHy96~elnZBC2zF$#kS+#R8uSSGJcPbrWnAoas) zVI2yKjV_SSXQAIH3b5=1{YnFXm;xOGfO4kib%&*@(cNNw#&2YdmzZzc&9fTHnhWNB zqXD&07aFo7@48n$u+Le`= zcO0M*8*P#`U*t<1W+1J>A`}@M>UTtE~UAj}p#8;C@P%|z;J=Opr`aB0FEbCP;1>68Ve^-)<1fFskm4ErYH zWxm_&pcv5>93XEisfB>kpZS!5mu+0D8O@F{W=6h;VW~bDYZ!bZmLIJY8q=%v^Ysn< z_crYmyf94F#+DkG9wSlQr>s2$>cx`#bxcSUuU~tOS$NK7A)mNZSeEhb4e#3ViJw6V z556gm4RV2~Bx6V4F8f5m3Q_qnuhe8-rg*2V>bMZLjq^AXLG>2{SQu%6cQp}4K?Mh->jVZWUa}tkPV1s8A*fq|+0z+J7HqVu+JsBnO@~j zy1)f081o72b$24qg2Y7-_${FIkV5qB31Y|E^p+9U$1vO+uN-_p&@knWAfhHRG#zcDy#QvE7yM*!;(1~9T zlRU7ulZ!i9-LZHvaDf!~^sta}8_npP)&{B?xmYOpD(!D1^Ow+!DB1N{TP#uFDChc| ziMrygIe#D9hT|)%RJKKy4fm?r-6AzPWCqT!&-BZJ#4Hmao@n`1kt$qk>%b1=1$i5? z`XPR&BBjg2LCgWphp?H&WL!zYJX*38jNmerGiwag8q<-*11!yJ2U{xX2F&UW`3p;Q zw`%x>)99B2Ra6d62%tl3$r(w^9<47ts(Eir-dVm;{M~ZY&NkoBE_EWvgxX2^& z$>qRUu=Apnc2Uir_Z${$HtYR?MTSY`vPW?xuYo9en=8+M5j5pZ++1HjSeNL{N9(#< z5AI-r?MsZKdK9k#DdjVg56O=Se|H9!YJ+4PZZASj5*4P^)gp2;A3Q0A?)!$<{N8t99NR=lTJ4Ve%YXC%dzSM72np~r&s%I3O=WNT@21dIRCODYr0X~mXik~fl))m;0!mq;+L6ax*@1}U{lHb=-4!eC^~I5k6HEWF6NLerwseS!X%kgDOHmH@({ z2V$-d>S+LnLau#lVs4d;Q%YSI4$lM}yt5mVXXA-pgAurQNS-MzT6Yn|? zOCqN_T%187t5bYjNi)QkCy`k7+|X+bOD`{5d?9c#zpc(KO~-74k|e!Jw3)r3=!6Lr zr-(Hyr*%An=@RqIrF2|kRW@z#Ttl^K@b0-VuMQr83)<{q;m|!k?~S>50byrGx4Psu z&fuN`u-K>>sRd0-tePKIth6>w!Dfb29`jhN0BP;aszs2-cDE_fjV;ivh{nhn{&PKWw6caBU4q_t!vMu9_#H}b z-uzY*mkH~gz+|n{r@g9aAJo+nTLc|mupl{;NUfLdD%e>L)KOp-PnfZ`#WK+NFx`{W zJCG4V4ymT^M-gW_x7qO4OZV`9c)RtwmY0J+N_c*BZs|g6k7jtU-}tscxvz>mj=@V+eH& zjdi)Ot+G@0C}EgW0CDG7V744S>8Kk}Rml765bt)Jj_zhX?y+jlWr+HfMi@%5y+!6i zsfTeY&GMz}j`?Ba*sT+@LS?}f2Vpvq_E;jeq~Y`nE}EbUMGj%1DWt0H3XnqIoQv-W zD8%@YLwfq8_s^0A1Q!maPLg3MxBKPhh)yDUX!hM5=9jb_qIB8!uDO!bM)_hFlAPyF z!ZT2vDD+@5Gj3ls2V@|wSA4%I^M?tk%)$p%Gu!nl8Z60R=9i9(ZjSUa;v*5KVEGOL zBUg+~(&WzDm8XC_OQFOVQm+sdCQiP(Hj|i~vPsn~X0L-aNUars81WaE$ziClme{0o z;3Kp|Y~33{ao+c`*{0n4Hzv3$4FXQPs=I=I$`RF+Q$@Kvwyp-$=0{F8S(iKF01a~b zT)Q{EJ8a{F(b$t-Q?ZD5gimBk<`v&0C^qhIOxcTZ%E@W|mGW;%YH0f8x5? zg+(qL3k`N%=^cLgWms{ctpQ8tV);w-vlKj5=Q_wDR(8>WCrXc^z?Dna_x4H96idYA zaApK&1Fk1ZAN^x$Cn|z1ZYZFAN<=n;FM2`5O|73M@RM>-scF+s5-snWXt12pWS!XHVJw1n2dqc-Vp1dew^+N7ne& zOKI+L1$^cy^)N8c7OoGaNI2Yn{k*4QcBPXMgnL+fSP#4$w7)ybQ8etG@VTb=7fcy` z31oay6ygBRWkX7_zL2N|Y{i=#fv9uv*ibuJzQCzyxFp;sKE3v}%`N=*T=Eqj;@Bw# zuML1#1wI>R^kd;UW~00u{3uPP)7=`8a{^WN$F4%V>%cSl@+E@NCkU~WJX0<*nCNA` zY7zfx{Bkre)jmx++6u3MF92Z23N{QG^s^3gXJ>L6*PX{Y-8gTCQ=ly-UvnY4bn1I5 zWbks}Q!iPXxrFh8-(;a+c3EsNX4QgL()w;+C0)^lc;)(a?4b}iR?fV90VhoSte)CD zsn&-(R%t)uWl(sDd;jc0&IdpTfK5nFa-fEmBVW^J-} zN(ZZ71)`jsJLi?N{Sl&oy8?w`%JO{)WSehVu9c=9lx^3=-N0WWp*>8VMld+|3n8M` z7u-C@T`KIWbvEH?8QkCE{!wK;*|a%eLF5Z}Z2|ulyRNCkQ5rJV+Rer@F9B-0qEL^c z#~Y3kroX>%g~zNtzXJTRToRFSrNbPga2bVE|60hf0U1e=>kO_mUnN@uRy{Edm-U6y zvhC9)oQ*haoor37fkJD?y6?{>5Y0jJg3^tM_X#*FVLJbU1!?1|%iskSb7--<@-^!5 zeOApn2h5H!p|lp}y2jN``YrF;GQODGSCjDk7h7b^E;P7sELIJ=YT7z735+g00yB8P zvTGKvp9 zoEJP8=V5t_38dBU1b{hmvtur8;(?o=K#9`}(KST;V8&L7HLS~^xjjx$h;PWvxfe?z ztKe3HS9sy^x!2ES7UMEDzsIngY5XZW^_iLRuqV}G8<*@_*9B8jJET?({kv9DV3X_? zyH=Q2I~u7T$Q1EMDJPclzVewQT?ZELc^NrckvkZ9=s0MJr`yQD5${u62ar1?y#%7G zraDkI6(%G3R-L^Gh}6z|Ti9xVDvh~XmKu9lrGb1sraNtr= z1Sx=W!)M$8W|5(T*#1+iZ7syBy*08;f=OWGHU_4W@iIkea!wzb?R7X!+$UWsS`dG3 zM9Jhdy14F5fGR{iClm}WB^=9U7S}(MV}POBE(ubN8Ps*g_N}q`60R9VI5OQ8a*r-- zxaD!Dh5*sy$+zLmn5p`f^;nkX3dl$(oDt-sxW0fi@}(}8q?v~;HMnVl7>)g+Ho%LcaSa<%QyDX zZVWH*!q-pck@BC>C+hPr#MJN!kN!{8v|%~?u@&cw<2TzN3**|qLvrUT|)Tkg;ZO-JLotuiJZc1@LW z0VA?2Fq#{B^G;Te4l-C$M&?cHm0N;YGP#kf(1Z&^w{(e$$GrPTF8AJ z2p%B8thHPhwC|^^Ex2!N)^19WT>#JZMK9EAUA@Q|m+lx&?w;y_Gtc>4`8~#OTxU7o zb8k}BS#tuPK^>&}qRqzoKbO-^DYBFNf{Lm*szdYJNt5EA@kX4QI3MMi?+!?P0=PI+ z_RU+G>JAu#nPJm5UykN{%Y~U90~`XU(Fi>rKx*N~&T=bv5xq`twcip6=bJ*Z?z^Uo zi5H!yyV{<27t;@8&JiR}(B`e126PANfb7m`0W2kat1!FyVW-nkvFGt{SZkY4o7wTp z**bBLKpudGnoroub+%w>QvP9938%;vV0s2eaeXa?DOn1J2l;$rZcN_~8nd0UBjS$1 zoQ{cM%ii<|%~id%Dg6`EZl_muohmA*&Do0^-xil9fe^sj zDmXasK}&cmI(YG0C_p{fXv~pfiyN7%FQx4?S;}^vL7;HX-;A@K8pV?o`VDGbayjzW zv0v8mSp&ktfM;X4u z_OOeXToT4UAZ2Z5kt@FUFBkn=PDzh_vP1m@SZ`Q))+ZtZ*;dwd8Y)4yP8{;c@LrOnHXkTOTfd%uNwQS;F3< z>6W*Y9_oYsww8=%<8k&GcT$Xg^j)?)(T;g`=II#P%d6Ap*;qJ^S7@5%Q_SWA!T+L=O1^ zw3hY%WoQJo9VXiHu%OGZtSBjKv0LpxX`G_P4Wec?V?QtT+0x>{w$}>s?DA^nTy}cB zI~S@U=SRLRo~FeCwL*-tDRXW?;vL(8XQ`1_$a zA#rjJFvyig?hGye1+<5g(cw!+WPzBkvhTWvyoQuKMD2Z!`;)3J2 z8{UKY^4(6>?K4{}?sB2XSYJ4dlKhI4W95D3gLy?Zl;8({Ey~SBI<_eu`($5wRHVkQ>hL`b^ z$odW`wUa?Jgq^vn=mm47ZML`MYpoy|crN8u(TssdOM#Tu&b5)w_L<-YEBPN8`@5{) zDQE4|d*~Mz4 z*ℑdj@F~&W&Mar!}41FXKZ-@u-X&FrCjojCz`v0$=njx$!;EMSzRp9MUH}h<%;U zRE<5MPjInABigW21^Bz$9d-fxa{O>$%ZkX8x=wC&@R^>;_DJZPro|{$W zY`Y0XGG+{$^*Fl-I5n=oR8>2TFlIXDMCa_Fo^6nbW9L^61el1mWeGX8Cn*j6|K`qP7*`(czdZ$wl_00`ha4o1qcG^;>)1Z$dd(tn+ zsc9??^-nvxFUZ{>B^~2h?qmpPd7h^upeeh8C9$)$FoDC<$0EUtq|Q1WHGF3oV)_ks zyyLt{zIKwk7>vMqRg@huF>7wwlMm`K1~CN@#+k&XgQ=NXcgso653#efSJsdEL78;e zdDBit7_8tGiqr8}mK|ZHuS|`T*UeZgYwYSrPOS4Yvd&Q%XXcD~xYGiu41>n4I8(t( zc^uq!d1B&)aoni)JY}wXCU;@f!LO@jbestpTSL&d^WbU4HVBGlXhdg^=|zG$s^Oqc z3ffYIN6?CNRMN6N3AL|L@YfohjwXF|49BrZ4Wi_8n;q6ZvT0=F-V0}$OTG`&qr;-h zzatl{3VN*vQ$8*@;qd|8qF^KZQ9KPPX#-dunF^!aT9Z=a zSA8&T{=x3!gWbma*4+R2<@5FZmmlx1a!MRbzWs?t7N92?hZ?+j`z-+lA*+*be`wop z3F(jb$)Dw2z4;-?GjX2I#2Mmtw$klS3HA80KmVANCUIZ<;q4W$3IN;ob%J~zcJY7z z{Fn59{rvm?_{*Pu{{0_*ck?Y#^2!iA=0K21Ms7KLvnt}Ut;bW* zP$TIS)(cOwN+zfZ1teq43IUvBPh?{DtzuV~eG)F%Nj zXFp4+Pxr}x(bM_w#0JstlOKM4hXnY;za%_w4}KFg1Aj+MEV0kr>d6Uk8tWcSf)N`w z0q%$7|IJ?x&+Hn`0ai74ws)oye~Z=ErPr}^0l9#$x9cw8`XgU}FXs7+ZvG>&iq8Km zyVI!$liZfx&a3ahUf;U+bNBJfN4~x#43e(&j{y12 zzq1PV=7&GB6iZx!Gdimh3~qDxCTwzPMH!PqBB_c$ynQ==ac+e37Swpq0%~8jik3ji z!FHPWA}*{-+Qcuuq4i7gReZVqvhFHAeZ3i6#eX)Ot2w-i=CmVS&P711l_Gd(8tercPnXi$bMMec*p7A11?mngvTKPc>Z(3$$ANDDv1l6 zB{l>FAEyFVw49?jP-#rB($Cj7@ZVeC-Q6k|Vw_XU`M~G4^jcP}BDZsWvyOakKEv&R zs#WU*0ViH>-0C(N6bHV+bhy8MJpAqLee%~Igd2pHa-Y2LD*iTq6D#IwFTIB4OUNC3 z{IpKp{rdGE|AoBLh}ETc^*YPS{e4<@e|L8%0^kj)=Z5_lnhL_;dPu24AJziGo^6=0 znx5w%TQdH<5)Gd|-hcW+tb`mIz(IP@`jC%2sAG)KTh_r~xzt+(--^AJ*!c1G`u^t2 zKlY!J%USuXjYwLtp*kW$4{edcISX#Ji)#+@a*H<_;+)Mf2k3PSJX+=NBQ>t1mXDdeME%|;cQ}8zevy z=^%iF0q)`X^rN3HW(E)>vC*?yr0dUclBTxmuAjcVyuAF+qB^|h<>}=;$y@yIv1(q9 zd6B)~gO{p&*>`!7zNC4R9onk-_p){6GKW|N6hXtoiuYJ~_N*W%|>P?^W~KHd*F=s+-?``OkkH^D-&^WFolfh_P&H-|MyGXRJ_Ok zZ&y}zn_u$3v-JOaX_MxZwg2Y+{UtA7zFuCw{p$GFFaN76OU$w>^A-cR7uOtqD)Rl0 zjSCYc`u}n*&GBC{&`)W$?@si8>v%%{N`6j$-~aO4m*2nr{_Q`0KOF!4haVap|LfP^ zUwHf<@PA*D!=bx$MMB5Eb9Hw3;}8Duk3SAoSGK!LUhdp$)*jBg_q;vt&Kc3TB(>YM z)$VX{(O>JNNiOc?#~*)usb1#}9uGNZAZ&^Zk-6P#QS6#5N$J02kU5jliO`<9T~7bM zdM|hMXwqGtW@VeVAJYaKey~d`nps)3yE<#yuG#PEs>lx?JGxJ&tf`y4Y$q-?)hWsn zn~c7bzhyi8ph;-Nq^Qryw6U_Lp+BX$qj&N_n=kuUH`&g$S^ewoJ!x=Rc5U{3;_S|} z_vggz4s;+zb@IEmJK~!%@vMByimJ|bSh|}!@JHGte6%^8Yxa?)_{X&0%GYU$Ro7y5 zE^FRsovyp=j>#o2CY~w%vCZp(HmBRyRgJIYRmpcXDe}av8YQf6##k=7V5F|5`=9Na zj{ZB%4pmBvE3y+V1Krh<-V*`l7bWd`f3VZ3 zp+k{7Qdbg{oMBZqdaiA{R1o{j8JE3u#>`#);*x3V#w8}BXn#7#DNe#>M zOR(R|xyb#OgPl5pqYX|@nV)rXa>U-4xgCfp(M^QdEwe^O&+m+D`(PtyO-@t!JA(}P z)6o~I3z%^HlRTX|nsL~A*ZuYj_+#cAU5)IoE-75hc=7qvLF1p%{Z8?=E)e-`egJRh z+mQ{K`JGA9$?d8pKamiO?-P#ta8AG>i3m~6b@d}sD5Y(Cqr=S0x@-N&c}dr?&|7Y& z-utTDTAX1*Bg@z6vrCd=2?_CwU7DPxt|a?j(|+ge;IwGq-A8g^zMY@Ss>#v`lW&su zUDqD}V@J%Zp`AU@g`)eFUucWzUf`115A~SS3iIv_P3N?$_TctB8JZ&LfHYcC?0Pc4 zddr$)LGLgx!`s7tkc8b;$6ej+3)1Nz2}yb!^Y1gMT#|ys-v!UDq1%_0G+m}lw=l_z z-Jz)5U{|v5*`X6|g2yyG)az67Ck(DoWUc&>7UpP9y`fKZ&1zXog`e=8Av6v)Uz%f1 z8$mZs9w9N)0;)4h_KNPSJ12CEJ06L|IsF(a<-GZOE=u>pk*d9OALW6seUQ-1Zbz^D z<3$K@nWB7PJ5-V+Oz8jc5z)CPEt4hPoPJ3X6|8YG%K~Riqglu!4sTfH@S!TY3u%%W zY3Per#H8XC(bmz`aRo899j&75E*TyR=sKDL4YtQ@l4i(*M~Ov3k#=A=5T3e_w8JaM z`vaBUR$|+3Ug#-|qc06z*SzJp1&59#&PA>wh%z^!yHmQlYTC!^QZl*#KI*WOPA)TF z>+qwco0Vn-S$Hjr0eUicdvXn`hS=2(ggR0T_W7x+y8e-1l>G-SexDVrJh*Vj=zmB{ zKOVCt8!B<*?9-Fm_JCe0~Qg;UL z`r6J5rkoyWYf|nb&kC~X!ryOMhtp2oWyO*HA!CtZ1uguNGz;b@Yo#5YJi|%Dx;o)` z9rFTjX4}D)n`|?vSjdp#2}`VV_FYu|5ryE+KqZVQ<%Ah~I^?|s_dr6^JS>`|H}&8} zQHHa{u;&lQ?7h#Ob9T_QP>)Dxwl~D|y>S<7=XVArgE&+1Vh7KETCncH-SlEE|690z z+OTk%_d;EENlI7>j#S~K?V#SUPK;AZg^~OP2P9?EUb}kI7;-moiEJQ<$E0>_=t1;i zMl-ICm%XNZMoOc~dDb0S6Uy*1SfwAmqD=;sxob^RRWYd7{ZO4r$q1T!P`XJ2?o{Q2 z!i+Oy4;p6URhrmOqzdnH>Os{S+ksqO`bXB9Ee8#FK&_%V({1kQs7Cm+v&;9Tw2hXdjC#!IgntMz+nFzGoIOnS!`1w6Aoi>XtB5TJ^C0`=O`7U#El1 zzI#IZjy8@aO`=m|{PNR3l7hjGAW+ed!mS@D(W2FjUik5edbcvWk5zNKaxFL6PF-LZ zu*c#38XuYrNtq4J@=~|zX!cxU!U!k2)1+Dlsfn|0iVu8q#xUNU=Gh{=((O4DA)!hb zUEG}<8&M+4=d779w)b1gASBkirnCjqsnMz8b*duX8_=chN`*URC0$UyZ+7p^0L$St zzjiP3C=khgN0GPbdwMvbw^>fmck7>%7c}#?4`mISjJ)BXW+2ZNn17>q}$a&v+Qo ze$|8{vfh*leoyw)P1rfl!AtrB90$QaZ``&puZ{4fhhwVKYUdZ5J6-R9WA}b^WzN{4 z%8l@Yi7x>Po+Bx6N$yO8z6QA{oUA~Ecb!PO9O;zUe&#oF>JZ_DBvNZ%4v&mXxMh}ecrTut%C-DI z>Pq6_-yw&{e0!ynsOsC7zRccZ_TjHS4eIP_+1~Gx6hY~MZ5)s0=Ok}0C{qobWeo)7 zA!$mc8U8Jv=+iOGg4hgsOjqA!aaf;P5`}#^@{` zPB2`|5E;`U{?3|;?hO09Gkqi?Cy#8M^~V(NgvE}7e~@LUk0?8l47aC)UF|;|cy%+Z zHBU{f3^zu7)&6Q-#mn4Z1=zp|a?F|YllM5=sXEzT4rvL+zztQxf57przI8cmCegk3 z*SpV<_|~fGr)DJ`$)6phI*YFnW4k;hJ9swfWu$zt6nUf>aqDSA*qn z3b-7$rj4$qR^~`v|3QEFkrUSH#JMc}ua_?`2s6olX-paac_9^})BhI6N{2}=<-cBz zmXw#5FJFHA^3DCQRkmF(mGNKMufKlzCHwaE+b_xCKeP0gDBGVKCS^ zIKF$o|C539U@(0eU{9Cxa(@g?n$ZLhCssb)qE}KGNpF1cQ(Eoow=ZA(a$h-bjTk&V zbUMteA!#;Y<)^BkM+5HYHAl?@z5l!LI{cbxAp=W5N7^0e5Mwv9n#tx>=$axDye9of zTR5w?f^X@D)dVm6S6d4;N)#!;Y|K;Se}> z!w~h@D5E357D&+O_-7@Eb^@28 z6mh^7aereon$918Tq@e3stG>R6H+k!yEs+5WBE1+r;PtHOX=>=ZO({|lftt{inyfZ zR*1=v&DTg=;bZAw$PgqcIuam^6D58eIwxEPLDNmSv!CDmuxJa_(;wd4M&^(2aHX2#r zqDJ*buWDLi#C2)49GxQbWN6#m|LP82nN!Jf20rEYWDpb5n5Q@!VkJ2@f=a*K8jvW(|X{eEPs9c=?c*6*~ zaT_=ztFG9wQSQEd|IQjCZo}MZoO8z>V;@6+7m4;4IV<#3hx+c_YaY0~Ival>%@F=U zIH}5s7k#KbZL&&p`qKU2$cAV22th@fz#&Pm^1P-G-()8ev`(J6V>6JCc`}DnKWtSZ zierWxeEHt!BoUE_3@9Z+$k!-=#U+2Z#|@O>?wBdZ2?eG*2R=1*%EyjVDcY)jjbsOH zX`YJX@m9oO(i?}Z8Kp|6^jg{++%r7+rT6_Uce#Y6h}LdTtQh*t)X~l#`-Ze2Bp^rJ z2O-8@V3-+Ew?`G*BVm21-f-KYnflwrl5IS7V)C8zwKw?zX}!$Bk`odITdrf;v7U6) z9iO>=)@%6e!>aM3B&(h-s^0?Ur&CAI)#&m0S|a1cxx$r8;)7}tXDX(S${8f||1)@{ z`>vgkN0LMV_MN0sSNHTA7OO}mtGEKF$YDEZo~+7V{%i{8GbSf8OGE8|wZevs(7i!%lB$SY+(r0o{i+Q~94s6eU@ z2>(%uYr}cD)HYo?kA3zJI*L^LFdl?HDep+u*C+#ha_3 zkEio4QZzQ0R+6UdLI$TRx;m=3t*U|(+-J|eze8}UvTEP2UuEqA-dSdnQ-_v|hxKPA z*bdov=KOpgP&P_76I4cS^DxMc~Cfbv|X%2k9mfcpX_-nE^ zu6l?A@{q;)gBwYZ_s^LA;568F(7{W3RzBG>Tu+>io!MKSy}P+Q>4tD8o$4grj4?9lv`o_ym1mo zteiNz5TA6jltMPUL#kbret|8Q*olCB(Y-+F757=K4%kJ`*alur6iE~n+YReA z7g(hb^N9JR`J3dL()RiP=I}o)9jQEWW^OhcQi6YV6|$m2IX-;BS^$_&Kk_5I%A)>z z-GO#)52oR37b10F@{^UEXE;tH$>rZbn70b#$oic;b^MN0HppCindS!oyMANu;Wre=e}QG8c9sKh?_~IyIZ2L%Xz4h14Rf-kcC;} zjrhsa5BUdL0GX5>^8IJZzpHV&GI=A-NjFuw&CKWG2W9aj1z_0g>{2y8Nm0@xyyyZ) zZt`?G4Jek;YhPzpa)a#p%jt@1oYy>7e06UtyUU)HT*odO&Ch^=3?=%uRAUHoNgP1(cqqsZeb3WDUeCRulXowF zVbvV9R8Q+lXp`+KgeNt98&s8uHl7m!dfZb}&1E94)w+l!TYX5H6i$CFgyEX2aA9w~ zqMi=pS~h%XvqK)uk?X2SGF5wMo%j%lGdYlG+`>UKuE~ZCAp3G;Ul=Q+^al2H*U!kR zUpRVM#i%sSN8Q(ss~}o&eGS}%2p^ARX3ZFvQ?nJyjNNSg?U_1kd#EDAM06<_l*&z! z4^x%P2ZZAW7tn`Tu?1RQ&qXoA0+Wd&2ftsMAoX1b)F{Z35j7?A8I$GIiLYP2yqYcY z^;dI^Eg$Y;gPn5(fx|mqGNcxsXo{lTeEe+&+cJ-?#22{)LLH=9X)9t0@u^#@&YCEj zMNn)QsWL+XU8zc05`rG7IzdeN;j+{Rq1TuOJk&7*DxqOT$@KD>xOdH>T9D?^cSx-RN0@|8LN$<=s>^Ih2XIugA71w#F zp~G6QqLR(pKBW9{qLdEDnz^U2rj zy*Cbr0=mXcIaQVCT;OEsaZo(fwL%Y`mC?BF@V+S$56%XsWMCq*S%`RdCnqJ^jXGvo zI&~+9Q;wnQ{9V)}w;8il->$_s6uipv!!!#`1FGTzhjSv*GbYe5?%^4RTqh)VYZbIc zV=Q;?7N*$v@P75ef1DlV*SN02L^AFr{&V@@xNm*joe1E~hmw|$-@=rT z$&&$ku;deikbcX`-TOH!c^(X`36!GC3W97g^X*q88GyTyvP^9M9odja6n8!zUhZV< zL}{`J_A_{oJu`CjhTZW)T$k$km>y77bbCso)j3~2fy=v#NFU<9&G*{N~n1A`Ky7_ zUsTYXcLFqCOX)Y>7F^uG0Y zXZ~_i40c*%m_lZ+IBFhM?__^0*e6+WTri~Ga7obc$l~#ten_@Lb%hZ$hk)Vb*r9o` zh7xf9M-msn;XY^K)=Die^T^;kRNZTr@uKjW|_IIMK*Ob;6GqWOUNx z5S7v73uw>GY9qWLtMC_)i5c9=rd=m3#83b3use@;MgKf??(f8qC4TMGXM=7WO_&^6vmiLhJmW^zVgYwm1BbhGsY-cH^R#&VkuS%=usjKOAe4l(2IN7p#AGs(yiPF{0?l%pLcJ<_8;~3&>3Ae%iY_Kzu z`!dkNvI882yixnxD&3j|NEGDpCA$5p#;^G`s0J2rQ_-HW1U6u0bMp&#PO^&RpqIFo zKU#Eu;OBbM_2H>ou7xpU%>|ffbKhWTo|~!G!J>qu+@8Ihfmm~USEjGyl6(&kss8=a zU)PVEmt!O6>-XMs`N=naf_@`hECRCH^#CH>cC)tm1_$Ro?lPSK_;FC0!j|7wHKvd4 z5i2}?WVnZSulsZo>UA5we=Q~6X#52?W@tBfq!N#S#?@;Tr;c-(gXL$@slG|+z=Fxg zzP-@X@l0pXFV9KN~mcP zl6CP3@?+PZPt;Wc0Gxx`--*%6m0AA29W$BBH<1P za@+^k-aapFQ#$5RluRBe92sqTxRMYb24k-3@vI+w?1gC^VjMSl3ZbYI+ti@ZB7!Ex z15jAbM|3CjxtxwI3WK9`aXo>+Z+#n|(!FO*2>UEjvW8?rAKcr7xOGC95QkfRJJk*5 zpyf+YGh9@$Fn?fZqK$Y1-cE1>q&82%@XH@!namqpH-^*f$~7bUgzj|A7qDhz%Yx09 zO1UtUA#e#`_!`L>#+2>~y@jtF5$axaI1*+lfS%^sO~!r3je@RvHEDN2vGK}T+rZD5 zb;%W&3Ig>#v?bS*DM_E&%| za0di|@oKMn-cB9R_H5pdtGaiCwO1b;r*{nJ+hc_t(~%ZmUmDEc0&5*<0eau`fHSCP zZdZTfd)=&ZRHV)u&ChYOk3I8ai5Mx+o8s->RCk?-4&83A$ z_F(G|$Dz>3i(N1Ci?V~pUy+H4^#p~0E(Pesnq+UEQ_XJWKu7~U2o50)SxY-`rw zE0->u$%I6mfHc{;=muG9x2ZM!*QpHw6hJ(aKrZby%!&~q=2kWbqu!85w0ID?YTVMO zWJCy(yqrLzxvsO9!Bh8$tC}mMN!Rd6o9qWDdIOx6$r;_lM%PLSYf!Xv${pp@k?P z#3B9=5N-USTtPlGBSiay|Sb z8VTEkTillmO0Y85ri{a`6BE`XojEy03!Jj0e=4_Q5swU-bA+5nL$$MBs%EQ7ycF-g z;&&CGT8K$hg@3xU=O03+Rpp;pNU>C)i@Bp(SEtYgoLQ)JkQi;yKH)@t5y|k(f=5qy zr$Wod-x=$}i=0CulU4`dE96I{Y8SlYP|JQboCiFM$(&HTaz#D$JUEZmdkKUKSOWB^ z{B*O|?p+y+OOYn-ysr?1iRx2~_h8pTA;S$&$j1jM)$Yu@8C*oLaRO4+XkKD}f3LcN z%ExdnhU7K1LPAzw_=w>o=AYj$c7CI_+vpS}3suolzt>HaV^L4WK@TY=G%0|va`yzu zvfU}$W{?tGj#Ed+H(_G35<1+Pef&7q2B1WPNJ6i{vG&Ep1HrSrV>uQ;nk&HBupFw~`N(Z1)ypAD-r6b2x|8R{Y zjR|&QNbhtX1q3^3_BjbO8qdSur?<}~QWplNKk|uB2da$VgOU*>xsEk#R1ZhhG?43h^C>4CC2R2Unk$6FG(%PUn_vShJ%tE19mu9qxe7N3 zD;t6YE35Je%@Y>mfiMb`^<+^KQt9g#g90Y4UDf0+BMGD1@`h`YiXf6HC}|?&jdtJm z>sCemI03jD+RR$)eMIeCMtVvI6E}A5R7a~a8R_~v4KUAGOaPTI6C*%@jj9MYdK7r= zKkBPevw;#`j5RUxq!blw|Itpm#c>+EaoZrz#m-GED;SkJGslcHrw?s2V~y<9tv|z) zb^~A)t%bDkwqm%>%G}B7cf2`g>#oxz^*KV=4Unzy<+uq=L_)ieU_3UQ;b>=j)kwgr{9M&D z&u$fON${7H^Gv^JTp!N24PWye=5Q|JkQ&D=y|*WAa=H?q(*N+fNvS!vr5##uzv2m6 z8$d&Z5rh+km?P6!h!bYu5KKPd80+-&QM;&ci>+|ZGVJ6320?g|j zw<>H>@G>+mw0?EX6%^&Iu^9Y7cU1SXP5`l()X8C}2y(h)2<2N9k#ruEtS@28cYejd zGJ;^dUP4A|*&j)#Y$hN!tz?_1Fab z0~)Mi#)eJ*rM3f^wQCRkqaO1a=axiSvA$M7y4jDwBzqV!@qqN`8ns@^2g{4H+F4(d zMmh(C@Nw=&((P{#<$t%&lGgQxojXDeznrDlHCI_OQ%BJ3-m1CEy_hu#s;>t&o7CFSxX339*hn+E?cGUb6c_Vs0u?D|Yn^b6IbXV7aR3c8)i7&O{!n zrg0XDVqB}Fr-zd&&<1zvBo9SG$8w-YN@`atJyS@%yM%i*`lcQnyxQq%K*t4Dyv_Z( z^efQw`%{k^Dzl1_&-p>#W*12w>|x+P^oB&H6Qe6b@fxbt0mxn=j_3TyDt2Qi&=!x> z7~~#?(CAPz78Lb|ez2VT!}MTcJQtAklEbj&#gd^|#l%RX(e0}dtwA}Wdjr?b?aS& zs#spCP_@S5!Mb}e8T-E0LMv#9;haS|kcZ;IK6d4633OJ!E@+Pn`U?2i4IiYGt(twL zpv@!j&$sV`=V3GMMBt1p%$NKUUa=X!^Ic-P@*ptUjKUbw80;k2Yd8mQosmMsOTisr zwU@}AJNDwjq2fpLghI3CJE-XQV&pMaWMtE`dzjHeJ*@nVZjASLsXYS_%zZ!#0=V$p zwlxpWIdNR2X5v7M@_>gwXWIgTeEVMctPvhiVf*QIlzh}}%YezaB+NRtnVrF#df z>7SLq>s3uJoCW^9B<_1>a!E)j$QbMX&1TFw&5|kSRo)S~dso7Hnsit4zpV*$ml!00 zhnHl}$c(s5?7HJNm*Q$kx)zjL?SjSbp+{xH*GKDIm+gN4@|FJow*W<8Ytd)N5IZ-( zB^PQ0>~A?G-6``97baz_O;`7~_ed@7^CJE}NmBG~?dXCC?z7y3d!4Lq-6?UY7kT*_ zd+0fH4UZmjet~A>-%@R_3MII6|+`pH6VQ_XbF;-`(E|JRkzcBr{|XW7N6Js8I|*fE5ImG zZbD}4!vtYH^(afod+po=PyLnb_QBoi>sR*<3Y{`csnC|CS-Ag7^37XLR20$+ zKdD4XvAh!l! zB!b?Ql5p#Rtm($--LX6*V)ox>?Teh;_0@H+PVQ}VVI$m7-*9luBD}{Ykrdk%VWY&n zjf@cUJqRLPl=3!P!w6MK5@2TKk(2x_ud8_z3)|xLv)jS5{w!M-8%hLaP`SK!4D*j8t zl>DDohrwS^vHd~|`L7pbN|QO2zU1Z0w;#X#=6?9$1^@4Vb;4)RNQGzq;+n%xMZW*B zaWDV&)8zw^*`G+P|BEiz;q_k?p8?dP9={qlSA zAIE+2??3#|==fj1{`SiY6Oe_ac!v#h7a%NO2!@EeDO@xO0n)`Ev4`W#}0T>Oi37*Q2Pf(p{oWLQ-z-i zYNnH?jLhB@63D0;CCNSv^>0PSm*YHT^fG&gyXPrmv)}zWn}VE)G&Nw*NhI=A^9!Ov z0y994P27KyA##wJovMx~Wzhl!rn!pO{=h4`kxHxXqxU9nv&c&ru;HX2uuMwT=mNgL z7L+_MVqdRv5n0-yI+graop$4QA^10xgZ5tEsEX8hL>Z@{@p2j8BDef^r$ zt7{>srqXbwxPk;pOnyFED01g5@b9`fLSzASe5#L(k$&2?(bRqvLUJJ>eqC3k%hv9u z`eU`jsK{?k%p>+%ji4ILS`sP5tT1`OFWP`>bK>1;Q`NQ9xdG2BaoKW0*rZ*-H6u!> z1{@jIg#BDTF#akhfV^HZ{H3DMk^O|GmBKfKw`rGJUklFcayez`NRQ935LltW8gUZZ zK(ztmUvP&-5r>%N`lEs#s5%=$dM!8Qs!DHUtUD)lHA1~n^&nyMh_|^!&YJG_8TV1T z5~FNp=&fC}H1w5w-qK`bHA%oH5PntvKO2Q`d3Zz?i)1XP*Xuba0<^17O@i(|SgGsv zB3{QRW@PwjbmZu7vT7(R)_~(ebf@|CkX% z1Y%_gUBRmEX#qErm}ZkPz!1iFNgJ*Y%)mxS56CcO7tW$< z<7DG+1;?|hzw+K)c+UDK^ z{p7d&g6!T!?{=>kPs} zEE|94NGz3{PEoOq!Y~}qpcM#d_Jf6#T9O(C={Lna+K)H@moZjlUjSPN42ME=7WjZn zxa6(+_qoi)J?ZfH9X)C663JkUmiN5N$Cl3%$LC9}IXpKx!Bp8LOiNj#WhX?*L{h-P z2(5-^aod5hd}%Bnp2VLkH+O7!rYd@Brit_jSPs|lOMSt1|8bbG!f1-%!kSLL?_1g) za>p|SK*U;dh2(aG5yTAIVvQl@AH}nMksKG@Sq(c9ipJ9AW(sp(H>PWrLKOH{h1YrU zE~pC2NENJKv)#grkMocPeXd{kx{-6$v(rKIP;1j{T zsQ*;}#zB@r%4hx&f37ypdvYs;GnbMXZ4&D@*=5isjthDi1OX8ej>zl%wio^Ux)7>6B=GSZ<#c^qrtNol?#go z1%5-1f5A;GE904xG48!jCHrjIdHIk{hqVl@;GpzsWIm(0?Df*v#rREQK7NHZPzpmMB>%?=v;2I!4io8H?g1sm8 zuG;gLwO1AW1Fasc{L|8177*hyMS*`6JSsr-cO~mdhRCh0!+Beg2;^u!s4(1$oG%Uk zo_ijp85GS;U)f_avN!5p-Px{cN{;V)vT{Kj3S;yI5g88Esk=ko96HpgDB?=V;B@$W z`I?551`?~^-reU7i21g`Efo7qdvbM+;315aaLqU&J8Nxy?dpupOTBe;GAtOi@5r+A z87_lu!J6~Jmrs$*<) z2!37-$4aoIrQ>I)kIdqRK#K90#7&NomYM2z7q3L`)30BCmffd!DkHjG08a1cMTZ(W z`KA~EmR2cd-A71c)(p{*81Es|rPSLrF>jsd2rAOA;Mix9_c}sv1|A+#69VY{mV!NF zg$9E9RJm-)DY63u-1>-I1_mMCSEtUI0JLrY<;#}^qiZ$&!0M6~A(rXB{gb7ep)a;C zj7Bk`CLU4`#+TG%E-q4km%!Pr85Sd|zfKC4qNj{1M?O$*Nyhq~UZ5>1tc=cwz%4e%KlN&6te9;T9 zzU)htu6s!X(Iu_ASxq=|>8Uoru8#LIL+|di$fzrYUT?HdTX>_tNvD&sh|%L6S6|{g z+ye(>bpVzF=~7vQSOUj%$_-4D!$T&VHQu=rm(>=@2U<*pN?--b)yJ*|a1<~f@pF$Z zr{Z1^@)LkVa;z{?M8szbkR^Tm+>nsNB*;p0JMQfE@X<=vCF(cm{)gc`#a=~&?{Lmx zJ>=HO&#!B`CQ}j}(W65^t;rZAhpRTDr7s&^I~@6HYU$*x$x($t-7efEE&>d*wFFto zyKEKVUUO>Gx1>BQ(u<6>4OKyVL;GYrVQKO*PNc#jAnfduU9a*Td6Xprjacq73LYv* z%Z;JF9Nj5o&%d~evrBVgd{GH7-=MOP1k49#urP{P#mj^PSE`Y#znb|LXqo> zWjM#R?=sx9y2-gNh|$$i(Vx3lewh;=<~=rz1b5NB^EY(!M5eKI8WdDo?OQ(Dw0!ER z?KwOWq2nXBn7vYh58<=ts>>yB=bww_nibZy`uskBC52tHDP_kqp7!R2u`h zqI$O+WIAX|0^aSjbIvVZC1z(i!kR^WEUKzmLqF7Ij|0cV_ddpo<1FfWktN*s=J@Oq zk+EuijN*{aT-uRa6e$uzFzXEzRv1Q7hQRtIYfcZvtAtP#DYzMA?R(4wK}zVCgt_-@ zkU%wkcwx>QT)=NiN3TWln&j=X(c5uKgN|= zW#N=GdP7gPEWFa>`5WyeQ8^Fm6MBl|t*ieeFF#!M+$p)0k?GMR!B7Zhf~Q=QEUJL?uS2EqMn? zF`f>Wm2(;5lDsBCXb?-ul2bIdQsE{o z*X7Zn7z6NO2VlsX%P;7&eU&A2d4r@yRmp)T&!Nu6j-2Mv!}+jVc}kjn z@>qS@mi9Qb$W~;pzZtjMJ5_4zYxV396&6U9(l$ zLNXVk@|n+B(IoYKLsB8k?Q`@#lez^z{t23z&v&?LEAce?pZGGwLGct?0;|GbD)PN@ zRK;uhoXd1)23h%*6*MR*IDi*P|3UgtuqoG*f%Zy@6XjOg5TcNe&+m?$=WYc(Xs`RJ!1iP?TkalAm*{v?C5&5OTrb@?>BJ zhD@C3{Mc^gSB+WMkd89qMkbNd9co&QO!+S;gtuW$0T}EmVQ$lDM3%S3o+C(CiG<;J zHZc4rOyu5g?po!aI+anQ_ETrD+6di%{KmG#7Bhltp!f&(WwrS@`K_KREe8USKfhc4 zmAqF0mgi^t3C!cflDK}Il$yPg)-(0o3FMngtsR!N&s*-;u+Yqc0~d|rY*=z-&6B57 zzc<3BTjY$%HHiLGL%0b32yUH3`esjRy2tv!?VrVmqgYOM7t{1}=!#V@fZHO;wx|0f zANfDvAD?7OU)-sNM*lgh93%PU1wPGH_D{OW{@c08?lZKpR9iqLFn=RL$XtE1R_yrn zGdE_KW+>8EoL3o8*|&o|7S;P$Wn67xzhoZO^Yu-Z!E7 zKv`O-u1@#LDR?qKM*M-(H}Xre#1H!4z^eXlVNuVuRGwf-ub#qU3wkx>WtQ_cWH(eP z*3M+1rM#N@DwgwfWAq<1ME|>5eN(JrbDD|)#u)G%3t$L==w~zLfWjT$k_}adW7L!G zJ4Yl`fb#dCGqNHvH~27@qJkO9W&uC~zbUsS$ZZ9+WVW&HesN>LPBUiz>VS}3%u1A) zGy^Ht8-8;E17AT6uX^T#UO{cHh*(Q%Nsi5CG#BRBte|f<^|PqW6F6&03IBV2Pny#w z?dR-7qUp4r?+~>043FbR0H%b9nYX#%_ADaqkxWxp}fkosO2FBBL znO9j$Wwdk2ClFla#-2iSb078^qD%a{i|F~QStsQ+2+5?7+D!k^VX0#W(p6W7+yspGDui;*)8BuY>oJ#>I= z-RG_9Ne^VBgwqL>Ou_LOozG!r!*oN>M@0k^JICp#&m|hx8k!J(23=svAaDi{D|vhL z9Fw;sFF1n07TdNn*fO7T0y&>Z$xp>IFpaxL5VQ00D(6rx(V*Gtz-Q0^*2JoW2m|QE z>3#;?Rhr}^s++5ltEevFuu}*gxVJ`9c62y(Cn6u33olGKkW6WB;Wo@%aqXxEqLknW@6#_2l?f?x|N%?(K{Br`Wy zvXG365x_oy9GO!&TFHu56`-uHK|7y9e|CW8F`b3(Z^o_l4~X5J1AO1>FD$v`i|8x} z@-Ct?_bn|VGWR7dB7%#J3=jJ3*`#LQ;bjY2$QT|HTlHSxhUt$9EF}SFB~^)$^p+mO z_1J<~wc&uI^K=H0U)2m&pT$-s180h}chU2^L$Qi;f_n7RyqSh-c}*@aZ=ME69Jf7= zV2B3)F+=_K?yB2h8EX&zYQX0m{9OU%pJD2(TMU#obisu?BmX<&HA17@a*=cI+6p`~ z?-!=22%7D9E3AbpdeV)9w9tI9kyDfJBxx$GEMzau?GxZzd;Nj(Z0lJ>l410V%axJ) zbQz7;M2NZp)UFCCcD~n<<^;fjv&2oc$wn;JmR8bTrai4BdaX^ZXqz|ao~D4H$l$d1 z&7E65dyNYwJ--c58fzW?*nlSOD0A1MUNR zWRj1fO49A=_~b>#P~VP>HV4w3!naUFW2WKUI+_Pm@Uh3b&&xL$6<2p*19I2*tFOLq zK*G9N%S9U|tVslK3l`v;g{?G%W{tnqCXEh0m(kA}e5p!1i^yzwdw29%N^;i%m)_;& zYsBzM8XwYHn$$E|0r!md)X3Y6XdmH|$vKD4U)&|M@DGV|Tr%`isjDKVcMV`>?}6nw zuC!L+4`%F>_3#!n@IYMk{mSu+^j&ywyu zX;iC|i@9{Q$dB3K<4|N?d%`bP%ur(OGjeAp9Xq#cU_FxmxMF^aO8}pdKlT-K)ccpm z{l3LV)mHqUK4EoIO~aju!;+G})3@o39r)R=7!X>Z0EVs@#7VamZXPpn7-|0j^)?F0 ze$~;_N~K($PQ{VGGax#Tq3h(jp|q!T5(Ams& z`!qT2+~G{@A@2NS+KZ$jbBC3j+auWs>^jqKagxfK`Zhij!WB}4PPeI~A2j`zf%=&_ zHR#5I)xx)qIk5eyA90tDv_h|##VZA`Epm?a^g5C44(RHf?CEOl8B|aD5gn{aU#`96 z(&dz;jg24qW2A|VHeA!r=iD5qHQ18HgjKU=?#NE-g3U-)hj&@Zj(&bh+=)5lCHn4t zZsGCP=MTrIc$_};o~Lw&&~h^B_Ea4${hgHNY$+X%N`N2aCs^!Ktw(!FIx;$Avz5V@ zXJR+-8+0&7?9tWLDJrjRmD)h3Na~t<=VIwN-=#E#qifwm;vB6FEEpXW!D2dL4$(`} z1~}Y(6V#Wk*0V@lQd|`qUV1979@>|n>z8B~*Nj+JGKjFqNWW^66C9yijGg*zpo2c< zjnBYaGdN!oW_;XBmRmo%z!>%KYbiJ$+;)44p*?FveI1*Q*>)EafAoyfEHo8fLxI`M zj(cqqQ{5@rwR7us*BHYztd)Bs`31+1V#@*fHmw;@KjWm!tt`7JqRLsFmSaksOgi9_ ze8?_{x*yyIb5|7uE!aqv;L_jqclYMlb^;Fv=ccPc3nW#fz~?C`15mXfI*I8cI4Lcs z)26D`6j(jBZ%kE0nCNJ$JuOY=*7#A!aZgI6Wx_oPS!EWp{hvTPM7L;$93=Km%;&MU zd$>?*tQ)q<`KG#w;Juy1QGzy!aEi8D!=b49A1a%Tz9?-0gnNYF+n(hFH?N<^+C z$87GhilTalhEJD)`|WXOcNc3?v-negbdj2eMA{&!DPuYm_x$wTM<+7W;rM0l8#s#Z z!n>T$VE=5~TPsd>R!wLE{Fq2E)zlr89N;dZ=X#p|{A1Hd(Way3xTt`%iwV2=!8cM%q+Ms^2 zr*v#gIqSW(0A4Y>kp0MGMy+#&gDe_U^DU#0lCR!QYJObpHq#JdNfT@5@OGam96R1zD%P!hS;FWB4)O4;cTe1D|?^R~*>5;76997P_ zYk4~Xu=nBAh*@v6NcV~y7~y;=Gc3uFu9is|??Bfj&)1tq>~U3t&q=dzxGSSvlznK% zONLYS=5e9vpy^zjEV)RxZ1mgOeI%D+(djf5{24vgB9dRjzD{GoD-`(1szb*&?)T;` z(@-B8Sqn&5nr^~R^ctu`UGs{9OM6xZz?7w#7Af+0w&6N~gjzp&I z1KItr=L#Lu;*aUm!yzkV1B3BcM8_WM8!{DY*=!!&8cWtdZHLPNbA1L#|Em(1jgKTNKCtW%#r5`)5w& z!nICjRgw`HVDN(%e$qa9pUSQfl&oW(6)7xjM2me!4I;{a zI>-~GgSxk+Zv`3mS((;V&P7oj37?P4zAD5pT(w3C3e!y=V93M*U1fI$M<2Kzg5!C4 zC^}pbSc@T`w0t5>CO``CDB3Q2BPEnh71f@;ArFPK+-Opm)3Kl#0*fMS{K zMdm^Vbq<*n=4EFZaFHUMn>=%$GCw5QLA}toR3;_IOnh`hW-i(2^cf7;78%BglLUrs zCuxzXrb#|}Hw+0eG(D1t`N7Vy$RtHZp(LL&6D#}hH(aKsv?C@dE)n7Gc`?k=+iE&K zmyA^Smak65NnCVa+B0!X;EGn7f?DJkIwvyr^71Kj$+rZtel7l4ZSkqjj4i&{dnpx{3-bLPQn zyI0o9YNgYUI3QP(dp%?+Y)mk$Y1$>Y`gcD>aFJxZ8Yw?q0S%+12mlc{YuN)^*k2u26>lTC+~!3kPM2nkTWh)7^y)d+mw(zVghsi#*5Sa} z-j{5yHiu|*d8*eQ^$48_sWnvLf#7r^6c|aOZIfIgZ`_w{ACOf^f?Z)nbYYNoObdd` z%-Gp8AMHLb*`YzdvpsK3^{Se5Gm1Cww8yN%0ZU8{p>xUT;^%!??to(df*Ogo;HUY zM^qB)Ps+o&8rB|7r;W%_&Uu_(;S1MMKLMM~8}2Eo?k)Dz&|l6zwYg5pxijG1bKPE_ zc4P~Xy^b1c-C)UWHufWs!br2>+FoL^>F$zU_Qd#s6Rsw1Q&mfEL`W~NT1vC(eHdH* zyhkjPQy}@C7WhbJ%+g<7Ay?I_Tiy%FI0O{5DK~L10z8eTdcQ8`SaCRMRu|O=8|&|| zhsv>9YagfhgoSI6LGyd8<`t?Yr#X)#{H$2LD&+<$ZR)hyOkBe&O*QaDBYbZ*95qhAZ#l3FZ#C$w%o&& z2s%EmC+;xB*%Reyv&*8!Xt13uNE|zne0@OqG3m(;7FV6PsFutwqKjtdS5JpFF(V-% zb#9K?7q@lB-(>Q-Z0)E1f>iC3EExlHr)u96|&2|=akS=HQn?dtwFm^ zEM7ssE6L<)^J6~xq)3M^5%!P#*FImOm@ED~T7gWWmt!i37{}7J`0e{zQ<$<2B2F^K4YGp0n4EH8` zf*i0GK;#*opH}Ie6Sv`H#Pkuo5Lu{pXbP|(v@diQP90eg*3|Gh^sQr}RvHYLjYR24 z_ogFC7MP)rCLw$ji0F8L?DW0D6m;7gJ+kPAub;qgAcC$g5i)$U?^bNF*?Qx<2dp^9 zH;{pMB#snnH1bg6KTUTgo1>^M!isyo?iy^^CY!$XmUo4zq~TE z#ag3GLnKqVS~cMGDn32bqv(k+giz}VrcdB(Z~`{Gy?8Y0b0EokFp zYIbmKK~~6AM{}YfMFv-NpSMlYpz~x(uYOZX?R$k&bAxEvE4;ZJtz|Idt)+*2TNEg6 zUF5PpR)YR)yQO-$sA{p?Qr0N8dJKI+5e8ednVZn@{yyXH;$Xvj!vjpLuka?T(lo93 zg*Wujnz&gz+sg&X-(r&m0>&vUjU9>tq(wL9y0P{`B0jO3(N@uXK8?euN1RzSi>|zo z1cHgP1#y@KMW&kxj{|g9>47ubBpM|UCrb$qZb$q0}$Dstp!qNn@Yqo4WW@Q^2f$m+*tT+U-Y@LQEi4{cogohv|8 zeM%K3Hv;rKCAHgXPfl6cF>p}Ffhk{|CDLWMD_vRSuQB?@R?|RqC=Q3MXYSo~Tcps5 zn+0`&_(!i3k|fMn0oJB7Kz0IOcsyumz^e4A0N9L75Gg8}ec9%92jUfU6b`X4j7tqW zv$bJaH!IECo3zU+O4+fUj2Q7|YYf%ZM6b!eGr9qXZQ185c^1kf+b;{R>MPQiL7!DT!Su*a=Y7?L+{VqG zUZngFL0UCw#Logo?WWtUiWhQr@V5Nr-&i1ApDLT>{$fR+BY?p;&8~GDuM(-5SS22C z${0Z6QUUz_Wkc0V$ss?hCk3Mjrrqj|uhL+lZPg;ywe49(9f{{QTij-4te>*yPu7U2 zm$vCqjCIa3Na}h!RReATW<6KHN_vOcaiEGLBv)m<=ToF(?UyFOahQ^>Cm7NII9fg% zX8JG;#^gNfsjJ5x%XgC#N^n|+jFvM{ObqDD-p1ODFc;Clo$Wajrw7!Pz#y5?bc?rGwtBc4>IH@onB;MMeXhFjDxtkImudC3k4$_A=Bq?%1bOmdm}Vv& zD%aWK(^kd5j8h)le}+}sYQvA?j7p3^n(ULYh}g?J?s6*PS!7M#7`sg*6kCs z2<6UCxQ92E%zcy&mK_NL@;Nm92D+&0cGQz?tO551WGy35$w%3bPv7PD2F`#+c00F5 zzA(){klkKp+botr+O)@Qu+w;$T~#ZLLx0^c*EgYUEnJx~c%5i7t}|erJ$A4a2kmWN zpPlkTo^VLEYkEk}B4dfKzT)OdbXe9%AD5<*T7QPrZ@VUY0=1Av)nu278}2pQ!~FWQ z34jsHrePj1MB_l~jT0A=vT{9retW{k$i&2O;fcraJP~ty{tW7zt66}ij>;J}2g@hW zHCFDu;?C+= z*2Wa_cm30|q#?>9yM}$)0@I5`5O~yRr85h$B9Oim842w2UYcS+^siVN;*&^&QAgu7;Nx8Tmkf4}zk z$)8E5$Wy=Nk+(*NLqX}}gSbo3#*STZ9NJ*`YFBmW@-@->cAt=B14htAY1MJ301%CJ zwI$MGCh1X|75Q!G?DO+>$2m$l^C(0wB)VB#yo=OVjJYlcGK+ zEi0-zE=4dE+|b3KY>pxH zuUNx7nGR)O9ss&Znl+n`awi+jtgT~_{8<%QyY`)7?B1D%?h{{sSuyNK+9Z`$4lvzgZcuW5@uXI|F&MeEq)FH0S1-pOGX<*~PuPo}SZk+^sY~S@ z0vE$8;B`DL&LcBXThz|9*(O~;-UM~JX&2S#+y07-H&R`C^Elm~o}Qv(nmNV+jes)z z3{nh=0L=t|IJ3OJo0hqrp239a?@G- z&s&~^_be@{w-qC(mh|{^QKxAust@%GknbZreWaB()J_gnAABd|tG+CXx8i|-tN_S1 z)U#IKrH_9xgg2YK{@}zHRyqPTpt_B;Ii2dl*+rG0xsBzB zZ%Nyv6}N12);~(2`<_M>&~v+Azkl~@?P|fUeDZ3eC#*?{W5sZbDcaM5VrE;p7n~^$ zjaq7&Z2A+43a%nD9V=@HiZNV~xbs$v9ogRNin9o6=y|Kvlrf{eDWN0Uh)PP6LpPZ; zE?(P|Qj@XmsKw$aq#e(Nw@U<{_md~UhBo7xa6xg^w4<2F(+G0|4oO595EgV^gdmHA zks#0AB@vAt2fHaPh5b~nhg$@DJ?hoN{a*8S>kE6mxrRv_F^G*RqL4)Ot{i=>+Xx;& zWSb?r^ItaxFN7XOf8|!7O;+>e8EN*u>FM^?EP`8)+Hk)u|7)>QIDrkuaXh}-+9_}| zVK;RHtXaXC6bwMAmLmtfDMb}*Fn%Lcq)(wI4_jaF$MZ-E9m6*Et)}2VZ>tf2u}z7a zvfO{p&!ZQ&@n)ldW$g3lnVw`Y5Y zz$dJY8KmhG;KxN$Z%$Saz|)UXx~#YPGLe2|wub?V0zg#)bCBGeCM4Px`f?$2uP zR*wn;UXz_jnsx?;-j(Gt#g%*h93tad{=IYOuH{~hG)ygS@uI5g74tW|Iax@dGuLb; z?9DE%*#F68X8FErvn{BZzo<=Kg&!R<4vtB3!Sao}Nr<0=LNr-}u!9g=`1tRWf)9zd zPiQ#BAl-;QE+11=TWz;eCDAecu%oYypD#bM0drJ|`f)_ z**0tB3{4ieV#o^>sZ7<{e*aeq0d#G3JU-1xk-m@W>qI=sFM6fl(G2Ze`_uD(>_5y1 z5-I?8mKK~YYgfFiOVYfW5aOOaW^!HVDJ5%|D32UE*Pz)QvjK61w50C1=t(6jo-0?h z$m@}!OC;Ji9~NW~f8mOic+PStXx7}auYJ#m&-rdR!vcY2!)w-`RH`z>WFB?gSwm%yK_Q1 z&tY-o{MOP=lFqvIju`}S`e$;$%i*gmlC@r?6~27Lx-&sQ4v<^ zfD+oxIYr}ASIk)qq%$cX7e0bE(sCSj=YnRs22Dx0~gKA2V z@V6yJOq##Gl0~cR$|G|y`+iP3$2sUr6A?Vfo({=Z%A7o82{f#NMhjg*BR{+b6_d4% zwBu0CY7;6_je3Wof_~+rkFS;M$hZ~cbvCBW0A0C7-frPwz>UNO!}C>q)sSJe&C*0Q zY%(jXE0fxtt9HB94D-U!t%#kLAwsa&xhGi(fwgsV8TuPYPX+xgxLUy9#O;XbmAK^W zO;NYU^6ZZrR7ftqb1z2}Wau#qD9 zdqj~q*RO1`A|36-n)j-+6X)AIO(tQWJg(nw6SWPTgD-sqk)p6WS55wRRciMngjKlP z^JG%A_U!u~en zsX9t_g2b8)9UvWPyRwKTSKi6m+&c_-GB{U!xmb@kUmU~6`KIr4^)aQJ)fN3glGT&As3lr&c0^Yt%N~DEa&G2I z69YWoXeW^6ghkB^Sr|UGQo1_f8VCvo;w)wRY^r5dDR7Zsjc!O&aA70scP*cV0Rp!_z+M0Q#%s?rvcvAL_wCh&)+$wi$|V*5bw$E<_3I6hUc6Viy^-po4Uduo zpibCR!f}jEciUvA^p(ue!@?%3&l&2(yn@!pWT67_r`AS119;0!r|Aqb#*?5(aN$go zq-k-h>)SR&ojJTjA1FF^esFbH(9Jb|Mh~V<+6B7ot%2(z|7_WRlP1T*#la!ygY+ae zqSxw)uv4SLpGiiUcFrtyljiwZUP6Qtyn===#Rw6)yCQ2WFY!!8<@_fhv< z#Fhbpl^k1d3pn$(3sS_=<<&IycSHx+irei!JhGsRVxj^rqcGhP^jk=fY#iheaJb!< z?B-52EF2d5LwEQUtc3;l^X}44T(Hd99-^$Gy&)%;MooY>NC}JTLN#aZg2q^!R4Kit z#3RQVm{)z+l{DJlX+-xjM@yQVRNzI428mY8;zO!N@VwUgk7drJ_YA7^{7qu0@SBs@{Uizd!>F1_)hT#pszTRQ)3J=yT4i@$<*R%BW zA!RvS7Y2UKKA<&fPmHVLE1W%jW$C5m?4d##yWpJ|me6)fkdRfvt?TKlsv;tsWzR(- z-^i0)NOh2b8d-tkz4#VB>t3?e<~}=Ntmf8IP!bFDu?(M8E8AZO;M(9RLnB=UA}Es}F^+zLuffSo)4(4A(2$^zM2M%vxtD22~A_5#v3l^W|? zD+A747%|bLs7}5#Cb~8Jhk#0*X6`@)x!-bC=ZjSkbxQk1v7mrc$>;q8{D*;FBqLRv z5i>!&CejoHr@o%;w%bAp?^uZg-vZ`ZR^}r$*<7z!Lql1(JPF~*tfGeQ3?$Hwbba4| zO!i$HAfRd>N_LjJzX5@=;-x?wi=nSvt6+RJW}R5r29*50V{QA;E=(918JP^a#yfx` zUp@9nmY}V8b)g%Ox~S$RH~sg z>&w3Bd6Ovy2;i=E6oxNLQ?76=h=w05SaB@)tH_jB@rTjz8yNmBQ~i+M>Ohwrxxwfq{I1R!EaHZVu;`q)WH|pD0KAA`s153U!BQ}(@{X>Fcsfs`O8bI^+WF&` zT)RyFJX!IDU*_OKELx|1C>SPk;ZGeh;T&rd!02~kF#wK7kvyX#3q5vpv+8AE*oJOb3KJL3XP#7-Z#;Z%xjr6++7oh1D1p6{hr9^Nkk39B zk7<0Tn{2ip^)4yR-WHva`@hKRJP}*$JTbJ{s?SNeJEC%3ZwL@q{bwa07KzMvbF})oHD2t@(eN+$9{7wQ3*VQ*BYT!njg!E$w%d zpfLJ{-BU6;QLM>tk`AuS-Zd+d^?&W1`Y^T#b6bP%ehO974-Ii95{Uw7nx_Rf>U3}l z?obMACw_aVxD|k3f?GQWIPQ3PTvpR7zudDXlW>tP6490gd!oxkE52CXyF8;P8e@3S z-uxBIE=ps(8C#v}bqV{9kL4NURX?RUAx7at`H8bM@Y(c_v}h-JlUN4s7Oq; z$?)j550jX)kS3OP44x^8+ji*J;(OH{kJhirXo#;9M^1rb%&rxjG>)4j>+B@s-aX^Y z;mAFXY!?e!^GAamG8MRp6S=O*)-D=FMH(T~v^$HR2S##T#hP;AG6UuE5CsLAkH5%O0*Vom!zyfrP-`n_4^Cu zSRcZQ8ZKhIoZQW2TYz%{)U+yY-Wt%bzT`!T)QD71@uIIuTi|{3yQq<8uF;5` zwUh5;sDSYu9iQU_mm=6qQdmGx0RiBqCcUy%zI9aC9$0ic@%S2%|UBo*7YxDiwk zOS^PgyJ{6aT+YmgcSjGN&w_Vq%`t}yc9YT?8sR3JiT%uL%%g&jBltzf3#;^S~J(=41Q>oSOG@O-TxNdI>F65|2$K+=`r8UCG z+F%N2;M>p1-OBNM3|dW?c`{(ZDzQFfGZvgmhHPS!__L72r&4auP3EBqmhY>QgH}?o z?%PM+VhiXa&(P#h!4dVFt0`Q$p%c#EtcqD3Qg4mk&n#dj8T_M1*h6IZRa)zPnw24P zi)0vIVj*cJsB&26VX z%_6ROQdmZKJ}!xi=x<1l1v_dIZ1qMoo7oukAt_-L;=j`q^h6ozi4%(j$Ak6s%TsCF zQFV&`Ly|8fmtJ8F4OCb_p$0KW72ph6^)OHGA^Ag5!5uF5fivHW4^Coxf7D;> z;n14K%(=#hUI2p6X-uB;12{8DzvU*S$UUT~F{o%$>qgS=Lt$oAZLL!aI9zMDbMVlj zfFCJZF;b}{L~x;#wa8gX+9w0?&4~KOYc{(jcW$@XK1JS3w+qQciVGrsHO}f=NiP6E zWXD3&vRIbu7uOE|E^VHQdrll2 z5T%B9mPrg*bB9}UB{9JIl3&A=|BL#+xtQ0p18 z)xdL4_L3!tD#JS31?a-TJ2&A9ci$UaXd zOPa^SvpyKm^6>=QraQ1gq;;lKP|;`6i-~xrtE@_wxe+la*mfq2CCizbzNAn}J&rYA zzI+Sij)|l!v|G(V{`Hm+0eBEBs>%I>H=@CPNJskzMMt~oG2W&)<942gukS!q!URoWx3GTaT6SsUOQnb2BVU9<`iVBWH zv}UIGg&>!?&4%tF2Pk{Grr_-$O--Za-c*piIoZ*{%2aDy;hsKji%97^^RLZ?y*rje5mF~ zuW)5KwKxst7rvO$%_N&Te@&!Rov)+&3rSY%s420h-^~D&S=8qQeNG|U4M~Mov!}0L z)(>wBA%+6ihfwRSW-$%;)#|Um_yZv({byBV?e6RD%P-N1Tvkz-a1u}l@++v%IfP!P z5qe!k;PqQ{dtbTGl}sH5*V|_eiJxz~Z~yt*ViLrfOhrv4`MWy$$}y!H+NR`X+g;lz zLweGU`SKU44O~s^EEI^@R}Pgp!u3%|{cfmjVnAQ6ryP{OPbbyu;;SE-?(csNR#aqf z+6yl)j};2Lu3|assMZWOp@|oQYND;E zAodrxT?7DKimZ}M3NkFo(ElPol@;GE#)x!Rey#M(tzXSP0Re#_qGf3G2U;Klt^qRY< z@b(UYy*45d_AlUCyT~>Zm4Q2bqE9LFuCaT}E{v{4xrIO%=P((<9)1UaRZcs2fKL|k zr-20`febF#xbE+4Lqpxh=Ay(XCMWDGkxs5}{KPN0>)dAb6;h7M_6_VM;%icDm#Ftz zYE!uegVV=n&A$Git6d6!;Z<_oqaTOdy>1`Hi2RPs8q5+uCJd#>=-e%n&v%SvNU>Cs zzG)X5^iJP3LpaV!L;2m=ojUM7vpMOUMQo;H@#RTF7g_Qa@OwtPAx#=bLulI7q+vhJ zm~&v-VEmO9g}U4fQm{)-RmGQ#Td-Rvpe92*&L#83?*fFrAwjumLA}YZXWvw>SsX}2 z=t6Xjs60yC5ygk4Z!TVs?tIi04S!}?n4Yr5SUX7(5Kr!RHJLP7X*0X4z0zlj?XjS5 zLz2k6;c#5^VDv3B#&Pbnt!+wL>?IqFn9kEb(a*<6xXZyvvb| z-M2|&8Kw`v7ENL7N?R-6ggHyJvp8|uCGuD9|Jwdb@Qxz37ayfsC=!2UP5;qKZwxEL zDV&OZrL1yQIZ1C+!u`=n6vIes+j@{1Rxw(S*65>&FT2vO9c$O~gO9dR+zpA9?V|c1 z`;&Mx&jEwiGFDuj9o!})xNbqA;sw+F=yNbQsGJXVu({jer=gA99x?FbHzYG2e}5kn>W(KEt1OCDtZ%8$-HCQ3SURDCFXJaP-QBWc{DYt zp@Y6B#jvb0j44==Rj3`fF}Zm4(Vu6xl)GC!+kV@*cVZD*U@Ylj5UvX+8hu0FB-a3t zk#kiUx`Dl^?@$#VFm;f(U8e%$~iTl!7%xpG%<$r|a9B$NS=& ztxZ|omV0GHu3Mpad>GBPlQZ)V&`e_mcG&eB5oHXip}AK3ENZrt^!ib5Yt$+hHr^G^ zN*f5V6DLo%D(cnlKgRNaqu~O6Szl? zc3evP@EC$x#7M5D`mwplSUI&;M|-}!KLaDP@-yY^SXxTX2hlKqjx_0AA$KV?WP z61slEYD1*D@0gn1nCm{81S>Yoy?fZ%);VK-JBSA$A%Z6+yEpJNEfl5Ip7$wbY{-&Z zZ!z?A1TCXmt7ledA?2z&a#IknY&7^o0VWzsf-Ufn6a{UbYtBm>*7F(@N;)EFB`LMg0)D z8;{SCUfJylSwT^HsYm4&FoIwo1l++JE&L|5Xw&}{8SjV%qs%x;X3hre?t>xb&g$sY zCKmzVs@nPvY7VeH@f|z4oLvQJj$_R^8Kx|1YQOPROuI;&i7vJi{aPoyaBF z!x)3Uaqdx|aTb9D-4Ie@c|-vRS+WEf88*1=x%-`Tx(iXj&5<1alT)6cbFqF7;i;~I zPMUV_W3|st#sVtP)s9U?27m`2NdDeN|Bb;%HP=xqV#%_hMEUQ+EX} zx=M4*OKm1&4GM-$vDU#?u`ZBBg3zO!8RIiROW7K>tK{JFtLDdB*^wrd;_ZzTObw(g zRno58aC%^J*YOP!ykuj)B$N^ri@Y}*EZIuOf8v>>Q|m;G?(9ly)+ z&}~zcXXK?s_?M>@+~V<3g%-U2?(4IULIV}$D%XPNiGCAy zmPC36>XA4ia=2|~&ukESWI&O>rPmyr9~y<&cR9mwL;(6hM9f4Fd&ES&790RKBNZp; zVPctU&yW~VR;Ga0TC{)AAAWrK`t`S8UN}=R{jZlVFJcJbzc?pI{!?pD}c8 z!hC%elrpkocWDR^#5?dF2mtWG`e9FQf}^HX#?dp3WD&{-@DBP+)_@_#A$eXZ0e$py zAX|3psDnyNzzBz~*)|2JD*rPLTbM(AVj2PR%H?QB-BROsB zXkt6CPO^NV+dS17yi4q@3RXM(PpQXLs|hd?fWUr@;CpsHB%0|rnjerf zPnfRa2FSs7hV5-!(eO1pS4h?rl?Yjl+S)?<1p3?oGJ95iHgZTc?6jLO zim)4L%F;Fcqy3!VVse!{okbmY=#=;j#h31-qnB86kgzX7s=yeL4_FSQ2l3;xlQhVg zlI3Fta;t74p?(Q3MOnYhSm3zR2a><#j6V^`&TP|rS!krBpkqXH2OW%BF4Q`s$w>+8 kbVeQXd48%)?w}NPCvWE)DOas)x^1kgKbV6qpsoV|02s*iGXMYp literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..b1adb8a3ef80fd0229ee09294505ebe679278c79 GIT binary patch literal 404 zcmV;F0c-vriwFP!00002|E-cuZ`&{ohVT9rJT?d0an~##J#_;ArcJEBg_{N19aJW3)T_#66suM z)B89j=jUp}Cbe=-$ndnsIO#SfX;p3-_bS^v7A2HavIz>7PA#$B)d$%tK z7mP32a@~^=O<(0!zx(tt2u0r}Zvy_K)vr>2*vxAG!{e&67{CV?96zCjNBv59ghX+f z@z)$;5;wUYXup9N8c}X|1uj4{!Ft0>H+;?^%y@J2Wk}y=lD1cHBd$|;Ikzg>BXf(l zSP@^z6{uuU=<;GvTkP_vXZkCx5;t#mdpqp!Zg-om>9(t)pckNKK`Q10Y$nrrY&lPp zp;w8+AtAA2LHl||T5m|WJvn)bSz7O&QL}`j7V0WE-Mgx)ARdd0xOQYO2TfFJnhZ1A ydsxF@bc{>n#t`WePG~qUakHIwSyb49`rPZ=%a52RT)o+GQTzlW(YEvk0{{TqXuw$j literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..1445bf431be238efb94e606a1b7b4759ba77f5f5 GIT binary patch literal 542 zcmV+(0^$81iwFP!00002|D9AzkJB&^zV}!7xO+&_7ph=S3le7@M})jiJV~rLw%0Rm zRDM0!GK@(mnt+me-gW(N>Ygb4%a$a+_?KGa0MztSgXNiGswd;LpBDPe ze#K+P;3I*S&4{v%z$C6Yagh=3L68rU{RM9v-3{Wo5}b#`#V3Z&-mehW9M3s3hI^$^ zawQO04;aER19+!ORIY=+s9>*hZ$_^{YR`$zlWh7qdxq*uLtm3}lrO Date: Mon, 28 Sep 2026 17:13:55 +0000 Subject: [PATCH 11/25] docs: record shared execution directly after the SDS foundation --- docs/design_docs/asapplanner-migration-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_docs/asapplanner-migration-plan.md b/docs/design_docs/asapplanner-migration-plan.md index afbd49ca4..78b909e68 100644 --- a/docs/design_docs/asapplanner-migration-plan.md +++ b/docs/design_docs/asapplanner-migration-plan.md @@ -183,6 +183,6 @@ each stage lands; then update the APIs, persistence descriptions and test eviden in the same implementation PR. The open shared-library integration PR is #774, replacing the already merged -#770. The active order after #771 is #774 → #763 → #765 → #761 → #728 +#770. The active order after #749 is #774 → #763 → #765 → #761 → #728 → #742 → #775. Real-evidence work in #776, #777, #778 and #759 is deferred; old #770 base metadata is not part of this chain. From e627dbb329dabcff5a8d18e8796da2b0111efc01 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 17:27:21 +0000 Subject: [PATCH 12/25] fix(control-plane): supply dataset_identity in the API test fixtures CompileAndPublishPhysicalPlanRequest gained a required `dataset_identity` field in this branch, but two api_tests fixtures build their request body as JSON by hand and were never updated, so both failed deserialization with `missing field dataset_identity` before reaching the handler. Take the value from the planning snapshot's own `environment.dataset_identity` rather than inventing one, so the fixture keeps describing the same dataset the rest of the snapshot describes. Co-Authored-By: Claude Opus 5 (1M context) --- control_plane/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index d0c759a17..66c47cac0 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -854,6 +854,7 @@ mod api_tests { "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": snapshot.physical_inputs.window_cost_model }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, @@ -931,6 +932,7 @@ mod api_tests { "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": [], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, From 890b69c45ba900561eeb3bcb1999bd231c0f6551 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 17:31:15 +0000 Subject: [PATCH 13/25] Remove obsolete SDS wire aliases and persisted metadata migration --- crates/asap_types/src/aggregation_config.rs | 58 +++------ crates/asap_types/src/precompute_plan.rs | 23 +--- crates/asap_types/src/producer_plan.rs | 14 +- crates/asap_types/src/query_plan.rs | 10 +- crates/asap_types/src/sds.rs | 65 ++------- .../storage_engines/sketch_db/index/mod.rs | 1 - .../sketch_db/persistence/metadata.rs | 123 +++++------------- .../continuous-summary-completeness.md | 2 +- .../catalog-physical-plan-runtime.md | 8 +- 9 files changed, 78 insertions(+), 226 deletions(-) diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 51f9813e9..8a083e52b 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -91,6 +91,7 @@ impl WindowMaterializationLayout { /// This descriptor cannot authorize execution: the enclosing PrecomputePlan /// must bind it to a compatible Planner DAG producer. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct PrecomputeMaterialization { /// Explicit deployment output allocation; independent of semantic identity. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -121,13 +122,9 @@ pub struct PrecomputeMaterialization { pub window_type: WindowKind, // Tumbling or Sliding pub window_layout: WindowMaterializationLayout, /// Unix millisecond timestamp on the pane-boundary grid selected from - /// the consuming query workload. Missing on legacy definitions, which - /// must not be used for certified pane-only reads. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + /// the consuming query workload. An absent origin cannot authorize certified + /// pane-only reads. + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, pub spatial_filter: String, @@ -137,26 +134,12 @@ pub struct PrecomputeMaterialization { // SQL-specific fields (optional, used when query_language=sql) pub table_name: Option, // SQL mode: table name - #[serde( - default, - alias = "value_column", - alias = "valueColumn", - alias = "valueProjection", - deserialize_with = "crate::sds::deserialize_optional_value_projection" - )] + #[serde(default)] pub value_projection: Option, /// Table timestamp projection, in Unix milliseconds. - #[serde( - default, - alias = "tableTimestampColumn", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub table_timestamp_column: Option, - #[serde( - default, - alias = "tablePopulation", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub table_population: Option, /// Producer typing for a SQL table value projection: the source column's /// declared type and nullability. @@ -173,11 +156,7 @@ pub struct PrecomputeMaterialization { /// /// `None` ⇒ PromQL-mode materializations and legacy SQL definitions, which /// keep the pre-typed behaviour (read the column as it comes). - #[serde( - default, - alias = "valueSourceColumn", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub value_source_column: Option, } @@ -936,17 +915,10 @@ mod tests { .as_object_mut() .unwrap() .insert("paneOriginMs".into(), origin); - let decoded: PrecomputeMaterialization = serde_json::from_value(derived.clone()).unwrap(); + assert!(serde_json::from_value::(derived).is_err()); + let decoded: PrecomputeMaterialization = + serde_json::from_value(serde_json::to_value(&epoch).unwrap()).unwrap(); assert_eq!(decoded.pane_origin_ms, Some(7_000)); - - let mut legacy = derived; - legacy.as_object_mut().unwrap().remove("paneOriginMs"); - assert_eq!( - serde_json::from_value::(legacy) - .expect("decode legacy wire") - .pane_origin_ms, - None - ); } /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. @@ -978,7 +950,7 @@ mod tests { } #[test] - fn typed_projection_roundtrips_and_legacy_column_keeps_identity() { + fn typed_projection_roundtrips_and_untyped_column_is_rejected() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; let mut config = PrecomputeMaterialization::from_yaml_data( @@ -995,8 +967,10 @@ mod tests { let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); - let decoded: PrecomputeMaterialization = serde_json::from_value(legacy).unwrap(); - assert_eq!(decoded.policy_fingerprint(), column_identity); + assert!(serde_json::from_value::(legacy).is_err()); + let mut untyped = serde_json::to_value(&config).unwrap(); + untyped["value_projection"] = serde_json::json!("value"); + assert!(serde_json::from_value::(untyped).is_err()); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), }); diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index ded2fd6f4..f360ab4b9 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -144,7 +144,6 @@ pub struct IngestContract { pub endpoint_path: String, pub timestamp_unit: TimestampUnit, pub require_plan_identity: bool, - #[serde(alias = "require_materialization_identity")] pub require_stored_output_identity: bool, pub require_registered_producer: bool, } @@ -221,17 +220,12 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StateSchemaContract { - #[serde(alias = "state_reference")] pub stored_output_reference: crate::sds::StoredOutputReference, pub schema_id: String, pub schema_version: u32, pub materialization: crate::sds::StoredOutputId, pub family: StateFamilyContract, pub source: Source, - #[serde( - alias = "value_column", - deserialize_with = "crate::sds::deserialize_state_value_projection" - )] pub value_projection: crate::sds::ValueProjectionIdentity, pub group_by: crate::GroupingProjection, pub window: StateWindowContract, @@ -244,11 +238,7 @@ pub struct StateWindowContract { pub kind: crate::WindowKind, pub size_ms: u64, pub slide_ms: Option, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, } @@ -261,9 +251,8 @@ pub struct ProducerContract { pub collector_id: String, pub materialization: crate::sds::StoredOutputId, pub schema_id: String, - /// Authoritative partitions for completion barriers. Empty legacy contracts - /// authorize state ingestion only, never completion claims. - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + /// Authoritative partitions for completion barriers. An explicitly empty + /// roster cannot authorize completion claims. pub partition_ids: BTreeSet, } @@ -1003,9 +992,11 @@ mod source_window_cohort_tests { "producer_id":"p", "collector_id":"c", "materialization":materialization, "schema_id":plan.schemas[0].schema_id, }); - let producer: ProducerContract = serde_json::from_value(legacy.clone()).unwrap(); + assert!(serde_json::from_value::(legacy.clone()).is_err()); + let mut current = legacy; + current["partition_ids"] = serde_json::json!([]); + let producer: ProducerContract = serde_json::from_value(current).unwrap(); assert!(producer.partition_ids.is_empty()); - assert_eq!(serde_json::to_value(&producer).unwrap(), legacy); plan.producers.push(producer); let barrier = crate::sds::SummaryWatermarkBarrier { catalog_generation: generation, diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 1046028fd..846c4df83 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -8,6 +8,7 @@ use std::collections::BTreeSet; use thiserror::Error; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] pub struct CollectorMaterialization { pub query_id: String, pub materialization: crate::sds::StoredOutputId, @@ -17,14 +18,10 @@ pub struct CollectorMaterialization { pub group_by: Vec, pub window_secs: u64, pub abstract_window_framework: SummaryWindowFramework, - #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + #[serde(rename = "window_implementation_id")] pub window_realization_id: String, pub slide_secs: u64, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, pub window_layout: crate::WindowMaterializationLayout, pub evidence_source: Option, @@ -41,8 +38,9 @@ pub struct CollectorLifecycle { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] pub struct CollectorPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. + /// Set when binding the plan; required for catalog-authorized installation. #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, pub collector_id: String, @@ -205,7 +203,7 @@ pub struct RuntimeAdaptationEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct TransmissionPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. + /// Set when binding the plan; required for catalog-authorized installation. #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, pub envelope: PlanEnvelope, diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 445f21daa..f4eb6e76d 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -291,7 +291,6 @@ pub struct QueryPlanEntry { #[serde(default)] pub language: QueryLanguage, pub query_id: String, - #[serde(alias = "canonical_promql")] pub canonical_query: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub fixed_evaluation: Option, @@ -474,7 +473,6 @@ pub enum FallbackPolicy { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MaterializationBinding { - #[serde(alias = "state_reference")] pub stored_output_reference: crate::sds::StoredOutputReference, /// Complete-window storage advances independently of its stored extent. /// None denotes disjoint pane storage. @@ -484,16 +482,12 @@ pub struct MaterializationBinding { /// Query operator grouping applied while folding those SIDs. pub output_grouping: PhysicalGrouping, /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub item_labels: Vec, pub window_ms: u64, /// Unix millisecond timestamp on the materialized pane-boundary grid. /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, /// Semantic query lookback, independent of the physical pane duration. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 752d49a45..9433816d1 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -125,7 +125,6 @@ pub struct CatalogGeneration { pub schema_version: u32, pub plan_id: u64, pub plan_version: u64, - #[serde(alias = "snapshot_digest")] pub snapshot_sha256: String, } @@ -317,7 +316,6 @@ pub enum InstanceLifecycle { #[serde(deny_unknown_fields)] pub struct SummaryInstance { pub instance_id: SummaryInstanceId, - #[serde(alias = "state_slot_id")] pub stored_output_id: StoredOutputId, pub summary_definition_id: SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, @@ -325,9 +323,6 @@ pub struct SummaryInstance { pub time_range: HalfOpenTimeRange, pub group_values: BTreeMap, pub catalog_generation: CatalogGeneration, - /// Reserved wire field. Cross-version state adoption is not supported. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reused_from_generation: Option, pub placement: SummaryPlacement, pub state_reference: SummaryStateReference, pub status: SummaryInstanceStatus, @@ -356,11 +351,6 @@ impl SummaryInstance { "summary instance placement must be resolved".into(), )); } - if self.reused_from_generation.is_some() { - return Err(SdsError( - "cross-version state adoption is not supported".into(), - )); - } if self.state_reference.store.is_empty() || self.state_reference.key.is_empty() || self.state_reference.state_schema_version == 0 @@ -934,33 +924,6 @@ impl ValueProjectionIdentity { } } -/// Compatibility adapter for old config column strings; storage is always typed. -pub(crate) fn deserialize_optional_value_projection<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - let value = Option::::deserialize(deserializer)?; - value - .map(|value| match value { - Value::String(name) => Ok(ValueProjectionIdentity::Column { name }), - value => serde_json::from_value(value).map_err(serde::de::Error::custom), - }) - .transpose() -} - -/// Read legacy StateSchema ColumnRef values without retaining a parallel field. -pub(crate) fn deserialize_state_value_projection<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result { - let value = Value::deserialize(deserializer)?; - if value == "SampleValue" { - return Ok(ValueProjectionIdentity::SampleValue); - } - if let Some(name) = value.get("Named").and_then(Value::as_str) { - return Ok(ValueProjectionIdentity::Column { name: name.into() }); - } - serde_json::from_value(value).map_err(serde::de::Error::custom) -} - /// Whether a materialization preserves source entities or pools a population. /// Grouped label names remain in `DataDescriptor::group_by_keys`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -1349,7 +1312,6 @@ mod tests { plan_version: 2, snapshot_sha256: "abc".into(), }, - reused_from_generation: None, placement: SummaryPlacement { producer_id: "producer".into(), storage_node_id: "store".into(), @@ -1424,17 +1386,11 @@ mod tests { #[test] fn new_generation_rejects_cross_version_payload_adoption() { let mut instance = observed_instance(InstanceLifecycle::Persistent); - let mut source = instance.catalog_generation.clone(); - source.plan_version -= 1; - instance.reused_from_generation = Some(source.clone()); - instance.state_reference.generation = source.plan_version; - assert!( - instance.validate().is_err(), - "cross-version adoption must be rejected" - ); - instance.reused_from_generation.as_mut().unwrap().plan_id += 1; - assert!(instance.validate().is_err()); - instance.reused_from_generation = Some(instance.catalog_generation.clone()); + let mut wire = serde_json::to_value(&instance).unwrap(); + wire["reused_from_generation"] = + serde_json::to_value(&instance.catalog_generation).unwrap(); + assert!(serde_json::from_value::(wire).is_err()); + instance.state_reference.generation -= 1; assert!(instance.validate().is_err()); } @@ -1573,19 +1529,14 @@ mod tests { } #[test] - fn catalog_generation_accepts_legacy_digest_name() { - let generation: CatalogGeneration = serde_json::from_value(json!({ + fn catalog_generation_rejects_legacy_digest_name() { + assert!(serde_json::from_value::(json!({ "schema_version": 1, "plan_id": 2, "plan_version": 3, "snapshot_digest": "abc" })) - .unwrap(); - assert_eq!(generation.snapshot_sha256, "abc"); - assert!(serde_json::to_value(generation) - .unwrap() - .get("snapshot_digest") - .is_none()); + .is_err()); } #[test] fn wire_roundtrip_and_tampered_id_validation() { diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index becf81bb2..185dbcef6 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1286,7 +1286,6 @@ impl SketchStore { time_range: HalfOpenTimeRange { start_ms, end_ms }, group_values, catalog_generation: generation.clone(), - reused_from_generation: None, placement: SummaryPlacement { producer_id: producer_id.clone(), storage_node_id: storage_node_id.into(), diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 96a86d8d5..e6d8faa24 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -35,7 +35,7 @@ //! //! ## Format //! -//! A single JSON object `{ "": SidMetaRecord, ... }` written +//! A versioned JSON object containing descriptors and series bindings, written //! atomically (tmp + rename) on every upsert. JSON (not the custom //! binary part format) because the record count equals live sid //! cardinality (small) and the schema is human-inspectable for @@ -46,7 +46,7 @@ use std::collections::HashMap; use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -437,7 +437,7 @@ struct SidBindingRec { last_immutable: Option, } -/// Version-3 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold +/// Version-4 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold /// foreign keys, mirroring the in-memory SDS registry. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SdsSidecar { @@ -687,68 +687,10 @@ impl SidMetadataStore { &self.path } - /// Load every durable record. Returns an empty vec when the sidecar - /// doesn't exist yet (fresh dir, or parts written before this feature - /// landed) or when it is unparsable (treated as "no recoverable - /// metadata" — the live ingest path still re-registers on first DP). + /// Load current-format durable records. A missing file denotes a fresh store; + /// malformed or unsupported persisted metadata is an error. pub fn load(&self) -> PersistResult> { - let mut f = match File::open(&self.path) { - Ok(f) => f, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(e) => return Err(PersistError::Io(e)), - }; - let mut buf = String::new(); - f.read_to_string(&mut buf)?; - if buf.trim().is_empty() { - return Ok(Vec::new()); - } - let value: serde_json::Value = match serde_json::from_str(&buf) { - Ok(value) => value, - Err(e) => { - tracing::warn!( - path = %self.path.display(), - error = %e, - "sid metadata sidecar unparsable; ignoring (live ingest will re-register)" - ); - return Ok(Vec::new()); - } - }; - if matches!( - value.get("schema_version").and_then(|v| v.as_u64()), - Some(2..=4) - ) { - let sidecar: SdsSidecar = match serde_json::from_value(value) { - Ok(sidecar) => sidecar, - Err(error) => { - tracing::warn!( - path = %self.path.display(), - %error, - "SDS metadata sidecar is invalid; ignoring" - ); - return Ok(Vec::new()); - } - }; - return match sidecar.into_records() { - Ok(records) => Ok(records), - Err(error) => { - tracing::warn!( - path = %self.path.display(), - %error, - "SDS metadata sidecar has broken descriptor references; ignoring" - ); - Ok(Vec::new()) - } - }; - } - // Version 1 was a flat SeriesId map. Read it and normalize on the next write. - let map: HashMap = match serde_json::from_value(value) { - Ok(map) => map, - Err(error) => { - tracing::warn!(path = %self.path.display(), %error, "legacy sid metadata is invalid; ignoring"); - return Ok(Vec::new()); - } - }; - Ok(map.into_values().collect()) + self.load_strict() } /// Upsert a batch of records, merging with whatever is already on @@ -806,20 +748,18 @@ impl SidMetadataStore { }; let value: serde_json::Value = serde_json::from_slice(&bytes) .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; - if let Some(version) = value.get("schema_version") { - if !matches!(version.as_u64(), Some(2..=4)) { - return Err(PersistError::Format( - "unsupported SID metadata version".into(), - )); - } - let sidecar: SdsSidecar = serde_json::from_value(value) - .map_err(|error| PersistError::Format(error.to_string()))?; - sidecar.into_records() - } else { - let records: HashMap = serde_json::from_value(value) - .map_err(|error| PersistError::Format(error.to_string()))?; - Ok(records.into_values().collect()) + if value + .get("schema_version") + .and_then(serde_json::Value::as_u64) + != Some(4) + { + return Err(PersistError::Format( + "unsupported SID metadata version".into(), + )); } + let sidecar: SdsSidecar = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + sidecar.into_records() } pub(super) fn transaction( @@ -1024,22 +964,25 @@ mod tests { .remove(&missing_id); std::fs::write(store.path(), serde_json::to_vec(&persisted).unwrap()).unwrap(); - assert!(store.load().unwrap().is_empty()); + assert!(store.load().is_err()); } #[test] - fn legacy_flat_sidecar_is_read_and_migrated_on_write() { + fn unsupported_sidecars_are_rejected_without_overwriting() { let tmp = TempDir::new().unwrap(); let store = SidMetadataStore::new(tmp.path()); - let legacy = HashMap::from([("1".to_string(), sketch_meta(1))]); - std::fs::write(store.path(), serde_json::to_vec(&legacy).unwrap()).unwrap(); - - assert_eq!(store.load().unwrap(), vec![sketch_meta(1)]); - store.upsert_all(&[exact_meta(2)]).unwrap(); - let persisted: serde_json::Value = - serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 4); - assert_eq!(store.load().unwrap().len(), 2); + let flat = serde_json::to_value(HashMap::from([("1", sketch_meta(1))])).unwrap(); + let mut cases = vec![flat]; + for version in [1, 2, 3, 5, 999] { + cases.push(serde_json::json!({"schema_version": version})); + } + for value in cases { + let bytes = serde_json::to_vec(&value).unwrap(); + std::fs::write(store.path(), &bytes).unwrap(); + assert!(store.load().is_err()); + assert!(store.upsert_all(&[exact_meta(2)]).is_err()); + assert_eq!(std::fs::read(store.path()).unwrap(), bytes); + } } #[test] @@ -1077,10 +1020,10 @@ mod tests { } #[test] - fn unparsable_file_loads_as_empty() { + fn unparsable_file_is_rejected() { let tmp = TempDir::new().unwrap(); let s = SidMetadataStore::new(tmp.path()); std::fs::write(s.path(), b"{not json").unwrap(); - assert!(s.load().unwrap().is_empty()); + assert!(s.load().is_err()); } } diff --git a/docs/design_docs/continuous-summary-completeness.md b/docs/design_docs/continuous-summary-completeness.md index 34549b8df..040378591 100644 --- a/docs/design_docs/continuous-summary-completeness.md +++ b/docs/design_docs/continuous-summary-completeness.md @@ -10,6 +10,6 @@ Finite drain closes input, waits for all workers and certifies that every accept Admission metadata has bounded coordinate, pending-revision and byte budgets. Completed receipts expire with configured materialization retention; pending work and the published prefixes of pending admissions remain protected. Already admitted slow-worker outputs can finish behind another worker's maintenance replay frontier. Unsolicited expired input and expired untagged replay remain rejected. -This inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and version-3 persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and leaves incompatible or legacy records unbound under an authoritative catalog. Reuse across changed catalog generations requires a separate explicit compatibility decision. General multi-input maintenance transforms and durable producer watermarks remain separate work. +This inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and current-version persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and rejects unsupported persisted metadata formats. A changed catalog generation starts cold and requires fresh state; cross-generation adoption is not supported. General multi-input maintenance transforms and durable producer watermarks remain separate work. Durable SID bindings also preserve retirement and expiry timestamps and a removal tombstone. Lifecycle changes publish through the same serialized metadata writer as the flusher before changing in-memory visibility. A stale flush snapshot cannot clear those fields. Recovery leaves removed and expired instances unregistered and preserves a still-retired instance's expiry deadline. Tombstones remain until durable state is explicitly reclaimed; this change does not claim automatic tombstone garbage collection or cross-generation reactivation. diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 57b6f7f79..890ae627f 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -14,9 +14,11 @@ Derived outputs require the complete Planner closure at installation. Writes must match the installed output's operator and format. Durable metadata records both identities and the immutable catalog snapshot. Recovery validates -that snapshot before restoring authoritative state. Old payload decoders remain -available, but legacy metadata without semantic identity cannot authorize bound -reads. Reinstall/rebuild those outputs rather than guessing their meaning. +that snapshot before restoring authoritative state. Only the current metadata +schema is accepted. Unsupported versions, flat-map sidecars, corrupt metadata, +obsolete identity aliases, and untyped projection encodings are rejected. There +is no automatic format migration or cross-version state-adoption path. Rebuild +unsupported persisted state using the installed plan. See [SDS architecture](../../design_docs/summary-catalog-sds-architecture.md) for the contract and [migration gates](../../design_docs/asapplanner-migration-plan.md#5-bound-query-sds-implementation-across-the-pr-stack) From 53d560ec621944f58b6c51e2cd16ae54ce82bebb Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:32:35 +0000 Subject: [PATCH 14/25] Reject obsolete state-column syntax in compiler roundtrip coverage --- control_plane/src/physical/compiler.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 6e76b2fc1..cf9d6ea56 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -7012,8 +7012,7 @@ pub(crate) mod tests { schema.as_object_mut().unwrap().remove("value_projection"); schema["value_column"] = serde_json::json!("SampleValue"); } - let decoded: PrecomputePlan = serde_json::from_value(legacy).unwrap(); - decoded.validate_against_catalog(catalog).unwrap(); + assert!(serde_json::from_value::(legacy).is_err()); let reject = |mutated: PrecomputePlan| assert!(mutated.validate_against_catalog(catalog).is_err()); let mut bad = original.clone(); From d6f688fdfc3ef22b4cf1a121fb06094e47463f07 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:36:41 +0000 Subject: [PATCH 15/25] Remove unused materialization wire adapter and propagate recovery errors --- crates/asap_types/src/aggregation_config.rs | 272 ++---------------- crates/asap_types/src/policy_fingerprint.rs | 14 - .../storage_engines/sketch_db/index/mod.rs | 54 ++-- 3 files changed, 56 insertions(+), 284 deletions(-) diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 8a083e52b..8e7af6df0 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -5,7 +5,6 @@ use std::collections::HashMap; use crate::enums::{QueryLanguage, WindowKind}; use crate::policy_fingerprint::PolicyFingerprint; -use crate::traits::SerializableToSink; use crate::utils::normalize_spatial_filter; use crate::AggregationType; use crate::KeyByLabelNames; @@ -329,164 +328,6 @@ impl PrecomputeMaterialization { self.policy_fingerprint().as_u64() } - pub fn deserialize_from_json( - data: &Value, - ) -> Result> { - if [ - "valueColumn", - "value_column", - "valueProjection", - "value_projection", - ] - .iter() - .filter(|key| data.get(**key).is_some_and(|value| !value.is_null())) - .count() - > 1 - { - return Err("multiple value projection fields are not allowed".into()); - } - // `aggregationId` is silently ignored — identity is - // content-addressed via PolicyFingerprint (PR 5). - - let aggregation_type: AggregationType = data["aggregationType"] - .as_str() - .ok_or("Missing aggregationType")? - .parse() - .map_err(|e: String| e)?; - - let aggregation_sub_type = data["aggregationSubType"] - .as_str() - .ok_or("Missing aggregationSubType")? - .to_string(); - - let parameters = data["parameters"] - .as_object() - .ok_or("Missing parameters")? - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - // Note: In Python, eval(data["originalYaml"]) is used, but this is unsafe - // Using the string value directly instead - let original_yaml = data["originalYaml"].as_str().unwrap_or("").to_string(); - - // Deserialize KeyByLabelNames - assuming they have deserialize_from_json methods - let grouping_labels = - crate::GroupingProjection::deserialize_from_json(&data["groupingLabels"])?; - let aggregated_labels = KeyByLabelNames::deserialize_from_json(&data["aggregatedLabels"])?; - let rollup_labels = KeyByLabelNames::deserialize_from_json(&data["rollupLabels"])?; - - let window_size = data["windowSize"].as_u64().ok_or("Missing windowSize")?; - - let window_type = data - .get("windowType") - .and_then(|v| v.as_str()) - .unwrap_or("tumbling") - .parse::() - .unwrap_or_default(); - - let slide_interval = data - .get("slideInterval") - .and_then(|v| v.as_u64()) - .unwrap_or(window_size); - - let pane_origin_ms = data - .get("paneOriginMs") - .or_else(|| data.get("pane_origin_ms")) - .and_then(|v| v.as_i64()); - - let spatial_filter = data["spatialFilter"].as_str().unwrap_or("").to_string(); - - let metric = data["metric"].as_str().ok_or("Missing metric")?.to_string(); - - let num_aggregates_to_retain = data.get("numAggregatesToRetain").and_then(|v| v.as_u64()); - - // SQL-specific fields (optional) - let table_name = data - .get("tableName") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - let value_column = data - .get("valueColumn") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let mut config = Self::new( - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels, - aggregated_labels, - rollup_labels, - original_yaml, - window_size, - slide_interval, - window_type, - spatial_filter, - metric, - num_aggregates_to_retain, - table_name, - value_column, - ); - if data.get("windowLayout").is_some() && data.get("window_layout").is_some() { - return Err("multiple window layout fields are not allowed".into()); - } - if let Some(layout) = data - .get("windowLayout") - .or_else(|| data.get("window_layout")) - { - config.window_layout = serde_json::from_value(layout.clone())?; - config - .window_layout - .validate(config.window_size, config.slide_interval)?; - } - config.population_key_encoding = data - .get("population_key_encoding") - .map(|value| serde_json::from_value(value.clone())) - .transpose()? - .unwrap_or_default(); - config.derived_input = data - .get("derived_input") - .filter(|v| !v.is_null()) - .map(|v| serde_json::from_value(v.clone())) - .transpose()?; - config.partitioning = data - .get("partitioning") - .filter(|value| !value.is_null()) - .map(|value| serde_json::from_value(value.clone())) - .transpose()?; - if let Some(projection) = data - .get("valueProjection") - .or_else(|| data.get("value_projection")) - .filter(|value| !value.is_null()) - { - config.value_projection = Some(serde_json::from_value(projection.clone())?); - } - config.pane_origin_ms = pane_origin_ms; - config.table_timestamp_column = data - .get("tableTimestampColumn") - .or_else(|| data.get("table_timestamp_column")) - .and_then(Value::as_str) - .map(str::to_owned); - config.table_population = data - .get("tablePopulation") - .or_else(|| data.get("table_population")) - .filter(|value| !value.is_null()) - .cloned() - .map(serde_json::from_value) - .transpose()?; - config.population_filter_canonical()?; - Ok(config) - } - - pub fn deserialize_from_bytes( - bytes: &[u8], - ) -> Result> { - let data_str = std::str::from_utf8(bytes)?.trim(); - let data: Value = serde_json::from_str(data_str)?; - Self::deserialize_from_json(&data) - } - pub fn from_yaml_data( aggregation_data: &serde_yaml::Value, num_aggregates_to_retain: Option, @@ -698,60 +539,6 @@ impl PrecomputeMaterialization { } } -impl SerializableToSink for PrecomputeMaterialization { - fn serialize_to_json(&self) -> Value { - // PR 5: `aggregationId` is no longer emitted — readers derive it - // from content via `PolicyFingerprint::from_config(...).as_u64()`. - let mut json = serde_json::json!({ - "aggregationType": self.aggregation_type, - "aggregationSubType": self.aggregation_sub_type, - "parameters": self.parameters, - "partitioning": self.partitioning, - "originalYaml": self.original_yaml, - "windowSize": self.window_size, - "slideInterval": self.slide_interval, - "windowLayout": self.window_layout, - "windowType": self.window_type.to_string(), - "spatialFilter": self.spatial_filter, - "metric": self.metric, - }); - - if !self.population_key_encoding.is_legacy() { - json["population_key_encoding"] = serde_json::json!(self.population_key_encoding); - } - if let Some(input) = &self.derived_input { - json["derived_input"] = serde_json::json!(input); - } - // Only include numAggregatesToRetain if it's Some - if let Some(num_aggregates) = self.num_aggregates_to_retain { - json["numAggregatesToRetain"] = serde_json::json!(num_aggregates); - } - if let Some(pane_origin_ms) = self.pane_origin_ms { - json["paneOriginMs"] = serde_json::json!(pane_origin_ms); - } - - // SQL-specific fields (only include if present) - if let Some(ref table_name) = self.table_name { - json["tableName"] = serde_json::json!(table_name); - } - if let Some(ref projection) = self.value_projection { - json["valueProjection"] = serde_json::json!(projection); - } - if let Some(ref population) = self.table_population { - json["tablePopulation"] = serde_json::json!(population); - } - if let Some(ref column) = self.table_timestamp_column { - json["tableTimestampColumn"] = serde_json::json!(column); - } - - json - } - - fn serialize_to_bytes(&self) -> Vec { - self.original_yaml.as_bytes().to_vec() - } -} - #[cfg(test)] mod window_layout_tests { use super::WindowMaterializationLayout; @@ -863,17 +650,13 @@ mod tests { PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) .unwrap(); assert_eq!(config.window_layout, layout); - let mut wire = config.serialize_to_json(); - wire["groupingLabels"] = serde_json::to_value(&config.grouping_labels).unwrap(); - wire["aggregatedLabels"] = - serde_json::to_value(&config.aggregated_labels.labels).unwrap(); - wire["rollupLabels"] = serde_json::to_value(&config.rollup_labels.labels).unwrap(); - let decoded = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); + let mut wire = serde_json::to_value(&config).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(decoded.window_layout, layout); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); - wire["window_layout"] = wire["windowLayout"].clone(); - assert!(PrecomputeMaterialization::deserialize_from_json(&wire).is_err()); + wire["windowLayout"] = wire["window_layout"].clone(); + assert!(serde_json::from_value::(wire).is_err()); } yaml.as_mapping_mut() .unwrap() @@ -903,8 +686,8 @@ mod tests { let planned = epoch.policy_fingerprint(); assert_ne!(unknown, planned); - let wire = epoch.serialize_to_json(); - assert_eq!(wire["paneOriginMs"], serde_json::json!(7_000)); + let wire = serde_json::to_value(&epoch).unwrap(); + assert_eq!(wire["pane_origin_ms"], serde_json::json!(7_000)); let mut derived = serde_json::to_value(&epoch).unwrap(); let origin = derived .as_object_mut() @@ -935,18 +718,20 @@ mod tests { /// PR 5: `serialize_to_json` no longer emits `aggregationId`. #[test] - fn serialize_to_json_omits_aggregation_id() { + fn canonical_wire_rejects_aggregation_id() { let cfg = PrecomputeMaterialization::from_yaml_data( &sample_yaml(false), None, QueryLanguage::PromQl, ) .expect("parse"); - let json = cfg.serialize_to_json(); + let mut json = serde_json::to_value(&cfg).unwrap(); assert!( json.get("aggregationId").is_none(), "PR 5: aggregationId must not appear on the wire — readers derive it from content" ); + json["aggregationId"] = serde_json::json!(42); + assert!(serde_json::from_value::(json).is_err()); } #[test] @@ -974,38 +759,25 @@ mod tests { config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), }); - let mut wire = config.serialize_to_json(); - // The legacy JSON and YAML readers receive their labels from the - // enclosing streaming config, in their respective wire shapes. - wire["groupingLabels"] = config.grouping_labels.serialize_to_json(); - wire["aggregatedLabels"] = config.aggregated_labels.serialize_to_json(); - wire["rollupLabels"] = config.rollup_labels.serialize_to_json(); - wire["labels"] = serde_json::json!({ - "grouping": config.grouping_labels.serialize_to_json(), - "aggregated": config.aggregated_labels.serialize_to_json(), - "rollup": config.rollup_labels.serialize_to_json(), - }); - assert!(wire.get("valueColumn").is_none()); - let json = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); - let yaml = PrecomputeMaterialization::from_yaml_data( - &serde_yaml::to_value(&wire).unwrap(), - None, - QueryLanguage::ClickHouseSql, - ) - .unwrap(); + let wire = serde_json::to_value(&config).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!( - json.effective_value_projection(), + decoded.effective_value_projection(), config.effective_value_projection() ); + let mut yaml = sample_yaml(false); + yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); + yaml["valueProjection"] = serde_yaml::to_value(&config.value_projection).unwrap(); + let decoded = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) + .unwrap(); assert_eq!( - yaml.effective_value_projection(), + decoded.effective_value_projection(), config.effective_value_projection() ); - let mut conflicting = wire; - conflicting["valueColumn"] = serde_json::json!("other_column"); - assert!(PrecomputeMaterialization::deserialize_from_json(&conflicting).is_err()); + yaml["valueColumn"] = serde_yaml::to_value("other_column").unwrap(); assert!(PrecomputeMaterialization::from_yaml_data( - &serde_yaml::to_value(conflicting).unwrap(), + &yaml, None, QueryLanguage::ClickHouseSql ) diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index 02d55a5fc..42fa2e475 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -269,20 +269,6 @@ mod tests { assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); - use crate::traits::SerializableToSink; - let mut sink = canonical.serialize_to_json(); - // Transport wrappers supply the three label projections separately. - sink["groupingLabels"] = serde_json::to_value(&canonical.grouping_labels).unwrap(); - sink["aggregatedLabels"] = - serde_json::to_value(&canonical.aggregated_labels.labels).unwrap(); - sink["rollupLabels"] = serde_json::to_value(&canonical.rollup_labels.labels).unwrap(); - let decoded = PrecomputeMaterialization::deserialize_from_json(&sink).unwrap(); - assert_eq!( - decoded.population_key_encoding, - canonical.population_key_encoding - ); - - assert!(serde_json::from_str::("\"canonical_labels_v2\"").is_err()); } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 185dbcef6..57fc7c953 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -3305,7 +3305,7 @@ impl SketchStore { // early-returns on the missing `sid_group_by_keys` → "No result" // cluster-wide even though the data is durable on disk. Idempotent: // sids already registered (e.g. by an in-flight DataPoint) are kept. - let recovered = self.register_recovered_disk_series(&cfg.disk_path); + let recovered = self.register_recovered_disk_series(&cfg.disk_path)?; if recovered > 0 { tracing::info!( recovered_sids = recovered, @@ -3354,22 +3354,14 @@ impl SketchStore { /// metadata is authoritative, so we don't clobber it). Returns the /// number of sids freshly registered from disk. /// - /// `capability` / `accuracy` are re-derived from the persisted - /// `agg_kind` exactly as the ingest path derives them. The sidecar is - /// missing only for parts written before this feature landed (or a - /// fresh dir) — those sids stay invisible until a live DataPoint - /// re-registers them, the same as pre-fix behavior. - pub fn register_recovered_disk_series(&self, disk_path: &std::path::Path) -> usize { + /// Persisted metadata errors propagate to startup. Only an absent file is + /// treated as a fresh store. + pub fn register_recovered_disk_series( + &self, + disk_path: &std::path::Path, + ) -> persistence::PersistResult { use crate::storage_engines::sketch_db::index::persistence::metadata::SidMetadataStore; - - let store = SidMetadataStore::new(disk_path); - let records = match store.load() { - Ok(r) => r, - Err(e) => { - tracing::warn!(error = %e, "failed to load sid metadata sidecar on recovery"); - return 0; - } - }; + let records = SidMetadataStore::new(disk_path).load()?; let mut registered = 0usize; for rec in records { @@ -3452,7 +3444,7 @@ impl SketchStore { registered += 1; } } - registered + Ok(registered) } /// Switch the store into durable-tier mode: install the read handle @@ -3777,6 +3769,26 @@ mod tests { assert!(!before.matches(store.summary_update_revision())); } + #[test] + fn recovery_rejects_unsupported_or_corrupt_metadata() { + let directory = tempfile::tempdir().unwrap(); + let store = SketchStore::new(); + let sidecar = persistence::metadata::SidMetadataStore::new(directory.path()); + assert_eq!( + store + .register_recovered_disk_series(directory.path()) + .unwrap(), + 0 + ); + for bytes in [b"{not json".as_slice(), b"{\"schema_version\":1}", b"{}"] { + std::fs::write(sidecar.path(), bytes).unwrap(); + assert!(store + .register_recovered_disk_series(directory.path()) + .is_err()); + assert!(store.snapshot_instances().is_empty()); + } + } + #[test] fn store_lookups_and_equivalent_sids_share_sds_allocations() { let store = SketchStore::new(); @@ -5102,7 +5114,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - assert_eq!(store.register_recovered_disk_series(tmp.path()), 0); + assert_eq!(store.register_recovered_disk_series(tmp.path()).unwrap(), 0); assert!(store.instance(507).is_none()); let mut foreign = record; foreign.stored_output_id = Some(fingerprint.into()); @@ -5114,7 +5126,7 @@ mod tests { snapshot_sha256: reference.snapshot_sha256, })); sidecar.upsert_all(&[foreign]).unwrap(); - assert_eq!(store.register_recovered_disk_series(tmp.path()), 0); + assert_eq!(store.register_recovered_disk_series(tmp.path()).unwrap(), 0); assert!(store.series_ids_for_policy(fingerprint).is_empty()); } @@ -5417,7 +5429,9 @@ mod tests { restored .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - restored.register_recovered_disk_series(directory.path()); + restored + .register_recovered_disk_series(directory.path()) + .unwrap(); assert!(!restored.append_sample(850, BTreeMap::new(), (0, 30_000), sample(3))); assert!(restored.append_sample(850, BTreeMap::new(), (30_000, 60_000), sample(4))); } From b9f6e6626c3faa00977a619d5a23ff86d28c973e Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:38:12 +0000 Subject: [PATCH 16/25] Keep projection fixtures on the canonical typed encoding --- crates/asap_types/src/aggregation_config.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 8e7af6df0..a170f3cd7 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -716,7 +716,7 @@ mod tests { assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); } - /// PR 5: `serialize_to_json` no longer emits `aggregationId`. + /// Installed materializations reject the removed externally assigned identity. #[test] fn canonical_wire_rejects_aggregation_id() { let cfg = PrecomputeMaterialization::from_yaml_data( @@ -728,7 +728,7 @@ mod tests { let mut json = serde_json::to_value(&cfg).unwrap(); assert!( json.get("aggregationId").is_none(), - "PR 5: aggregationId must not appear on the wire — readers derive it from content" + "aggregationId must not appear in the canonical wire format" ); json["aggregationId"] = serde_json::json!(42); assert!(serde_json::from_value::(json).is_err()); @@ -767,7 +767,8 @@ mod tests { ); let mut yaml = sample_yaml(false); yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); - yaml["valueProjection"] = serde_yaml::to_value(&config.value_projection).unwrap(); + yaml["valueProjection"] = + serde_yaml::to_value(serde_json::to_value(&config.value_projection).unwrap()).unwrap(); let decoded = PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) .unwrap(); From 621cc6eff45fa182365425ed63fecec9eb345d15 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:39:09 +0000 Subject: [PATCH 17/25] Remove superseded deployment API field aliases --- control_plane/src/clickhouse.rs | 3 ++- control_plane/src/main.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 0bbbbba3b..5a3c92eb1 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -204,8 +204,9 @@ fn moving_window(canonical: &QueryExpr) -> Option<(String, (u64, u64))> { pub use asap_frontend_sql::SqlCatalog as ClickHouseSqlCatalog; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ClickHouseSqlWorkload { - #[serde(rename = "sds", alias = "summary_catalog")] + #[serde(rename = "sds")] pub summary_catalog: SummaryCatalog, pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 66c47cac0..edd4afe08 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -200,7 +200,7 @@ struct CompileAndPublishPhysicalPlanRequest { queries: Vec, data_workload: planner_types::workload::DataWorkload, dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, - #[serde(rename = "collector_ids", alias = "target_collector_ids")] + #[serde(rename = "collector_ids")] target_collector_ids: Vec, capability_snapshot_id: String, #[serde(default)] @@ -235,13 +235,13 @@ use physical::compiler::QueryFrontend; #[derive(Debug, Serialize)] struct CompileAndPublishPhysicalPlanResponse { cost_comparison: Option, - #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + #[serde(rename = "logical_selection")] planner_selection_trace: Vec, plan_id: u64, plan_version: u64, status: &'static str, generated_at_unix_ms: u64, - #[serde(rename = "collector_ids", alias = "target_collector_ids")] + #[serde(rename = "collector_ids")] target_collector_ids: Vec, lifecycle_estimates: Vec, } From 51a686e774b52571369bbebfaf0ec2ad32dfd37b Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:43:56 +0000 Subject: [PATCH 18/25] Document obsolete-format removals and cleanup validation --- .../pr749-final-sds-2026-09-28/README.md | 4 +++ .../cleanup-clippy.log.gz | Bin 0 -> 192 bytes .../cleanup-control-final.log.gz | Bin 0 -> 8687 bytes .../cleanup-data-final.log.gz | Bin 0 -> 20215 bytes .../cleanup-restart.log.gz | Bin 0 -> 374 bytes .../cleanup-serving.log.gz | Bin 0 -> 531 bytes .../cleanup-types-final.log.gz | Bin 0 -> 2812 bytes .../pr749-final-sds-2026-09-28/cleanup.md | 30 ++++++++++++++++++ 8 files changed, 34 insertions(+) create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-control-final.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-restart.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-types-final.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/README.md b/docs/evaluation/pr749-final-sds-2026-09-28/README.md index 8d215597a..fe2280b85 100644 --- a/docs/evaluation/pr749-final-sds-2026-09-28/README.md +++ b/docs/evaluation/pr749-final-sds-2026-09-28/README.md @@ -53,3 +53,7 @@ new-version cold state, and fresh input becoming queryable in that version. No production workload, production-cost, or independent human approval claim is made. #728/#742/#775 retain their structural, synthetic-selection, and deployment execution acceptance roles. + +## Obsolete-format cleanup + +See [cleanup and validation](cleanup.md) for removed wire adapters, strict metadata recovery, and downstream verification. diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..fe91d31512265fddd2bc437edfc0b63f26e1554b GIT binary patch literal 192 zcmV;x06+g9iwFP!00002|EW$y=AolF&pTS;9sapfdEQx--@bmO z55IoRO;@+>tlRvQHOe{dvi4+6cRXdC(>8Nm(^RD{vOWB;aJvs5KJ3hS0@$g79esYu zE(*W_q(i0tG?i}o;rVBaLUy65=A?#udDV8W0I~Bh@{8l8veIQbGw@V)oqyyETRwmJ z^eOE5tSlfd1H2wHI1z2z+mh`cO;zX`wrxvz;99n)Oi1g&`X%fZ5M8ROq!%`stJPU& zxpI2`QA=;y+Oi2~0zgIoDcMAp!y(pfc-FPXs-w=}u@=ZA>ui--eTB`#9vtw2y5Pr+ zYEN6iHCDrRUIAWLpKEii2^tWEu^AJ63K#(9=aT`NOGvW@@~_*htXqAAqoiX11OrH` z|I~R)Cw!>PHB+#s;#R}1^Xyesoqh=YtZzWouDe{6y}=Sq*CN%_rUl@@RyooMPR9;$qx1FZ!Zg;jHCy6fpZx^(#274fKq7Tupaa!oat=N`@6yq&2`E4;G{q$b%44S zM0~9k(74tWL8(A_4@8q`;C&LQ3$J#t=8~PXDzshkQ(({B9BY#EM?mXU^^dqY-CUWg z3Pm5{$n7NYhFyAX3+-rPzwxJ-DVy|Gp!(t1(w{y!S`ik(KYlL&@-7kaP%5< z1@IOqoFQvX=pkN0_Q=3VAfjY%LBcxx^%6jY6Oa&J}P%oh-%~hlJVDiEl5R-mapcX+F98tNo zs_ooT5{vj}`Y8(s&}$?pRI_9oz&QiZssiAJI%b8*L6g)2epzTE0K(cV+AwIj6}RlQ zRH-qt{LrJ!hvPq#$Ls}+IyiPf^#yF?Wyg-{^~?9~f;oH~l(ex#?sEQ5?dWASA-XQl zwR2=U^{{nQmud-sKOMZ&D6ZE%?NL*%z{BW~&jOmGPmp-8fYwuqV3ATBG6CQgh&I~; zQqe?M3oXYj1v9ermgT^PngMj^KyZT0GR-+rLtq4!;18f`vU)X4zd3lMkw_9NEo?7gYdFwI%AFD596{ z`2uKc;XinoorA_SUFFr~x308k$7c8(e272L-*w)#%bBSZ*9z=wpe{rQSrQm`02vRC z2(5*Vmj%=;g?hT}Em-T?HmP2^lqkP~Hz+Eq+herlkui8NuP8XstqROQJ0&hygHl$= z0EfR~WYYP$6JZlQ{tl%TC{NU$tOdz|SZt+XFa=b*sliGGH@*R@yvDcOkh z-STr%4&*5lk!L1kEfligiS9jB5fv~kfz4Xs+btH^QJ4s~Gc-974=ya<+OA$F#*pL@ znL@KAT6t(FG1Ud#z#=2Bj9ac7gvZD&Bc&5L3niJcJ>!* znjRKZpjF^jd^Z2r!UAM+z8wgM6)I z!w(F*`nW}wJ>huSC`M*6Xy6y?_FmSokrJKU;K)TSRgw{oemSA5K(5+G^(@bDr@BVY z2KNw71Pz`RUkAMkq4)d*q7bo(dK^ChxBa|kDRVeD@U8s{NI1Oup|kK0w;rPZAj2?- zLqP-QY`cnGe+9CtY01_vO;P5sP6fPu()pQ!D%+?OquJzB)vvlTL^McOWYJ+*ocw~t zC_$AFym+q~9PII;@R0cxmct8jQfTm7XIh=e0*e38_Q^t9Wfzcthg@G0SzJ1JX^Ozn zNpgT4-9K2qTV+D_@!McH{xm8^QuCV6dabpa4uj;yf=b&ZU@`A=5*g>M;0`G?iBc18dI1Xy5 z)+}E@J^+ngqpr?H!q*^D=y+NEd(7iH{=i;KeO!jVF|%NTw5ImMd>OR6g#+vS^?!!c)dco$05VTI71im{hI{1KSnF41}Bthc7p2Jtd_F11mgUAq= zZo<3>NQq(UZ2?50`81pf*1kcruSF4IP=Gdf*^>=!E206@nnek(j+g-X5dMxV>p*Zj z?dkt{H<=WPB!@&tA=went`?vqeVCx54ai1urPTqsOl5Ee$5k>w&R_^}jBY>-Rpxf# z2ny>Qx7NE0{y}7Z1BwwWSYywXQJ%ia<=7vD_rG=En-i|^L`j#O6bhG2UNw=G;iML0 z%gZj0pRQJ~;D`t`(U@x)YY6nB>JB)6t(_O(koIFKg-5F|hVpm^)m1HFC2B_G2XKj& z?-fnpM7})bjxn&lxsgN>eGbF>RY6WH=F9SM&`cX7^1${rWM>d13~AnN7v2;EiWIHT zFz`yd4HDCp`)Y+0OH4e9m-X}EU}VIkgExhjNA{+G3#7z&%nDl`+6{08RAK=Sg^Yvm zfmie8){!r$>3~$g#6IBPy`!}u{!FBoQf_eLfNAz%exAx^H9NQ6h0Yny`$w#{K)V-p zjay-J`TztQU>U2K)Fm8A+s^qXmy5_f@L}6Tm7le4(2HtO>LgiVuh%e*3@^0W7*~aA z+Ck)%T@3cZW1Js#F9*$l@-Pavg3D7{2%$p87~4{AoelLvTQeC8*+67nEs7_>KV|28 z!l=?mmEXwup{QnVPT(VN!9lKUfl(~9wWTO0i5V5!{LK(U=!Kua52{3`;(3C|xS&)7 z6L72J2IOGK9K+ZRyAKnPde(?*DM|qPF4oZ;P%?wfo|!!wsrV(J96mrxa3fF+3MD6B zg%9#Tp(`?IfJ*kT`x9`~c?D_sSO^CtPg*rTI*Y=+Yz4KL1=JfiT^mFJ(l4pGP_D3g zyG)#{@-V`2z@5QCu@8zFbx$-=4Q$*q1|$K$$-XJPfqBM}VzL>yZEg}folPK&>IjWt zv5uh#$2Nc%O2an|IY!)Trie@sx`Xw$K#qaMLfu>dC`sY1NNv%a>Y5^yCH2X9%SNT} z^sm4HReOBCc*FiwV(K*tw=Ez^whKJZ2Kd?b?S1-m_5eTs>O7t%8r z?gfinj3gCv9EC(8y;0&62+9Y^yL1ff|pB!LmQ&eA?89kOtzJJwKCqLYE+G4y%=-|%sJNh1bdW} zL-Top{f+&FyZobfuIHUZsw%@qAY2k1b?{88eou<^$qrvgPF2N2-2%KY=^)N8LR}<% z;P{b`<2?a6fKb&Httfbv16Ejsb{UEW!(yADQ}CcNK;&6lV(43-dp(C`>G;>}z)|k8 zKJbyH81jih?d|YeG^Ba+V7%wPoX8g9rsj>&WCfEK=LE#QJj%G@Qw8)gN-P4&nwI!q&ql zr+W{^5~YE{(FUAkgVW!By4aW(sz~EXgEIhr0F4{6ycb=!PcG$m%+7k=na*L6oh-QF zMVJzqg1nw&^gkq}M^J_4`>@cf(A8lBNZ}wZbO{8MV*JKQZc0OKUO;dWP#W|ZU_5>Z z${O@2v9jmv%i|@z3@Ss9f^TQ^EmSe<V-tc@==I0UJ%qj@& zO&=0l43;!8OXMJ@_c!8<exvM-O_wC8`n(I~R0nv9G%M73L{#l0lKlzz6)^cO712M-mTGOEbJ59Z13PHkZ2 z4|{!2Rf92Ym)r-U$A6bKZ zwC{P56Jsvj$4{(Jth*p5s_GC&$ zGv7{^B&>ado?_q@`8guFxD6TIFb%t@DnDEh{&N?U3%QDe>Q;=&}*Ask?TBr`L zb=gSEKEBfur9DCKppN6a904R28D(IT>E-l;aB6cMp^}U

s!glNdlZp4I*d5r_Vz!>oLe>{hZ+US|v881<6Dbj>|qau>*$u{u*uwO6o75 z$QVZys4*(|Hf$A!Jd`!fen>09-iQiO$wZ?mQp@y7E{B~cf;Yvx1j2^QFTK87aKfUc z?pL_5{~8PZBpvM^pN>O3TuDHYq$EcYoRp&95xgFBJUc2yb{u<%^_ch#)|5X386^v4 zWk7cM8l$C=jztUjmUI>dM4Wd#;487#uXC>`=2ZR2ueb8`qlN#SOTGa=4kJ&%bO5|6 zkb6R~0Ve=pt;zG?2W4liX(zp~HBe=L91rxnPQs7RFA;?YK`bT1#ytq!BL4g$k$K{B zG4N_&jTOhFlkdvgY|#OLf&`9g58C=mQ4$K zMHuIEm2^ZiDZ}mS_?JT9*vR$#0#VR=I#ac~Y8!J6n~am;GN@c6wm!MgrK{R9Ziv#9 zQa|`6dypOyRQ@h0XI~E(EjY`e0dR~DICM<*PGwp3n?RH|=kavq!(k7LfQLecV$45% z2C^^7EVpX82+9xB!fKu8G-rfKE>r)-pM;1;S#t86^r&!Xmuuey?q6~KXi?s6+Fh{O z`N>^-z<irLqxb zhyi$zKJB;;UeZ~LE^p{zEOgN|lZMa~Fd81yS*|N#x#&WJUfl8#{<#h^OGbEk7v$~c zSB|HVygcY$%ztIM6t7%XzPhw1V%1a}o#5aJZqq*N+ctn6XX_p|O^U57m_XW0z5pyxfxrssaw2OQS%D?OZo_1TJ-o2J`TkohG^A)jlH)m)Sn2^rk%5y^ z^-3!2pDuXu6rLpRE3vQ@Eo7aLPQrmRj2Aec+XRML z*x-UHiv@9i?$MjAOIjPH75-ED;iaj({7#jrEjWwQ>SeGZ4||Nzb3^?KR#bVKwc7@e zhe@u2IG4PS-RxW#fP*@z)iL(c#-9BLoqmIzhn}DdRU2WkPMnO_xr9>rbyoM#hnBE( zYz^;JCU4`k8g9hG3X2tW8Tau!jO~Ta>kt@u5Y?N;yzEuX4k^NsYJZHOd!q%%mRlTs zvVLL;*Ib1D3qb}~A96P({2!{6d2)43-csm)($wnMO`o60 z5%Bdf+^OZ}*l~eC=loBvA)Q-%=gU#nix%=?u;&2Q;Q5pN@h+}19KQ!D%DEzE{d73r zFy$Qt&{y`A+M=ZtsaJq-wZ?aFtYYH3kqj0LidPzC(@=j(In(2MpG9N0M#`wA5_+=F z+&Arp#9M!v;V^{c6~fp9X@Zsm=4z9X2?&e3bwb$+C<>hNNF%hlQVooP8oRrww;c)N z*n#b|cFQQ*aOzEsHZZ{p=UeuMY%Fh`m!t#_`Ibia{8WNdMR`)%d+3E@u3^gr3Bj$R z7O6mHu0rBAq)_9G5a0w;1@dugu|o|#si7k?f)L}#=%GK5#m155h>lh@K3>b}BajbS zc_@in_^0R?dP^)VIwd3A093pBOhT{95bH6<$5KS*w5GpV`#yq`kPhb#IJuoc=wdC* zm{K+-XoRSTWN=0gDjStl3kV1)Opa3y$^%wUS42Q)?Y;O8`cc1S5r|(s2$V95bp~6& z_~@U5Nld=8i$}GfAVJfyJW}HtrZMiuBt1qSH*nKr8k#)9)3oAvsR!=G%$OHHxJix# z`$2F}Eg1&P^;k`gFTijaMhbtT$PRr{&1L|MJ(80o@r^6!m(3>O2tJBcu5U>*QNDX zJYAltmjV2P8Z4O(@T$gd;j36Qh`(;0UpXLRBp}m)@pZ zHB#zsN+69D;)Y8hG2Z9cl)zCLv}fgjIW5ff@$?4%H=c5^pQmzMThJ1Vmy_~o?CE89 zg&VBLJhCN=^=C-N_P7wtFEDtA6UJxWgsw;E@Cal&z|g_=Ry0`wJ{0Jm5|tZg;Ij9* z)e-O==1~Fhy1ZA^%zFr>*b5E`JRNe?Fe6a4p5{iv2+PkX z+-2ucl{8}*KTHzLUW}V&HXn_zEn%jl*oZjDN5lYAG*~<^n+LEQT|iKp2dH;@wDN=) zgfVgC*y@WP!W33&=#CP&H!9m(?f2ciflqYTMya`bah3J{E(YDBJwKcz%8fH^Ixil{kwf!_59@ zd}a#xu`+vA(XD4RW(c3A@a73`4#C?~T{||LdkS8Ztg<%H;RHPL%DX<4 z?URFr@#ZQS(qL$Atnv9L+3cbhkZbq+xLp4>jk=LUDS51OhnMQXp|qiT&<^WWs0K6E z2l!4XzD18q$yj+U-xld!NB0%l0%Z^0Ug$5avv+ee*9l21^8vLT)P&^3D9Ra}V)~J& z%oT4-jL4SiU3RViBu&TUK}ds#WNs)G8O#9ix8(74J0LeHEzw#YTr-phW6uV`GPmm` z8aWJFA9^t0g=6GaE+z2@pT2=|+IHa*d8NQ;%|T>?@6qzT6ksS_A`(Xl zYW!BMcz;$*D_AON@vhC?-CB1$OD;bqYh_qEKl?!0>I+vK_J*&II;^vRk~@^20@E8+ z5-5wzMV#3?n%2C7=&$3(G?BeMy*gK&++Emy2Z}oF=uTiV|J+2_`0hDUo-#7M4(W`; zbWzwuiZ9H$N0UG$T^wPk#J)qvVq|`OnCr;V>G}Qo9eM=QR(t4>JxN#RJS3YYlEz93PwDrtCZcmRe;y6-p#u~z6XLpzQ+Xg~$oi@LzG_1%HTIY& z7zMSZNFkl9nrKCe7t@H)FuZc&-o@qg8@6!*$K;2ZBXoi z!kMq-kBlY|?}5hHIUQ-ZS(n)#-pR_NE8X+=h5K7lq!w@!^3&9d zlexk&5EBRQ%Z&^Ye#2m6$5$$k%pIN;wjok)K7YnIfCVqem+7@U@?Po(I50HB$5ZFb zbR-wjXvH%{rfXmqgO7y|{)S2ViHp~a_&$U`XRm)6KZt$`!w;HElF4L3VQwHKGZ4^y z&F#84o&TIe@Po7gUl-V~N(3Y?FV_n9THar!DDU8HDj7oM7E~vXOcGDJ&U&RS{`Se9 zLKHQ*MiHjBLh{6H>DCLS?tClo+CxvucV(+dVNi&}xraB3&GWVo#i1|OLphfJS5kA^StXqd~<&3i+!D4VC zU%1(uLvTLPxW+(`B(0cQH1M58uFoUV-f$rZ3c6`(=cnjJ5MXT#%T+-gs%Rh>8R&sSza=oV+ zXwmV&9l6e^&aW!WisYe&Ci!?}07uDbcxNOw{BYZgPa>i-xuExFMkvRdEImm8baU1& z8(iTm7fke4hz8O9-Jf1xV8NDLRVT#6RLz$WQ#@_@0?={nBp0rxH(FA1n_4K6VW)$( zqr)%{vdPw(ts{6)p@1`5c?PTon#-j|7i@c+IFJ&e?Qu5)L{ESKsC0R7dslh6-D{O{mJ{=UPe7O?A!U0LtGeE9O4d-raK|NCdh zA^@*ap`GuX%|Bvu#|P_nKYhI1FxC1I^y_<|u>AaZ@t*+Y&8WAGtuM#aKUX?$I%-vpI)Y(iZ@P#U!w_@G)KFsN88+lXXG~Zu>qBX zC+Icg5-+(=Zhmi~qGJJIDroybiYY*S{lNv$5&_MEmWa(YGN{^3)Aa9eL(!i;e0c$i N{(t=ORYa_s006_Kz}f%+ literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..effa0a8c301bb993ef2c6adaddd660ba9156288a GIT binary patch literal 20215 zcmV(oK=HpHiwFP!00002|IK}Amh(1}?f+hdSJ20t0cLR-2U!@gn9l!SOorc}H-pk*|W@yL#$(Kb{ z5+xd<88%ZXTcXjl1JAnCy(!v5;fk_49DGkibmQjGDyz_8yJ=^Xfa^Ni^WOjA_d_TS zV_g#|H~7Lg`?j39QQr;w(YJKIZoePTW9^2PzIN&Sx{*sy^YU@%($|BFlPx}~;Z&S_ z-O);DKSOZGS#-L#uJ)JW&=qytzKxwxJGrK;JuZ(s9(mS9-;PaL$laaJ?yYLduw}2# zw17iJx2GNIPM5X2&{v8RoykP>H8jdHym|V?LPEj!75)0nU*v~;NtL9$w%YA{9g0ik z>rx)eu6KJMwj7F1b>B9J>WH)HT<^}I81c@NBk`B8brFZQC;n6 z&yj{G3w&!UrRfi$I(KyET;0*_8_)EceZIlpT|IhvCj0h$b}t!u=k{-*cH#7r(Ou1~ z((O-l?E8U+dQLDJ@Ou6CC^3*^5Bz8vj16j&s!?cEpM=@OV<60K(Led&E@ z>v5=P*OboKyfy6yZ6bZD?+5(f4MQKW+TIUi-#|NR@awK^!Z){5FBFraPiU$1F>N0S zd!NWPPiJ*-RXvhysoNu|WVF1fH+8Q0j(vW=gP|RJqFGnW`$d-SJaK(+>AKxJY{dGl zBUO@>mneiqGKZlePFMCVt&!dv)s~N9H%+j&oGy?|D5WE(kh$-)(oL35Xbj zIQ8ZBE21~QYNPb3L}Za@rXy=%1kn5j`fxk3c-urd#LCidP2y7ixR9VdHoI)FmDwN` zuBhLSAJaVv>nI%Xg}$Q;NTam<5N6!yZ)aOI-8f{KUK~O^IlBDF&Q}lr;6UjuLbjf( z!2Yws_kG)gPZup|MC+zM`K?Tv(Z=WzR^_@e)p8f@k96Ko-iBk;L!mD`9{JDF_ZQMx zO4kki2eJ3l&33yw9m~;6s%sdZ(}mvGt~%2vJ5tl$Jh&Q-yY_y)S9&QMBrg+vA)%{rECt>Cq6aI7>hHheq<9-jl%{1q@<`37yAxkIEy6FLamW{OV zf5p+0`Qd0k3jSR|s$AF3KdVLSm==MJC##SBwI{gUI#LkceVuPzY`yBr)NNEXq&bo3 zB9YS%FQP&3G~kl(Krg02N0plb--fSVPPWMtm+M|m;&tcDV;Am#$Nz$JP$~_n4s5%> zaJSf4d*S%}=4Or!M+q4O&cx~!0sq;xU>LVwy5?IU`!d?F^iA9du$o0WU6h04nZqU9Y<(Hoz}G3JR$c4K0&L)#O#WkXNAcfx6-;M|j?KAX#bc`lPJryu`WllaG9 zArLW|F)#2Ml&HY(2(eLxM)>+>v#JkCrVPRsFwQHSZ$+eYL%aru8r&R1@U8 z2?YyQQZ>M68Q>mQzo|F@hYMZXZ7UU7sTZG>pR1q?X^Wi%%-9!F) z?CE|s?#gR`MU{N*F6rlLI?=J8^H8Gg97q+x4I1mHW))%V-z(f{xVid9g0}P^5QF^W z{eJ2RTs(v6AljzB6s0?|*Z5Nj`%j{lht%n^>#Ouh;JHAZghQ8)SoE~BceDX8z=;QP z1a5rFS!Wyio#Tx@RQ15#5X3Ddz@|TA=Q};8vug%oBbh=AQE#94l=w(A++Yf}+5!XZ z`xJWFHU~PM@!E!9>x4rqycf8}^ix+Nsi3NhT?-|dIR<@cB&+9HXzfduA>yk>WT6F1 z;&=m1cIPdyaveYoC)fGhbc7Mng5ZS%fsjMk!~hlE;G4)a8Xb3)XVcI0?R|H2$GY9o z&X8g9#x5`Ae=Q!<$EhODCmyMue-AM*c^k8{1?B+u7Kt_JGq#&+@-FO`v0qD|X^r`@ zZ(;IIHS9-apWAbOyTq5FnF8>WvimKFr+KC-aC0xFmtK!y@RMedJoK>(13j-Zr@D;6 zWb@FHy{_^!Y@{w{^MLjvWoRa?>F6haCF`OO$gq~}`67w}Ao3allsH-Lt>r+(fQvd{ z4u|TKAKtKrOUcgNsyDk!=YshKGU0zUV%&;Kfog6sDuw$nP#nviCAk#@9w860y%09;w?4Zp?NV7F){$5Rob+oGRkAV<4HmhQgl z_akYW)v;-NFJW}~T<^>pO10yBH+rBPkCHT>#P>cq4@ZPmIJHd8VK=NOGJsXs#7M-}3F=}AT3)5N5{NErF7YjJQf=>#Dr=0vN|tBTo1$T(Bj{}p ze#pdOlgk0eN8mKB^%w9P?&q5D0oliyNgTI-xg=tb#^f?zsPLH{t1thSsL7|DD`z>K zzPNU@U**VlDog01^alqm&wW-=pVbt*l?32-2mtM}7xgGkzVa=v-@b$eq#&0Urf;DlM&8LpII zDts%P=Mi!p|Jbk}GU_IxH?UT>`EagQ0lUZ!;+y3x93LB`US$}~sR{d0lwX91zuYbM z_+lRpNW;P=krX?Q!HfoblK=Yk*G-e>efjX!@q|-g%ThAG=w`lgHb=V`wUjKT13&&| zf7wh@PeK#s>6Bd^+C;)>E~(86U)fBPUZSS9?V*cK)=C(6bP+2A&d)nKic{4o=}i}- zIzvN*SjBzQIKL=a&{XN!SFATe){PArpO{c1zgYds9;!y+16fouGEMc2%UNG?jZO?! zMeFtJRW`e23;7jP&7K}sK)O#!T0_5iKvL|_?laqPtfv(F+PgkHFap*L8OC$A>Alfo zep5Yl@49M0al-aJq?4_{Kyyzu$5`NE@_=>S8lE_}*kNfbLft!w>-|9t;{i3vc0@%5G2dx1aeY@65|GhVFLgoU zyO*Ee*NDxgB&JAAfdrK)TZ`;F@xPSo2Is0j`cKD!g_ssAnU6zt9NW=Sf?fFzXY|Td z>^N18Egac=bMw2468a6bXoeSG}K+ujf8g z#03cvsj6!)c=_&0ESODCc7$_T7Yr>Zn$;>79!r<^|Imw1B9$~O4x0U(vQM+kGlZfB z4J7rrF6ZRt#zNvV7$1k7Eu?H?VL2^7ND?&~1@Yx`&SB z*7PE!%(CHQj0w-+!ng54Fj>n!8C?RMfz5?-8)})w+Ya-VJV1=bLB?rMsH-IjXG#kqr!0y_|?7y~u z@H^SWT*n5bcWuw!z2tjs2j!T~FUd!<8Wg))^Hi|gz_2<&v3H`IM>UTmZ#%##r=@cZ z%$cDdIdHe)=A>bCsKyrH*4#gFVO@u>bG;Iid#qG)T8WlJ6N_WDzXv_t(=FCavW0;w z$0R{W{%z$3Oj*MQ_$MkrXb_T&A6pi@Z#ljjl}%F#Q)K{@cUgwS4mpu{_)lH9Id^c5lC(%oLMp~zOuqTc4ny_&fkqoGI z*>m~}Jf5Odh}-)`)Vb1;b6KwNJ&b=JUBA0cC4M>W$B-hq$liZ5B!{qXd;c?J;Uc}$ z_FqN~ND%kMBrhX(ue80E)D)_VC2l`S-F&Nm2Te`4+q>|+^v^^82k(&>-Y=mgASC>y zy4oopQgoxAxlCWoz=FZWM-Rw~@^7S%#3f5xxO6{$7NnTC;otxIX|km0{5(;KM#sH> zspS818Y29WJVH-_fkf_cQGQ?|TvNs~cvwcCYX-tkk%tObk?eY@e!C);av#3^F{&p?IoXG4M@dp;vvawdN7HK?odBy=qG-GQ<2VR z15*N4*__0u_0vcUXA2-ENURbH#O=snx6hu)u~j!aZG%jTeBZLv^zj0qS8`d+(A6XK2Z}(}g`DwjU&z(WO`{;!) z4k#%R)|C9GhUgGe5Rf+sV7+)!{kcQBNr@7K3$(E+u(hN41beO)1y5yZ?)%qG=TRmX z3b}d9h!FD0-h-6B-;(|wV*{crHJ&$8;z z;n=~r_?cBcBKhJ4zRZ32ce)P$Z@3Nr2A5%0cj1z>8})|#&#(WK>c^(lN#fBQQEt<( zuQ$3)m*5J3+y#oaWd^8UumH9AnZ}*uPnvJSdJb$3*g~PP)ZjnbU6M%sugzogUSa2I zS`JIoy5bDCqqsGU(*Le0B4#hl-P!ALMR2xv-w5T0h(|BJ))^eI8T`;K<0vAd2~){F z0~Y^MX_KUd8u5E*iPzB-MI`x4{>4sSU9A;8M$=`z$fi;Jv5v8RR70FXJ3rXDg%`5rG%RNf7 zc0dXeZ@;;MeQuzJH?{CNZJ@T4Ep8>XCZQ`E&6NpV8|Yi1&}MrnqUVY|WASFTVHdIZ zO^fj_BK2y=$A(2e$dj9iy(o?8o+rgmAaXywr)dpIBN%{#z`KVZyh$P!-e^CL;cvWE z_pZDRg@GdWYKK&OC@4}5m3wvgy;Jnt#@^6UTY$3WPsD625Unib&$KIc#{-Vp4DSLVbrHE%>=&`Rk z!Q0%~$dg2B6KjO#Rlb(!wAQKl=1uYeXxTW*d zdwxO*5j-u6RmiA!Ef8P22Y?Tq+l9j`2GuPZuu%YE27z}=q!CptB5RpNvnw9Vk5(W zK6^E(#dr8yDnEIJ*D@R#3vQTeLcsMBxL8t^I1PF2G2D+W5ONs@B%Nop)+cSp`1wAy zDj8-~Qnyzr|bML|5t~8>w#7BW4g^sT4sb&Pmf{;fco78OFZXffk?1v!PWI?k>gP-WU&h zqLN|wL`q4^xxzuXRA)M$rI&dIjYJYgIyoq07k&`1^KKD+ap^63GWG+FYWX#KGWNr3 zWMndo|L`8U&vp0cdm7jGNa*CCko_Kk&&?|u;3hT8F4Fgd58Mco&jTHF$MdA~W}q|a zJxIgivq@32>YR)0niuhTyU4Vb!C1!F{h0I ztMif3HcYxx_!jD5%*t6>3iAyWd>paptL7bA|J_~4K<@s2UF+8wNLYh+xoEdJQWA{H z!2)8NwvCp`qVaQW(!;@*GWtb>uhlmf5m_v6zZ`v$20IRbkPlV!mN2}A#)q_)##6no zVH9a=jcV{j`v6O*;&iZhasQ5*_~qEF4;K7^ZXxIrRcPaf;yj|9$H(p~Yev=80r~%v zKSB2$8hm1gT}y*s5GK?%5({?F+A=-fy#9aXwlh7R5I6 z*`wG=*8|Lk=>6q*veZ5C2j9`r7G%2-PMq7`gXP$U(Ji&TSIS{ zG{g;~XmvguHSs=f7!Edz{GGl{XTIk%+AtvWErCqkFi4QDD%=wz&zs`L@2s5jA7{=@ zddS~($F%CIP?d8#=X^0BI;Np<=eD7=(I5ZS+Ixw@a{`NIuoU6b{&Vj=w{h;=env;O zFz!lsEW-Xo4=e58WXx5PV!8QF#Z9Vg1a9r5UG5~QvWzWH<~YuupRBTwDnp{e!Pog4KtgY2R4z?;z{4y$EDRX+m1AvF_yUsms_b~LgsFlcms z*Uw11vuQJK?j$rAF2~B1cBe~VFg?y5`KU@dI5fdb+R?NJ%YG(B)308jD*Vh*+1XsP zb;(+C;Y)VV^Sj5bO|sr%H^Z=gZ%Jy`6$hrS8u1l-G_H0#;ySx@@2$sWrJW4RwVTWF z1+cT~2-9AQ+UOKXnFfj!ncpkVsHoSD5+&pxwFV`>0E^Zl- zbWa{cm~o;_4(^Ed;yK1H<2KMiAF4iPooyMMF9|a~?zzhiIK#vk_3v9L1RmUWdyYlE zXheM-S;rj4GYMLHMx~#6B5$F7&RwAm64f z1L|j-Xc#gKr1BoEnn|v69q0Co^mJ_7j&BF&My>aKUeW*CYo9mk@#+Mx>L|t+`su;k zijDR%8Iwu;n6o`C_moIWBWO|_{25hR-?s|9T^&j08lRV>@j!osL?-!y5-aHfUiNLL zI^n{x&Hj|alqy)R>IY1Cz%im5N$1TpqUoyp5? z#!nP_#gk52z^aOrsz>w>joZV2 zL~rD{oTGGYTc^o+tV7@_gzJvg+)ZQZv&^q&At|h+vc5LpO^FkIS`&I4voA(o>Z;?{ zMO{S$BA@Em1%ejkg&LmyjBfQe6rxgps7Ic5*)o;0d!x5feO@)Cmcy&(*r(T2)Xjwe z5dD1TYsF;T(Yu4{7g2Eo+4w$W8hvbZyh7cQuIiVL9Zus=;>YZ2Qgzsq@CxjXW=%kI zlvbUv;46()T^6yCe7cBLd%?8|d(JmeU-LXG?TewblQv+4U$KLR(6)95`SvD{PdmrY zb~rw~Q8C|+PbO)xE6;7!e##;5tex|QlVSM|e~#_2eQXj0>o+hur*+B8HHilnsA|wB*RLF|>swgG zSC37ksYV%th8{QQx{vK$(QKhvuNhAHXoM1`nbQU8y*mre7#e~V7k91~4xShkhI>94 zNK+MHVgw+UXSG50ken%0B9XS*F^@>7AyAc%wFB`^YbOA*RRTbg$s~L`wdnhneoUY4 z_r4axcCrQQiXGh=cJKonn23`!oF!D%^eOGy{=`u9^cv216=XoXZ2MvyYhga+SBAPq z7&I?x^1VOLU|8J>zyjDL2lMwCNK#)sM+@(Rj+<`; zT{{esbY7^kn;OiP6fL$*?Od#H-U3TDtYHF zkQvH-qy%q3m*d|krpg{WoBV?ku|j{pXO{KfFTeku~Zqzici zagj=Hm8~8QHaK4pORbuHJ>n!`*i7+<^%Lp6)SAPSXPXiA^QmU0Zg=zzxzC(EMU%pu z4mHgX<@(beJ(iv?m=wYRN8~K)-Xx%)t|4}z!0GDs;dVZF_}m8MhLyiyb`4!)*s>&4 zM)f>n)8Y*iKF!GP{gX_o7t9I_NPPe?RT4cn@%S27{y{cCBOac}CB#kes*T%|T_pQ3 z&5_NcvmYEA71Oc8Ef6I8MfTEj!Y9_0`LY3_$n_k7V86u8jU&x}tXrc0F5QoSfNPoO ze9zX3a#VU5`gPIOBdH;m^RBJM@3e{eEm-DR!0Ta2u&g5DwWeP%uo-W@_@ra{!H-Rh z{zmkYTQOxo@VuhT{h1l5I&mh`vVvro=4F>ZFX3$>BO8SQlAIDIxQ#p(0v(B#1Mh3*lLzzU7hKi$mpt?m&~Q&+F!Z^ zRq+S0qvF8HaUwrBpl`FwE)q4QB z_9t;sc8{HHSx1xQ&a>e%=eSh%@*m8N1##H5X)!rRgLzCMmsQJ(r3bHhvb}7)~71PAY_XV{k$PVk8_My8++MOmJ+jNXj!XW0wTM>!?fM zSaZJ%SAQU3Yr0ymAjvLe11`TRG2|pP`o+O<9C0}1rg~S@_%+1^x0O|g=v9Ct^|3o) z-jMGVyylgYR1IjGBjdfQY5<|}q}%j@O$zKJrcuc(qqlpw6jA@RssXzfP)#hydunsc zVtdN@n8CW1&SZy`a(Bcjk&~IFRPI}(wfjf$@0GL>RjD{xF$$sS!ssWUUG>}tNfik8 z&Cp*?F`Kq?&C(gL_zB}qH#rg8WTB(jRwa}qoEn1=7*(XNaH%Bie_5RUc}KeeY-BZR z`?ejjpN2{(tu6=Xth%f?#UoMe!H3N19%zT^ynf zCsAkw0s5ECA@(5wr;wB2Cac7rT(dv5D@4Uy7DZPrny>U89M^Y>4^K+1xo+ju)}~ij z?!gc6D0?SqX9jSEzWul_^x6DI#GGLLEditE{QdU4!>A%VBMVp4WZ^7o&6;?hqE3gJ zthn}YAX((uYZB`eis{?i$q0Nn6H8mLa2mpl(8KM!ZX=2iV=+8BaiILym1gSk1@F}(H zIsY!?C(eIO?^*8>_fg(UH=TOiYpo`}3(g{c4jff-2Uthr(7iR^=BWDS!U znSim$evtlOmzg($|2z0vEd1ENMi#5;ujv0864i^kK-#ck z9Mu1w`ev7IEk`i$cg?Z9F(K4oJ9*dXJB~JOw0|8kqy{u0_IPtz40UqBzlg!!YDP(q zkjP{5?Sq&yf&_7*=UrQOnF;WL_#DmXIJPtgZOHZ-`qu4Dv*mA?n}i-o*!yOfRox9G zWbh3Vv+M~^hHHtnB%kX_o3>%hZQRtZ?Z)ebE_o&F<5{Jrezy-Iy8x4o>R@&)-Qn1U z{8o$Xd(XCtAUPoXuHKd2AN{A*Lu5Blo>*y8S;&&TX$7F-s#&tMK*Sr)b^<9c0AgBT zj=-ogl9wXAnxD zx{_68p%X*!#=pcTg^Q1u;H(j9mQ`t1Rr=J3nUB4+*+{^XYm#_%mIVTXXIdKj-~jxwxe%E0j**0SrQ`oC$hibv*6b zeoDg1x4%=3IjyO$J{diN16}MA7b{Zb#qjIv$SOy&pg$5xr!T#ZY_wjYa>pLhNwK4u zs1Xz zQV^II_;JbTIN}}$WcWFw5WXQdv;*A84+PNKyh&4-_}N_Nbo}Gj7d~3{yv8S#e2&l$ zV`|20f>5G1!BJdwPK9vYt!s5lVzA(jR}7gYDaYBj))Ku$qS7wgJUB-me)IU6I7Nc{n)|7$ zjuAs7hpHE30zB{q1>|$*gR_?Jx+5yL2F81qyP!}a5ERkI{)%vllaQ##%(xJMmFyNR zl-24a_>14F&agzn3JTWbfU^qU{r+j&nc0I$z%l<&3KEc4wDQHh;hjNH7R5^|j3ugh z8P*Y1H>(EC{!EkUOH{KqH#8pSA84Y&Ro0slzoouPYpJx#1TKjfwx`?Lb5y!2&7xel zdzuW<_>T;`{~UH_{(LeFAxnzD`(}GDzhAm6V||fRT&q=~t~9>XGISx=w;Jlm7Mcmc z2i@{$O=VUoq&n%<6`#~$Q-n?7ne}ROOo3J8aP!!56TQ+iZpf1yTC!&Cd;+H@*ICAv z8u5b&(Z#k9_l*&R=334k!Mfq}b!P7xD%u4$MqynjzRg%&Lr2vn#h3fqnZ9OxiEY1Z z_!C(w?GaUHQ4@L@EtC=@UU_%>m)!IoL;_rxWj4&FCQMN}`dEPxS0_vJ9<(hrcR*%( ztio#m!<0ErPsx76IX;1`Z0sKYgT?l#-Dg}93K5~>N;kv^lai6bfab;|v0$4^j8DKe zzYXxsrjyiITKmcmc+6u-`U}n}p~=K#Xuk2*Ig9NUHgAv%sb9#%)}N5)Mn`+gwuKS* znkC1@$gxWGQqovTJ`cIcn#WGbC&{cYKje=d=6^4{U0GvXn8z37wKhJ_Vi`kDd&W+- zA`ervonmiXuN$zMS!i2JRAvcL`^@%Aua8BV(@WSX1#LI?S!5=Sts7@joN;4`cs=%}Ww#dw*`ZCtj~z!tXzu0j{!c8s>*Y z*blVcIcdZ-ZMeFH`3W0q6Z0-ik3K~*O4!}&XV6t#Rs;}qlrXWO+q{6TG52tCoxfUY zK08HHUv3zv;MEB)ik%>27Bljw*JyTHcFPD5QsnUa~$+&qCR^Dx&-xPNB7x*_tLP`rn& zb`rC_yk;)b1ItJ8Z;=rXDZfxQFg*d$wA(1{Px+GS|B*g7?C_ z;Z62c83Em*Q%YuxlC;emo_uC=v$?us@9!IGZ`%&F(!xB3NQm zeYMPcbyZFn?t4~r>UQ2T8MLJle9LwRpAab}li7A`+FoFq^>ckMImQmf?H8>2YdFzK z5r;XqQ|*3k2F`AAC;y*eHTsNTIC;**MtSUvjIU+Go8_z#BH~c5pkmsxK3gM-?B0At z)Lf1UTQM-0fdmT0daKKB{D*(scz4U-Va0n$|9RR`mD4}7Qx^N_>5}h4`x$7Z_r`$8xIQ!g%}MEou|Kd!`W9Tr;@u{GQLBQhmXE%{ z#-a#?21loFD5(NsyWBZ9hh|(Yo5H+K23kq>9+^5wwr&qRE7+P@mu;z;#*1Wy#lH5g zxd|lfLpz*ruC_Cw*fjZRXI?l;$hXYws za?fz7jO&t%jD=K5)hphlkP-*NpD~16HbQ5E6E4YeZzSyyyC)^(z}0R%-N0-sOApGj zKmPK?Z#Q|;%P1uGG`U$^eT}=tH+I7XE6e-AUEQ7Bu>8v_RYJm z+b%7HGBw$7-k=aZ>eH}@&au^k3F#MZ`O-r>V#g4C$~E=}nGVgwJdk_u`YoG}x;Pun zEV)DN{%C7IY<*`K`|3=m{wBZuYs0XYvUgHSsx@}3u{f=H+Ere)bebN8!eMW6;DEz}itxBfX<;$J`QD<@_JDQgb)D+7x<+Dn zP++)dE!K=tdaU6Q?;V{(f7fw$(WvkSS7GZuE8>%ucjC29%*IL;IKv-!QyX*!8;d&w znaAXcHu?%J*k$Xzc+>0-A%?rYX;O5uZ|m{g2yinqB6xZmX|W~S$UKU%_Oea`o~)@| z%GEr$PSH#U?hTIpFQv;<)=4mo4V>LmGaE(T)=mSg1bI_b*_5H9d9(TE-4W{zcfBKA zI>0;C8(;iz7slw1378O2S?405ump8UJG^l9z3queL`>vB#O{;iw@!l8A3g-xiS&%D z$lkHj+F3uV71Xt7@)g2>_AlLy70_UIX#7}$-R&RXv=dVYs9WZBOXVga6}RUQ6%j4& zB+ErR^DZ=xzS;sV2ca9NV54Rou&ZZHB5Z>!UWuT7nx{*R19sxRTzUzF+ z)Y+v?Y?E)t`|!ArZKgeD#0GR=Y516>wF7_n$`hTitEy(A{tAkgMj#a18WIM%HsWdg z5z?M76F}8^LG5ru7IIe=PZMfkto49LC<~#qaWu*_6>oEy&*v)wCsJH5+-#yyXD}O^ z!e1W4Z>`?6hMWB2|K2M%Ju+zuzK+D?*}Zz-%*U-))=u!0IG$o@K*CDH>xc?sAF|3? z>r;s_LhZuIhrCh?bzBj=5q=!Oyo-gsQ$Y&6lB?P2%#~|Sja5)(C3HZiOUa1{03ZYuede0iMT0f{^dYqMP7Ya_Z7f-PotWa0}LI zs4`-_rfpF){SjRlv??tLTkJg80~(_OeKZ?=MLSMFOJe&sPESMHgg9=cgGSa3zq->R zwRQIm)7$c6)(O>)nI7?(uT+i)tt+$6B<(-VVxJ}N>*$$oeJvqgNiVIJHhGJ1NbR~G zAUjnbRDG-l^Kh_Mr_ zln13pIq7OeC)dT_+?{v7e;HYm>D&i7m@cf$>19;!VG0tU%$HG31NW>vX_MX`K55b_ zpj*(Ix!%O6ONaLt)0PHfdduKVidStT`8(TAd{ncNGO|A3$uFRVT95Eu7OiOylyouo zHARVZ!qIT<$EI0lEnbfv3%q=fEP%TBqus5UGq;f1`-DyNze#}}2kzua!*t@BuiLiU zFmHp;lLgeeADY>O*%xKY?q9Cdt9Iim*U}v|^H~k8SDfqEy`-DbN;Mz=-RLHPtOd^$z<-KBj5t~8=8de8t8qdm${N=DxPL47> z_As7Y$VJ9#$M~JwuJC;}AlZytrMK0T!l*`796>tvK3F@K{jh~x-;W)8MbS8-;B2{T zuh2cl!ZHft#ik)!4ty5pmUgOb$cjKo{%skMu+R16JleN$VM^qdQWL9|dHk!m(8r-Y z9M)4iCT&6^She}L|G2@TTQo8wP+)WTKXw<>M}!*CucRSr@WY0yeRlm@(x3=EulTiy zcWNN%7?O+bhK-a|0C3+W)3Zpk3~mbp<608ydopN>dw((4zc|J>j2~ayzNY=)o?yd= zCq@%{8r`PPuNz)-;-prTQBbR-Z_;+ok`m%!pHRxf$3%`;Pi4BliFn~D5R(b{$Je^4 zEBQ6_V@^UxNxjTHf5Uh#|8RTUW4>6z`@2nA%N->HaqXN)n2icV+H*+X97_>&mlEDD zeG53Boeybsmcs1s1+!#2id2K{Jq5|rWT%v96FH$FaR-2NAZlF=^-zH6kWp6pMUB6im zPj6f_A`_n>1lWD?`63dD48J;e6~AjAzZR)!l;lb1-C%ANc6G|q$w%-u{ReH0*vUMF z%2vBv9~@HBU)6sOqA-^cp+H)X<)I|%U+=95tZ`j9wQB~K2>!%HN4X4YlAkL!@6}>n zpS*uG#LKJPmALL7adHO9bYM_V}I z)U?U0L^vO`NtK9>*w_*8j$UHE?NlFsp1piV;=0Fzu&4av->N2??8B7l#V(G@oXpp@ zQIJ1bj`_L7a$vJzuuQaI%H@L+@~?;@d+xgK!^&e6fxfz`U3MXMgqlr)CgeYV|J{}B zENYu6mhVFB@zjmG8u{7W8kN*zVi6})#5+^?7Dgcq9w%J)Nku*ar^ib%0+N1;Q`=X6 zwTr1v6$Iy4N!}wCIPwD!ur9ViTzDuhslI`^ew1Kkh^$%AK?DIQ}fEaw(^kh9@QD!F` zNUIMZ(ygKx36|-8Ne%5vn$r3TH(lzevQib~*~FgDH12tn#7+6wT&cBQ-TDEf|7y5*2XGoKiTDD{w zIjI{%l@^g6%?0eB^x;6Hm6^#x4D+YfsoMiyW1%;40U3M!kS<)o(Yvy&pS2ClLo}Ff zE<&On!|8QLS!rY?wtCMsa93A9Ery=b67 zQ_vkOPL#$(PuCV+ZVv<+w&XZ8x0vM^911$oD{?JLrl7`24J0k^4}fH?9k?SU@Dq9- ztMH`zMYOt*1mXnmLjk8H(v+3pkt<3$7%qwgy6bTwZPHeEB}G!@5v%5aajX}uf(m#X z7W+vH{0ds?k|*%wau~RuIC0p$!xAs)zKnb)pgy_75N(QJAqc@^$dDhDR1a#()*0QL zI03Mj8dZwkqwiDwH4i&+#imb^)XN09d4$>t*H6$49dafBHeIKKq&FN~bTu*PezSp*u1&_&BY9BwYHS}c%laD?|U zX5IUDLAk(V3}&eAag`4jI8|1fPPPn2Tv0KkK6ov!!PS*|38N9FhUR>fi(L=jXSRvM zkR8tns+Qi-gF|y6VX!&V>B^pwAk<^T5u_Ea^y`fkk!6CNo1lo}T61Rl%cg<#c`JJ* zeH;67$6QsjSr%IRf}GfzVpzh0=MQrXEPX!MGjOGiMUn|LQagcTCJ99#&#5sj=zSsyI*D9SagBx3Q?N? zZ?sy(R!vNslkRJ3t5`KV)O9lTNRNwP%i^SAw`I3M>9!*a)4*^30+m-AM%$)rpy$;aL7kUvgm40Ls? z2nEIy%#Q=lL;mVJ* zk_4c-;FCgwR|R7wh3?pB(ATn}HMLL7ld-YBfvYtWy}WWA7;8P+cy@c_9SiXO{xtT@ zmh-cgm~!)CkSsRPHT;MOViSE5{EOR+rWwskI0hER9sZmefT!wsN=;X&>7~NTz!*c+ zP->HTuSLjdNNr&;`CNB1TO;AxQ^*MwHEM=@O*#=bBslx*q($j5=ct0bShSRPbc3XN z?|mx!Ff& zcoMh3*2xmI0kAGK14&NOu!xBj>OBgWJ`l2&H&nW#zMrH9( z?sg@s-0Nb5!~JI-^i(nWj@9hjmbGNbx{8){x&kiU(AUW>sH0VDM6bk+OK1h1H2DiK zPk^LQr_>dtwKl)*B;Gi*40(MwNDxwb7(BpIOut)8j{W!IPY{a^Ks4`&EDgeGG$2)m z7=c-mV|q<3Hq)$1tE1iMe@aKZF3&{e?&PEcWSn+$w^);l-qv^H^&v!^GgSsEmX}KIm53DM@P~%Mq?J-H{`^&PJ}3)6axoD0SEGNq(}YFe5>+ z_$P{hKW~IgnNZF{QNK4ad=3TK`{VTI=|$*|H7}<2FN@PJubQ}MnX_u^kYirJ}WHqN!GuMQ9vVI+-n5wC&Mw%9= zeH$8pC9N_3IFGi!j#+WO6R~IM*``%AYEA}4PiTbQ5)ULBi31b1j`=JvxkNg%K_;Q7V&1Ts{eMZO*yW;Mb6*zpl=rcq4~mS zZ_AOvNCke?=5Q}yHl6r%gyyV5u#XsH2~23NV8(u9(~0*#%qL#T^e5u@Sh!8qUT309 zM3lE^-Ob{JxM_F8Zo8-;T9q$O{%pTV+2GtuOrA!1G%x?_-)DlDw%`E|5Y8dQ}RF<$MF+;?W1VYu9 z%2$K5mp9E`U~k3hh)t~8u@Z0|**qYdBC9cMCJ{&UX{#&2QDZ9W4h1^NrOK+26OB^tim5%l%T za^rxkh(X$(MgvYfwHy=DpO?{Cm`YCays+UPMfmW(Lp;(CqByV?8dm$g(^@L2p8^~% zKOdwqh=69d;cODT@!;WznYym4Z+ey5eVN*#@(U28Li|SIa8cq6!Og0w{HCsU>@UhO zm$DRILE_Ay-;wf^p!Uc-`W(c7xsXtcjg=TDS1K=Zka@c%^WsI_GzYex$@k}Le*4(0 zQ`#18;ruL3qHNIw5x&112h^&-rPxRFPw>TC+kMb*R5uJf|weWDNdF)ZqM(Zx(50tzyu@k#u$;p`Vc)%N*2K*y@d_U)XH) zTT;T5#5oN&TN03D8>Z2rTejd`rM8jDO<#8``YC@OU3BFN65mAc?>kX-fO- z*FQKe&i;W-OjsqSU^LUcjvIN!IFa@$ymRR!RB__W=ESn|vEgWO^^J_fHx6edpFA!g z&p5uYD`_CNtgp#JLP$N-4iQmHQmm{WAgQlU2)>FQ=joC>{mZ|KLa3~nV5qBd!fbgN zy&9l#ns9lB+RthihlRsW5Ea+Si|>gLWPTyf0p5(IG($NRE+tulXDF+DDYRBf{7}1= zv(+t)bRC(tgVzeGjAW8b7;sj$F;Q4WAVQbKv>#%!j-+xjbG6Z;pUo*oTDO4ZrZki0 z=y%Ivf8mNXJ#&aoZt46!QMkX<+1`7OSx5rTWCTdgg5QMgjMA$x7l2h zsJH3)E#uYSE_z?L@Bqmp;57RB!l`T@3;O+iV~oNBw8aknb`cu7O%AZ@C4+ye(@ToT zk!y`0Ty!#T-9&|Vv4N3!{1G(afO9W66HC7pW~BsvOY?@0C4p4Z^HOM4N%ml;v^BWd zNIGUa@y$;<&udn>Wv}Mj;A@J!&7qibj1<^JqVk+YxRKt3rQr{?CVsIhpt zDhjV(IdDvv(Zy5TI_(eX$5gvLdmSaeaA?oiyZVx6)+ZO>|7bMYl4;ZjU&u3X@U;52 z#tX5I47mngdT?j&kgWmhD9c57I$rXD+L}2>oM!=jH{$Z;PPrFlOgL-%>R83xBb(Mo~^F5&BR(q*3WV-EFqHt=@?Ym*tp z^`K)oeVcFd-H_r`A%+;Pea-6A^q&JUDwl%>*-0gL%u&}D5{g2Ofi10y6rnLJ(4C(S zD=I2B*?M>47lNf0vJHKOe4yNGJ_P}9C^p@>@y!WU^J5Q70 zX#1+3)k*~dOwwEtqW|aTZ@>QfYm8VEM?%LERO6n;x4sIe2Cf%@8~NBoi42qqDoQx@ zRr8j#qU1ht>XnvYoa+L|Fx8gqAWXY1aG0XSI0$Ewz0eP;!<1B z#b*4^vPF-(!G}%j?SA`Lyzj*S{%C7I6u%X}{wulg>n{oNq2B(HJgGM>wC}_Hz;R>`Xs!tUV*Kj)6HeZ$?mGv^Std9A(E_qN%WD1rS~B2ULl zk&Pu;#$Qy&rsdnkpqn9sw+pqI3Xdm%aWMng`m>v2HxzhJjjP@Sew$S3f$ zDjH)Ce;t@b5A9U^`CLA5-#FNYhN=e>48=q#Il0tMIx4>NS)8jdhGnBn$y(zsi8{o= z7Y){3=xR0s2zZy*_1%vxH(3Jxa7ss~cG7)Zz-3CjxK*_}l z5)ug01_eEv>yK>2nS)^x6HvT?426ervn507%#v16_%kz$^0?xJr156aMfqg{l@1F)#)f==uYYa-B?vXiLr#xUH6aP8u+s9~UOgFBhEqHW z`%2SRD!oYrD&cQNhs&r}*a2}J7D@Ql{v^(o(}cKcVaw3@eaJ$BEeA!;76kG`jJR`* ztGbOuLl{Ef4nud7H`Fp%jFxGmeJJ}`@7*d(M~M904Sn^wX(+mwlQOhyB*D!~7iFD$ zH;gk0>3hg99Z4$m+T;p4v|cbL&X)yz%O+#{EdSgv(DPkhQSpi9dxD20SSFHt%JAh8 zo`4Jav>-G{J!q~%#*RC@F@ak=*Gwd)f~^S*F(fW#dHCGA9gXMN_{v0j*SBvz&38eF zR;;NhDMsC{^~LJkG7-N88?&~<;M4oIyFdUlPUz;87?kHs=vM1HbA4SCt$7oO=tvWp zxPf98gPn8x8h_Uzv~|6TB9VM9MVFlTZrmOhnHiFH3k{n$DfxyGx$noaabW3w&C!m0Ph1(8LKuUW zSjDvu;smJ5Ts6KEZ?vvJ2JIjLxomW#vyaIu1zlBul*d3mnp7osJxVf*z$y$!aot$S zGbo29s_lHz3(JKR-#-_of?#CImfgsjOe9#%<4WBt_-E@S@wh?4b_T%J!iE?pcr~{i zAO}p2f@KkpAI}rAKO0#E67Dg6#Qb0oSot>HiB*8G$rVVkYtGd$XwY4ug1My7lx%JU z+LQ@Ut(GsCyGWi+=G{@@WH+KF2ND*PW(*KT2z@tk08iOQOJBG!?~&_9&jDIT2l@3IDfg);Zs zX=j0f(ml~FP^~m}!j|kE#_v(+SizPrCiELt)I$b#9DFU9A&ilvKsMYQO*_t&*LXgH zg~s^RViLUM4zq+Plf2Z#(VXDlL|mG3GEA?H$C*K=OKxt=t>(h6fw_eideN){`_anP}xr*tT)ZV7`S$*z+7nH!>|!9 z-IurMeo0eR?lKhXA2zl+!y8U=n z{UGhvCghXV{ay_WU4^vJRb)&MHI%nB`I~oSY_|05Y)spM=*0XtAzIAis;N$D#SIpVfogo{TjKo@F^*j#kTM9-!PU7gL9rV-ly$OCZNBz&!o32ei~6zyDFyCbMIgU z8x40jri**Yrv2$C=1HF*9O@hmoI*KpC6*OT6BxX30CW znYku4M9k$Naboq!jg-Sf=47Tv;I+9Vwkgu-^GOUx^?l~h77W`s>8>pt!$2Wfq*@`> z`lFsHSYvXIK;$P3^j^?yX|H!+$c?|G>r>ijZ#Py9RFPoHPqOF%5K~?qjTN)RvP4l2 zNB8#6b=58GI$8n~@@1kZPZqHFe6+E6ig+qhz&ovn-{}wk{mXB^{ri7@;q=Aw|NZjo zFJjQ+zc^(^{!{BimT$5lsGG<}v^6!6ifi$N$3O&V;#Z(4nn`eZYy z{SS|`&TIfbeenE<4j%ND@&JkA-VX?OIm9G>^e^b{K`iws7rX!;pdMkl=D8dGU!(;+x6=D zvaQ?AqA2JMsGgCEX#tnXwgWbt$I;Te#BP_6*t4K}KO(KSB-}iEd5l?_!9Ss9341Nn zui#?#JjhNHMop7tMt1{C*jpXr9Qi&(I)x)z&U0LEK<6wf>_B~PbocfU UI^mVsY`G|20Y^Tl9MA#)0JYDy^#A|> literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..597452b687dcd774cc328713219d9fe51f827ee7 GIT binary patch literal 531 zcmV+u0_^=CiwFP!00002|E*L@kJ}&+zUNnP+&%b_WLHtUr&Xn%`#4q=OU&3t!N4*@ zqV(5yu-#1}ZK|m11%csv%m-s3#3w%lV~y)X;~hnBWw6SD*w^KX|VVro}l#exzXz?vMm+fURR$UNtS3wurN_<$ChA$YZW< z3-UFGmB~a|mcs9*y)z-??9{;UtlpYVMxR)Xq6sLe=T+CgdFYA4HKNJ!!B8(dMukjy z?mwuAfTumHwqWsZY?A!sUuumzpr)4^EZ-@n?ZG(h50lyK^@7I?!AAlun*q5%MKGtw zH6}+B>2*XKfls*R$kjx+MT2~hTx0Nt*>4417>mkv@Q$&ok+jL$lZxC~`%kz($zFi5 zS2^6lYmnNrJLgF@{b+IYt4}xbnv|p5Mdt+8?B|nGHI$UKM}7-70+#bO42Eo^$^b@Lc#rPrRCc V5Zw9ea>-DzkXiUf{mmEYU~KxA_!#V@-p*fLYvLzZ?@}|R%XAEUiKm!d-9vd``Zt<_nTkt zx??MYy!-RtfBxg2k=*fSE1L&0?cME@bq~Q2`MchI{`L2poibA2_}1SwP6k{a#l3&G z!Joe?qx^x|&0qL7{I%(w-6>6*|3qW^pgQ$~+W&5XbbAWF`1c!SHXrV9fBWRq)0=-q zV_3^*RA2#q<6U#7)%Mo;QBjVx91L2HL z_Fe=ln$AzI_R`7DZ*FgIH}>Hb;cQD5OI|!^EgZ=fKV>27WMl=4a&Uzg3V-dB5f93= zLbYUq3dd`NO{|^Tc$H;s5s)o%gnQH@34XxI<|w4@4|1KX{sYtVz(2T1o$ZNg3_CLY9XIPb9Hf*T|3^YZH4UBMBGrb4!hF;UG&;pT)COaP=*FA>^3@7geH{nyNZ6ZO}Unlc|yIz*a**ZviQpk?e`82&6u%fr}iw*wDze&vmEk>=y5Ro#0j{9&k}MS zeWNC@XB)IM>(hz7G~_Ry#^#NIfqUehhBM{fqoAa6(x6xY11JW@kjuHD*2=AV<#^}3 z$~b&MLiIx}ztUKv8mUEcbQL*$LbYp!)uPyo_}i~sc*%PpgT6uIY=(K zI0UDvqWhWt*pfrDVypTX#t6$N5yS3_KrwJo4$)%Ns190sYZyB9!Q%VYb`t#NbePXytx(6uc_zmWzFoi{OC*x&*ITySIj|@DMP6i64(xTDUFYrRo)Wlp& zI2)IKIyHQ78W#mkAzy@H9y(nqdZ=%JqX!8V{%D$npRsqQ{5*@DJwVrIUs+Zu`GosP zQ~d!SC^#C=if^7Pa8H_l&+L^I?H=C3^sm9U+i2Hzp9UZdy;oM2){oHKlqNA^J2mXlLDwZ1&&Q${6nevj!5 zNToX(t7B(ag>3TzTxZRhbVS%M7M-&&fi`mW<3#7K~52@bPH?R9c1vCP!$Q;Q|wQN$#7u`3piZ$x=a3 zg>kxcoqshKa#*ZMGQ_xegO?m|UnQWRiEJ%=4s=5XIOC7XTuZS&>tFy3tlk}`c>*y& zK)QkB^+42%;WAD+p$9M7KQ9kMZ?4+4E+x$?C_d40=a6FAt$sF9Xloy0zdqzc>#US+ z!Y50cPP8&{B;w3TMEnJ4o#>ixX24d3xg*Qo6Q?zId-my?9_6W%EJ}FHb23DxyJ@iF z)NQkp-+5Kc)NV_e(E%K3B1!|x?2vqGCfozRK4ISZuCx4mZoac^Y!KragLmM6AhC7$ zQhu1iA){lZn*%iu3vPxO@XQcHod-&-C>h__14`*Q%8XM7D0(b3lo>sHCT$Se6GP#i z?#YT-Z$Euxo9Do!a#1}}sPiF@0GFc>U2dwxIW}FkQlEy;MyDwR3aB4AC$+D{+0!m@ z|8_HWAWLbZ!&;ZG#!OiByj;W2CINCWRUzC*&w~Nmh!Dd7P?}6aRy?zZ&p=|w2FO-A zedIf9j;#=bkuGZsE4n_Ejq)`ThP8QGkCzePWOeRHlto;}C2tw!^P-1c+E8qHB^?fw zn?xaOK#j;Agd5-s6^s2QJ5D*-lV)v@ms7`y7@EwAb z_;N!0KJv?2F85`GubULbYn+R+CJR4NnS!6vVt%u^CVKEL_tH!$Et(VWzyKnd1Ppv~ z;GVraYAY`rIruqe%SfXfFSi`G#zs1mH1{v)#jgVVz@|$x?wRAa7s%we{?YUw4x@AocBT**Pc|br%^kNfc zc8go(W;=&}m{PwVL^w$C)QOQMHSaQfQJ*S8ch7qXUmZI6j%@cXxv6MmKWgAIk~ zZlls|=3_jnPL4}BezJM~xJvqzpe)xF7rDYkN7QaI$&0k-?zy^87RBqt9a2vjS){Md zoQ^&iw?=lfuwbPNm`kz!gPId2nBRHhD8YTIiWqfTb;2^O{eo049iQx))c@s@=c31q z>^F*J(MntF!u+7u&q!@QO#gD!q(a z%53g$??3u? O@BRl3J@H!+C;$K_!+oj% literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md new file mode 100644 index 000000000..306bcd35e --- /dev/null +++ b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md @@ -0,0 +1,30 @@ +# SDS obsolete-format cleanup + +This follow-up removes obsolete installed-plan and persistence compatibility paths. + +- Removed old identity and projection field aliases from installed plan contracts, along with superseded deployment API aliases. +- Removed `reused_from_generation`; unknown-field decoding rejects adoption metadata, and payload generation must still match the installed generation. +- Removed untyped projection decoders and the unused materialization JSON/byte adapter and serializer. Canonical typed Serde is the installed materialization format. +- Removed flat-map metadata migration and old-version readers. Only the branch's current sidecar schema is accepted (4 in #749/#774; 5 from #763 onward). +- Removed the recovery caller's log-and-return-zero fallback. Invalid persisted metadata propagates as an error; a missing sidecar is a fresh store. +- Made the producer partition roster explicit on the wire. An empty roster cannot authorize completion. +- Corrected stale recovery documentation and changed old-format acceptance tests into rejection tests. + +Current semantic checks for dataset, definition, state format, revision and coverage remain required. Low-level raw storage formats are distinct from installed SDS authorization; this change does not delete current storage operators or codecs. + +## Verification + +Foundation checks passed locally: + +| Check | Result | +| --- | --- | +| Type library | 117 passed | +| Control-plane library and HTTP API tests | 431 + 8 passed | +| Data-plane library | 1,165 passed | +| Serving integration | 13 passed | +| Restart and new-version warm-up process | 1 passed | +| Strict Clippy, all targets for the three crates | passed | + +The accompanying `cleanup-*.log.gz` files record these checks. The restart process test covers same-version recovery without re-ingestion and new-version cold start followed by fresh input. Downstream verification checks the restacked Level 1/2 plans and current storage schema. + +No manual deployment verification or human approval is claimed. From f1256f07705ffd8455a4325e402a0cf17e88d905 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:46:46 +0000 Subject: [PATCH 19/25] Record passing restacked plan, storage and serving checks --- .../pr749-final-sds-2026-09-28/SHA256SUMS | 9 +++++++++ .../cleanup-stack-plans.log.gz | Bin 0 -> 493 bytes .../cleanup-stack-serving.log.gz | Bin 0 -> 531 bytes .../cleanup-stack-storage.log.gz | Bin 0 -> 6661 bytes .../pr749-final-sds-2026-09-28/cleanup.md | 5 +++++ 5 files changed, 14 insertions(+) create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-plans.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-serving.log.gz create mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-storage.log.gz diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS b/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS index 350b890c2..3de8a6051 100644 --- a/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS +++ b/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS @@ -1,3 +1,12 @@ +660508b5d9d2880e7ff30a4b57996f7534c5b12be3cbc30beacf6eab48f32bf9 cleanup-clippy.log.gz +2e5e821d81072af82ee16756dd1e73483c5e9069fca075e7d6a0b0c7cbcf857d cleanup-control-final.log.gz +56124b048b5075fc823f5bc5a6ec9140fc210e8ad6f716957ea1dbe2292f25ee cleanup-data-final.log.gz +2f3ca8601161e2c66c617c50454fe15c52f74f2932d89431fa6c4a5935343aca cleanup-restart.log.gz +e46819b5d36bd48dd1ec4a031d422b605683fdd92e48b643a98c1c6ddfaa0f9f cleanup-serving.log.gz +e8ed248e8558fea5c5836639f48c6445ffe97075d8d470ec2445dd63308666f2 cleanup-stack-plans.log.gz +c80eedc58af116d467cbcd92905826732dc83c9077d5e19036ff49489fffb455 cleanup-stack-serving.log.gz +143dc4d024d5663663a29e232f9a60b87c6e22719656f949d628744a50d7ea7f cleanup-stack-storage.log.gz +ce4cf63a8ee52e6dbcb7a6131dc5fabba8a49b3c3326fc9ad72f6eb14f205064 cleanup-types-final.log.gz 46e0a7b5a3c3282188347874fb2f16640eb6ca2d35ed9855d6ad03f0abd7cbb6 clippy.log.gz d15dc24dab9b42725dcccb3c27cbb84c91be5e45c5cf4015d9b1492b3ec2150d identity-integration-before-fix.log.gz 539c519be4bd3b50ae7fab2e0bad8c7a349bdde64d541ae79244f4110141c376 libraries.log.gz diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-plans.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-plans.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..b17ecd21f651ae83f5b0b8b95808e99db1168c78 GIT binary patch literal 493 zcmV%W=;+_oF4;~> z@WD94Theep{lAA0{a`G?*U|Zq%-;MY{S6Yv4W%nyf^l%&h^poj8a|KCO?^&aE)DZY zUAvN&4rFT)`K4FZTyV;57Bg9?X+3W88GgeRy2HM05$i$sUEdGO#Uh@Cu4k#mbVBlB zCnpCb`BkhtB#kDQOgcWX%rqH0+Vn0zh&)+9 zX41D%!GJk~Q)M>J$GO}S@_4RuMu~C_nf^B4&LQS>>PoZ?Fa91+^Tv3ZLgA|Fx>~h^ zTHR3JUc^(Mq~_tKRkR+pRM`%rr$v{8x7I&o(~LUXG~}twI&XXSW{V7KA_q@&gOA`3 j6yc@sTi>H9>-x&6^pio$dW2j z$+BNR%J!NB*drKr5hzpNgQR33#5+F(V~y)X;~hnBWw6SD*e|QwYAJ5&;iwg<`swr2 zhmQ&3vD~U=2Ts@3!N(m%0IUg+O85p%BI8{NE2uBn7KJ!h%7p((=0r7Wi1)@B>_Lm? zeE3;}=v!k!d`Zp+GK2XJ`l}!nJD?j}2;;>4rh0RaBU*e)&Si%gj5Q0$xeg0)ltHPS zma{jB5chyp#8UyNu|(p0If>L7wyEPaT+LTLG|g&rzgj)4??BzPYEcw%Hg>z7ycK!P z)onq($B?x*?%*qZ%pPpXEJvi8V#0S^%IFLuw6xkr3AHAWifZ~%pAj`{r)a4dYHN8Zgp V2!8qNYPrFp_yJ2w>fY7`}pp|?l13$ zbEl$u|Ih#V=YRg!r2YBrUbPR}cJJRkIsXv7*5Q5BA->hmx}BnN_O1S;-?v`Dr|;(# ze|xhlLAB0C@A_u!mDSs!m8)Gr!UZdVnU^9MyZHp)GSXU^KWn7cv=7R=TAFY{|)|k7ozj( zs2gpM#_I6pOL)++JvH6_%NNzQlUMEe%NP2={6QmUYrLCm7rhxn6HoAu7B&|b>kkjZ=NXrlK(Vm1vUK?7 z;8vm=n}_DXz*Q;uAALtRw+#@kRUW7_hXb5pf97LsAk?mtZ!f4w_2Wr3 zADRyzYh4h0tw&%3)kp1tg3P04ZfL9;bf|EE0?EAn&UoF%{=9zeqYDj)sUE01;9)v# zXDS6jtd*~~eCGfG)ppA5r8#c{Pr}I;6jwgTjSD55c8xz43?d z-1vj?psgBEW=Eixz4r0E^&B97Vfp9A*w$+mbfdi9y7e#(By_i}0xJJ;fWvz|C&npK2@w0RfvEOiZ;fu8dPVTF)Z<4`rMW4N zQ)~te7Fk<$Bac9j9a%3(Ezul|_N-h<6=~Q2SWcmV=cWZZ={2B%L_yTsSl&I!P}MMM z@(xI8!X3aj2lb!@XH_?20l1w&UeQ4XLLf-W+s$ZhKwj+^9w)ApN947Np;zIg?+7=r*mWMBz|c}nKp>(?*P{g0U;}_$ zrC?Ko(Wa`9ye16=rYpuF9`u=)8n80~Z22=fPN!*5Y)T^~-T79@YDDZ2hk&dA zF-JEMDgy>XFbOyV&XGs)>A{mFoi%w&AEQOhpH{u%pmnDu<^|u4bjs+W>Q{hCDnfyS z*k{!MM1!%+zWM6*Vg0JvDiOP7b1eDRzA=g)z~*xUNJu?tjxeTp`1;i`8j4Rdm|oUJ zMFW50-GH%8CMP8%%zlcx2DGFyBltMv)3=&Dc#h6iJXY4tDEQFgf-^PrHQXlo87ryx z$B+i?j~ZPm+o2!R^06v`V)KKU5#$e`<Gj|+Fh*$0pR#>DMRz!?hj!F4s}=H# z4xnO_@e5d6zjAM2dw4nY2c}G-ilpxl8u;@gr~~vnb=*LBgYplv1ZaDQS{kX1lyn3o z7YZ^|aU&!V&-g>MbI2eWfXviU*9`^)Q12<4jSfH1^bsDR&k;owodozdon#FZECKQ%yiePA}KqKj~(WnB7;^YRl3=WcRqq>UaSORUKHjVN|Rk zHW~mNNih_^9!3{w^GD3KV9pk8=<}?Md=}AL0Y=z%FzCmIWP5^wNq`CoM~`omZz7&e zVWI2ilbQnMjMs+8Qiw4viBNP9X8`O#Pew>_&8UBsQ{J!u*IZRVU$DM%NG1~@q8O7_QIE1rL zm?I{&w℞sg<7e#0X1{XNg4fdIJrErkCmcl(D4r=Mg!^>mwLBBzzR6miXuO5R@Ju z0Wdxkxl##Gpe%UO=6H%=oi1>C0i1sR@5|!~m`_YU&e6T}%N@cAWM-RmnA20%G7v1E0n1U=L_Z%Uu$Xmd}2%gCv9b zx$QvlCcgs=a1C?{xIQ9GtDoSQ(}?-g3g<+}TZoJW3h<{paBz!KZ0TE0K`qFK-##Wv z;)Ag!>XL8(@FP(!;1GkgpkxFA<^Em`zMZ^BwajbGWl9=MTI5TVF&qkRPR^S@9l0O| z&Gqa(&QKVzQ$KD-g=iSJn;z8>%)P;&!W6-*@Emn<4P6mH{M!im@G$71@If}%v zr0@04OcOCBoVkI{93KS@I(eM)2%H4A+4NCk7Xt}tkY^$|tDappEKWdd3S7+md4z+33< z2G%=FI!zY8p?V+FOYKCljhfJtc%w`KegYU3>>tPqUNV>_nOnQ*NE9jpn^c&vQ>`{a_Z5dmFwoufwpzd7?kt@UJ8Zj;u2S0E? zE*sUkcdAPU>K^4Nr9OLccr51dg#G%)RxmxSmq$UYQfdsC!#Yh?WcA@IU9;slF@;q!A z>W&lh*d5Yg)M;H=?{}(?ECt~3{~nE+3*$KOuoih@{-I96aEQ;r_hhUzNFpI|UyCR` zoH5F<)PcBx%Q?M?*kePt@c$_^uSV6vHBthE(=6S}^XA5p+zz&^GXH8dYBmuWFX1F9 zk3f5mEZKZfXb@^U7y7OZTO@KIF`X0(G92Oyja^OyF4vqF3rk^x5It}dTDC7pUMvTS zt8OKPZ<7KQ(n~yvB2C1&ygCchpp2HTk5;0kIZ-7dW)f+y_#? z^^dFB?&6LyMJYb|g(X&FZT6AbA;tgzeOY7SMAb!QTad_O(kpAh(3VZt=a395#fIc;I}kR=o4Jaa#YGWIl{_o<3oJLV6EH_O zCpFj%RAVEF75)rV{V?dxz(w}w8ZfNJ0vG)p$q3cYK*@%>p3*5h(7D(%7dAly^Ae$z z$qxzIVMdC+TuW^SK)@;AJM|Ir$`sTF94#vDyIlKta#$qME0FUCiLIC*aOFisl zTdlB+&D9E>?xRW)E>#$#W(9W-#4zXob3Bm(iQ^(e_y>r*$YriR3ZpE{JAKe+w7$SN zoW@kS%$Z-RX-15~BWp>KHryHt`zctm?ZRcWh)VK$2X_r#+OF&AGBVmq3+` z5iC~jx06C_u^TDn?B{wZX&{j&@J$me)}d-$g8c@F9y6N4c4`l%^*)fV&hs5G_P|9m z=u&n@9G8`p`n^cE2%OKK{wP9Idwo)mW>x9JcR&eg(kKy(HglaOyLfm&PH--4ti3uJ zNMlgf6WWOqswu7QTxre=ZYeMc(}=~Z*pbRdY_L*V{7TMeq|$SZ2Zt+Ic_#LW=aUho zWB835;VwOhP#Z`pHlb;Jt$xz zxMfcH4?##uAV|AF?iJWrI`y4C6za*9&`;K}J@C+gb=-lbER2C$n_Aatu7bTA?9?8X zua+2OX${6ws81fv6M%@VTi72v)9so=U-AYEUgnTAz=oz(;P-6oOEQ7w_uWpJTDXu& zxzY^#@>u8I52M9XTtF>ma3ZT|1j9qCq1qkm=eD0ZGGc65ZwfNb-jRk*F^jT% zq#~%4@lBmqOG^}PFP|j{g=pG%Zy6@l=q$q$aQ^U7w`&iiDO+$Zsw!v81sxx7V5c!| z4GpG(O12O{HozGS8txbelx#Rwbd9bz?RhJZpfPVo^DPo?XkoLSlGD=kv6Q%Cx=zNac9{sw&7_@4#BGV|SVF;)+PQ6DjD3bS3R; zE5W7GU&Mg4p=}H-a>9uNT^v>bFVQ?=3M4X-O6H^ssdH?RiI(HH5+*Op3}=5T_Xj`G zz^fzpLO%70kwQohRJgaxE-`HHn`6u$-5q9TnS62pY;oS8*amLZs~&SgoFJ8N7T zf5;Smw2Dd+`MQAS{CHBf?vYF?r^|DZ^5H`L)6Eg!d^nC_1aceWlCe8K1Ug?MI>h8% zWV)bD)B+>AjN@i=KRG$T6#*TovT~w+82OH+kKC^EZPT7809z3tM=NHCIM~ZFn67|a z2O?`ESv8ToCPOIIuGeBXd2W`2l;BIUiW1;6@5zr>*pTQ;o3`MDdEk@emxn0`J+d9Z z?I)CW5%-jhPY%3Q>Hf$D1{YXoBQrmlML~+|V|nq0q~41}g#`1@I$l(+=fuhKOv&Q- zv$m9}3(ZOOhsKR5R9n9z>biAtUBCA$k9BF>rRfIZZdC`1Mm9*iv!#|3Ad$PGEM4g3 zyq0)ccb-@6CnIZa}VK5wUN?9|?240R}Th>TeWL;*1q-YiMl7<;&Br$iurdn2VjM1#e zS@e})QrZ#oE;7Q0j%Y&Fb4?Z_xMbX=*F5l)c0&O4r-QZJo4mb2rt4lipn%yzi5vCt<^6RC^S84jw2kfQQKHx zYVMLhF3Nb_0x1S{j*ld0GSDQHz;NF5ILdsRMaJAM=<#E7B;#IYAIDD@c&l)_bRLc# zcbaG1AQs!TtJ82(WY<}Ube*C5qY&j}rR$-~7iU6_h0r5u2+|S_@Y7usjZ~?CCV)&C zR4CwA1V?3}SO}g@pMdIxgV%owikXX-YEJGN#H80W+!c00Wgf+cB@1ao z^&n%>tPB9fg7PO5(1dxe2mn;}3joH}1CUykeg!$EVPe8@nLd&)3i2g#P7}MlgaF&hP3c2fQDa2ESFHxcK#&*NlGyjTK%*b z=8xT~>*|#hPD4ASb>kW5AZdk5WykB)kV7}O7`qW2T(qe;Av_PpO2Z~L0w@kLp!H8& z#>F?{dhcB@5#CEhyxc^YUgj~`AnhRvN1S)l*z?Q5T!)*2^hob}hePbrxYlxLVH(`b?^~%GM@sX8WlXSyj;Z|LbhQGhbBHQ66!C>( z2{nc+tHI%5a0=K;-a^DgCA>-S8EKO02Ihbm56(n?0;?%Iy(n^f0h9((73vDZ&8df( zzPg&SAr%#v;0lwk1oUdr3>jZ`MM)qZV)f&pvmv9-@}M=E@g!rrDC{Xasf5s8-q#z$ zMdL!s>n?8>eqC5RJW=fn*YvT*S`rukPOB`>7Uc#oa?9kED&V(0h%~1RQ5~ekPkaO+ zbT-oH8n>K*aGfzv%_B0NjYC5`v~6i;*-s=f%^DDqbQMxxU3*>5n&SgtHy2^Tc8OuB zo^(_9-%U(2%w_og9)R5Z!aY;?R^Hz5i!un@>ZXl6*D!V6(dJX3YGr2Q!rh$5Rx*T> z3U7kEO?!Q#I%0dS4BtFQ@N(}+MwzjtraxE6C1lfh53flfC8d!1Bi6R%%FBEiIK97@ z#>WT?USLajXBku8I&CV>{B=i_oo5SAhRU$S>)6;|UX=J+S4JwLv59EV^9XRV0_jEU z$NA-pT#Ct0YyYU$v&zj4rOTMBW#*>P#5cUxkhjIvtd%#C#uBBH46*Q}vL#LKN>i~K zv1hNheUB>=6EG;+_iJmrmsrlc{8^d^*ICU@nn%-&2OEvN zwq3#t`~m_pe0dJlfeT?)N%$M+;Q>uWl!c4Ix#FMtK?>v~A2dI6tU+p(JFdmU@K>0>LnwIodUM@iRWj z!fOtB6jFWWYc|^h4~;@|V5RoQKyNXMGm`>a8a$7W`y^B6%N(8C*t(iubW!wN_JbiY#H7E7}z-Pj<_g)L9khf$_{MG!;LO@!TRwZzI!KnZJ-k z*YyCaZ4htaQEI2crA*2$-8P^v-)E9SP>D-YxSu5o6{@oqH2sR#s-z;Z7eg#_<7${l zC325gGL&)rve=-szs+<@)m%)M{6b_9P~+(wX!`V(yg)@wuGrNyQijfh3BJGw?{=R* zf7*>I1l|2(_hH8-EdB+YfVjm!2dyyD2Y>(mw?B3q0Mp$L;QV8U$UtvsXDlrB?vH;9 PZ{GYruE_&LjgSBUx8tUp literal 0 HcmV?d00001 diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md index 306bcd35e..109cb6826 100644 --- a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md +++ b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md @@ -28,3 +28,8 @@ Foundation checks passed locally: The accompanying `cleanup-*.log.gz` files record these checks. The restart process test covers same-version recovery without re-ingestion and new-version cold start followed by fresh input. Downstream verification checks the restacked Level 1/2 plans and current storage schema. No manual deployment verification or human approval is claimed. + +Downstream checks on the restacked #775 branch also passed: Level 1 (3 tests), +Level 2 (1 exhaustive cost-selection test), storage (332 tests), and serving +integration (13 tests). Workspace formatting passed. Concurrent remote fixture +fixes were merged and retained before push. From 269c774108b39c32fa5342ef03154df939ec483b Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:51:31 +0000 Subject: [PATCH 20/25] Remove generated evaluation archives and keep validation summaries --- .gitignore | 3 +++ .../pr749-final-sds-2026-09-28/README.md | 5 ++--- .../pr749-final-sds-2026-09-28/SHA256SUMS | 15 --------------- .../cleanup-clippy.log.gz | Bin 192 -> 0 bytes .../cleanup-control-final.log.gz | Bin 8687 -> 0 bytes .../cleanup-data-final.log.gz | Bin 20215 -> 0 bytes .../cleanup-restart.log.gz | Bin 374 -> 0 bytes .../cleanup-serving.log.gz | Bin 531 -> 0 bytes .../cleanup-stack-plans.log.gz | Bin 493 -> 0 bytes .../cleanup-stack-serving.log.gz | Bin 531 -> 0 bytes .../cleanup-stack-storage.log.gz | Bin 6661 -> 0 bytes .../cleanup-types-final.log.gz | Bin 2812 -> 0 bytes .../pr749-final-sds-2026-09-28/cleanup.md | 2 +- .../pr749-final-sds-2026-09-28/clippy.log.gz | Bin 207 -> 0 bytes .../identity-integration-before-fix.log.gz | Bin 11720 -> 0 bytes .../pr749-final-sds-2026-09-28/libraries.log.gz | Bin 30489 -> 0 bytes .../offline-evidence.log.gz | Bin 494 -> 0 bytes .../restart-process.log.gz | Bin 404 -> 0 bytes .../pr749-final-sds-2026-09-28/serving.log.gz | Bin 542 -> 0 bytes .../pr749-sds-foundation-2026-09-28/README.md | 3 +-- .../pr749-sds-foundation-2026-09-28/SHA256SUMS | 8 -------- .../pr749-before-fix.log.gz | Bin 722 -> 0 bytes .../pr749-final-clippy.log.gz | Bin 173 -> 0 bytes .../pr749-final-data-plane.log.gz | Bin 20271 -> 0 bytes .../pr749-final-process.log.gz | Bin 353 -> 0 bytes .../pr749-library-tests.log.gz | Bin 30272 -> 0 bytes .../pr749-live-before-fix.log.gz | Bin 589 -> 0 bytes .../pr749-recovery-e2e.log.gz | Bin 626 -> 0 bytes .../pr749-store-before-fix.log.gz | Bin 488 -> 0 bytes 29 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-control-final.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-restart.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-plans.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-serving.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-stack-storage.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup-types-final.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/libraries.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/offline-evidence.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/SHA256SUMS delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-before-fix.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-clippy.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-library-tests.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-store-before-fix.log.gz diff --git a/.gitignore b/.gitignore index a90758c77..0461e0072 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ uuid /store/ .claude/ + +# Generated evaluation archives stay outside version control. +/docs/evaluation/**/*.gz diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/README.md b/docs/evaluation/pr749-final-sds-2026-09-28/README.md index fe2280b85..3a91b999f 100644 --- a/docs/evaluation/pr749-final-sds-2026-09-28/README.md +++ b/docs/evaluation/pr749-final-sds-2026-09-28/README.md @@ -45,9 +45,8 @@ Passed locally on the final implementation: - Production-process restart/new-version warm-up: **1 test**. - All-target strict Clippy for all three packages and workspace formatting. -Compressed logs are verified by `SHA256SUMS`. The before-fix log records the -identity collisions exposed while extracting the contract ahead of typed -Count/Rate support. +Identity collisions were exposed while extracting the contract ahead of typed +Count/Rate support. Generated logs are kept outside version control. The production-process test covers same-version restart without re-ingestion, new-version cold state, and fresh input becoming queryable in that version. No production workload, production-cost, or independent human approval claim is diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS b/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS deleted file mode 100644 index 3de8a6051..000000000 --- a/docs/evaluation/pr749-final-sds-2026-09-28/SHA256SUMS +++ /dev/null @@ -1,15 +0,0 @@ -660508b5d9d2880e7ff30a4b57996f7534c5b12be3cbc30beacf6eab48f32bf9 cleanup-clippy.log.gz -2e5e821d81072af82ee16756dd1e73483c5e9069fca075e7d6a0b0c7cbcf857d cleanup-control-final.log.gz -56124b048b5075fc823f5bc5a6ec9140fc210e8ad6f716957ea1dbe2292f25ee cleanup-data-final.log.gz -2f3ca8601161e2c66c617c50454fe15c52f74f2932d89431fa6c4a5935343aca cleanup-restart.log.gz -e46819b5d36bd48dd1ec4a031d422b605683fdd92e48b643a98c1c6ddfaa0f9f cleanup-serving.log.gz -e8ed248e8558fea5c5836639f48c6445ffe97075d8d470ec2445dd63308666f2 cleanup-stack-plans.log.gz -c80eedc58af116d467cbcd92905826732dc83c9077d5e19036ff49489fffb455 cleanup-stack-serving.log.gz -143dc4d024d5663663a29e232f9a60b87c6e22719656f949d628744a50d7ea7f cleanup-stack-storage.log.gz -ce4cf63a8ee52e6dbcb7a6131dc5fabba8a49b3c3326fc9ad72f6eb14f205064 cleanup-types-final.log.gz -46e0a7b5a3c3282188347874fb2f16640eb6ca2d35ed9855d6ad03f0abd7cbb6 clippy.log.gz -d15dc24dab9b42725dcccb3c27cbb84c91be5e45c5cf4015d9b1492b3ec2150d identity-integration-before-fix.log.gz -539c519be4bd3b50ae7fab2e0bad8c7a349bdde64d541ae79244f4110141c376 libraries.log.gz -f6756607f21bb4fcd7688059d4b55d3ecd36f88968757c30a9374f0b5d8e6735 offline-evidence.log.gz -235768d931779103e1f4a6c9a998a236708d09650df196d582dbb41f8cef8fb3 restart-process.log.gz -89acece7344c0ff746e93e7178771e5770381f9a6b88b295a2a7f2a99fc1b437 serving.log.gz diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-clippy.log.gz deleted file mode 100644 index fe91d31512265fddd2bc437edfc0b63f26e1554b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 192 zcmV;x06+g9iwFP!00002|EW$y=AolF&pTS;9sapfdEQx--@bmO z55IoRO;@+>tlRvQHOe{dvi4+6cRXdC(>8Nm(^RD{vOWB;aJvs5KJ3hS0@$g79esYu zE(*W_q(i0tG?i}o;rVBaLUy65=A?#udDV8W0I~Bh@{8l8veIQbGw@V)oqyyETRwmJ z^eOE5tSlfd1H2wHI1z2z+mh`cO;zX`wrxvz;99n)Oi1g&`X%fZ5M8ROq!%`stJPU& zxpI2`QA=;y+Oi2~0zgIoDcMAp!y(pfc-FPXs-w=}u@=ZA>ui--eTB`#9vtw2y5Pr+ zYEN6iHCDrRUIAWLpKEii2^tWEu^AJ63K#(9=aT`NOGvW@@~_*htXqAAqoiX11OrH` z|I~R)Cw!>PHB+#s;#R}1^Xyesoqh=YtZzWouDe{6y}=Sq*CN%_rUl@@RyooMPR9;$qx1FZ!Zg;jHCy6fpZx^(#274fKq7Tupaa!oat=N`@6yq&2`E4;G{q$b%44S zM0~9k(74tWL8(A_4@8q`;C&LQ3$J#t=8~PXDzshkQ(({B9BY#EM?mXU^^dqY-CUWg z3Pm5{$n7NYhFyAX3+-rPzwxJ-DVy|Gp!(t1(w{y!S`ik(KYlL&@-7kaP%5< z1@IOqoFQvX=pkN0_Q=3VAfjY%LBcxx^%6jY6Oa&J}P%oh-%~hlJVDiEl5R-mapcX+F98tNo zs_ooT5{vj}`Y8(s&}$?pRI_9oz&QiZssiAJI%b8*L6g)2epzTE0K(cV+AwIj6}RlQ zRH-qt{LrJ!hvPq#$Ls}+IyiPf^#yF?Wyg-{^~?9~f;oH~l(ex#?sEQ5?dWASA-XQl zwR2=U^{{nQmud-sKOMZ&D6ZE%?NL*%z{BW~&jOmGPmp-8fYwuqV3ATBG6CQgh&I~; zQqe?M3oXYj1v9ermgT^PngMj^KyZT0GR-+rLtq4!;18f`vU)X4zd3lMkw_9NEo?7gYdFwI%AFD596{ z`2uKc;XinoorA_SUFFr~x308k$7c8(e272L-*w)#%bBSZ*9z=wpe{rQSrQm`02vRC z2(5*Vmj%=;g?hT}Em-T?HmP2^lqkP~Hz+Eq+herlkui8NuP8XstqROQJ0&hygHl$= z0EfR~WYYP$6JZlQ{tl%TC{NU$tOdz|SZt+XFa=b*sliGGH@*R@yvDcOkh z-STr%4&*5lk!L1kEfligiS9jB5fv~kfz4Xs+btH^QJ4s~Gc-974=ya<+OA$F#*pL@ znL@KAT6t(FG1Ud#z#=2Bj9ac7gvZD&Bc&5L3niJcJ>!* znjRKZpjF^jd^Z2r!UAM+z8wgM6)I z!w(F*`nW}wJ>huSC`M*6Xy6y?_FmSokrJKU;K)TSRgw{oemSA5K(5+G^(@bDr@BVY z2KNw71Pz`RUkAMkq4)d*q7bo(dK^ChxBa|kDRVeD@U8s{NI1Oup|kK0w;rPZAj2?- zLqP-QY`cnGe+9CtY01_vO;P5sP6fPu()pQ!D%+?OquJzB)vvlTL^McOWYJ+*ocw~t zC_$AFym+q~9PII;@R0cxmct8jQfTm7XIh=e0*e38_Q^t9Wfzcthg@G0SzJ1JX^Ozn zNpgT4-9K2qTV+D_@!McH{xm8^QuCV6dabpa4uj;yf=b&ZU@`A=5*g>M;0`G?iBc18dI1Xy5 z)+}E@J^+ngqpr?H!q*^D=y+NEd(7iH{=i;KeO!jVF|%NTw5ImMd>OR6g#+vS^?!!c)dco$05VTI71im{hI{1KSnF41}Bthc7p2Jtd_F11mgUAq= zZo<3>NQq(UZ2?50`81pf*1kcruSF4IP=Gdf*^>=!E206@nnek(j+g-X5dMxV>p*Zj z?dkt{H<=WPB!@&tA=went`?vqeVCx54ai1urPTqsOl5Ee$5k>w&R_^}jBY>-Rpxf# z2ny>Qx7NE0{y}7Z1BwwWSYywXQJ%ia<=7vD_rG=En-i|^L`j#O6bhG2UNw=G;iML0 z%gZj0pRQJ~;D`t`(U@x)YY6nB>JB)6t(_O(koIFKg-5F|hVpm^)m1HFC2B_G2XKj& z?-fnpM7})bjxn&lxsgN>eGbF>RY6WH=F9SM&`cX7^1${rWM>d13~AnN7v2;EiWIHT zFz`yd4HDCp`)Y+0OH4e9m-X}EU}VIkgExhjNA{+G3#7z&%nDl`+6{08RAK=Sg^Yvm zfmie8){!r$>3~$g#6IBPy`!}u{!FBoQf_eLfNAz%exAx^H9NQ6h0Yny`$w#{K)V-p zjay-J`TztQU>U2K)Fm8A+s^qXmy5_f@L}6Tm7le4(2HtO>LgiVuh%e*3@^0W7*~aA z+Ck)%T@3cZW1Js#F9*$l@-Pavg3D7{2%$p87~4{AoelLvTQeC8*+67nEs7_>KV|28 z!l=?mmEXwup{QnVPT(VN!9lKUfl(~9wWTO0i5V5!{LK(U=!Kua52{3`;(3C|xS&)7 z6L72J2IOGK9K+ZRyAKnPde(?*DM|qPF4oZ;P%?wfo|!!wsrV(J96mrxa3fF+3MD6B zg%9#Tp(`?IfJ*kT`x9`~c?D_sSO^CtPg*rTI*Y=+Yz4KL1=JfiT^mFJ(l4pGP_D3g zyG)#{@-V`2z@5QCu@8zFbx$-=4Q$*q1|$K$$-XJPfqBM}VzL>yZEg}folPK&>IjWt zv5uh#$2Nc%O2an|IY!)Trie@sx`Xw$K#qaMLfu>dC`sY1NNv%a>Y5^yCH2X9%SNT} z^sm4HReOBCc*FiwV(K*tw=Ez^whKJZ2Kd?b?S1-m_5eTs>O7t%8r z?gfinj3gCv9EC(8y;0&62+9Y^yL1ff|pB!LmQ&eA?89kOtzJJwKCqLYE+G4y%=-|%sJNh1bdW} zL-Top{f+&FyZobfuIHUZsw%@qAY2k1b?{88eou<^$qrvgPF2N2-2%KY=^)N8LR}<% z;P{b`<2?a6fKb&Httfbv16Ejsb{UEW!(yADQ}CcNK;&6lV(43-dp(C`>G;>}z)|k8 zKJbyH81jih?d|YeG^Ba+V7%wPoX8g9rsj>&WCfEK=LE#QJj%G@Qw8)gN-P4&nwI!q&ql zr+W{^5~YE{(FUAkgVW!By4aW(sz~EXgEIhr0F4{6ycb=!PcG$m%+7k=na*L6oh-QF zMVJzqg1nw&^gkq}M^J_4`>@cf(A8lBNZ}wZbO{8MV*JKQZc0OKUO;dWP#W|ZU_5>Z z${O@2v9jmv%i|@z3@Ss9f^TQ^EmSe<V-tc@==I0UJ%qj@& zO&=0l43;!8OXMJ@_c!8<exvM-O_wC8`n(I~R0nv9G%M73L{#l0lKlzz6)^cO712M-mTGOEbJ59Z13PHkZ2 z4|{!2Rf92Ym)r-U$A6bKZ zwC{P56Jsvj$4{(Jth*p5s_GC&$ zGv7{^B&>ado?_q@`8guFxD6TIFb%t@DnDEh{&N?U3%QDe>Q;=&}*Ask?TBr`L zb=gSEKEBfur9DCKppN6a904R28D(IT>E-l;aB6cMp^}U

s!glNdlZp4I*d5r_Vz!>oLe>{hZ+US|v881<6Dbj>|qau>*$u{u*uwO6o75 z$QVZys4*(|Hf$A!Jd`!fen>09-iQiO$wZ?mQp@y7E{B~cf;Yvx1j2^QFTK87aKfUc z?pL_5{~8PZBpvM^pN>O3TuDHYq$EcYoRp&95xgFBJUc2yb{u<%^_ch#)|5X386^v4 zWk7cM8l$C=jztUjmUI>dM4Wd#;487#uXC>`=2ZR2ueb8`qlN#SOTGa=4kJ&%bO5|6 zkb6R~0Ve=pt;zG?2W4liX(zp~HBe=L91rxnPQs7RFA;?YK`bT1#ytq!BL4g$k$K{B zG4N_&jTOhFlkdvgY|#OLf&`9g58C=mQ4$K zMHuIEm2^ZiDZ}mS_?JT9*vR$#0#VR=I#ac~Y8!J6n~am;GN@c6wm!MgrK{R9Ziv#9 zQa|`6dypOyRQ@h0XI~E(EjY`e0dR~DICM<*PGwp3n?RH|=kavq!(k7LfQLecV$45% z2C^^7EVpX82+9xB!fKu8G-rfKE>r)-pM;1;S#t86^r&!Xmuuey?q6~KXi?s6+Fh{O z`N>^-z<irLqxb zhyi$zKJB;;UeZ~LE^p{zEOgN|lZMa~Fd81yS*|N#x#&WJUfl8#{<#h^OGbEk7v$~c zSB|HVygcY$%ztIM6t7%XzPhw1V%1a}o#5aJZqq*N+ctn6XX_p|O^U57m_XW0z5pyxfxrssaw2OQS%D?OZo_1TJ-o2J`TkohG^A)jlH)m)Sn2^rk%5y^ z^-3!2pDuXu6rLpRE3vQ@Eo7aLPQrmRj2Aec+XRML z*x-UHiv@9i?$MjAOIjPH75-ED;iaj({7#jrEjWwQ>SeGZ4||Nzb3^?KR#bVKwc7@e zhe@u2IG4PS-RxW#fP*@z)iL(c#-9BLoqmIzhn}DdRU2WkPMnO_xr9>rbyoM#hnBE( zYz^;JCU4`k8g9hG3X2tW8Tau!jO~Ta>kt@u5Y?N;yzEuX4k^NsYJZHOd!q%%mRlTs zvVLL;*Ib1D3qb}~A96P({2!{6d2)43-csm)($wnMO`o60 z5%Bdf+^OZ}*l~eC=loBvA)Q-%=gU#nix%=?u;&2Q;Q5pN@h+}19KQ!D%DEzE{d73r zFy$Qt&{y`A+M=ZtsaJq-wZ?aFtYYH3kqj0LidPzC(@=j(In(2MpG9N0M#`wA5_+=F z+&Arp#9M!v;V^{c6~fp9X@Zsm=4z9X2?&e3bwb$+C<>hNNF%hlQVooP8oRrww;c)N z*n#b|cFQQ*aOzEsHZZ{p=UeuMY%Fh`m!t#_`Ibia{8WNdMR`)%d+3E@u3^gr3Bj$R z7O6mHu0rBAq)_9G5a0w;1@dugu|o|#si7k?f)L}#=%GK5#m155h>lh@K3>b}BajbS zc_@in_^0R?dP^)VIwd3A093pBOhT{95bH6<$5KS*w5GpV`#yq`kPhb#IJuoc=wdC* zm{K+-XoRSTWN=0gDjStl3kV1)Opa3y$^%wUS42Q)?Y;O8`cc1S5r|(s2$V95bp~6& z_~@U5Nld=8i$}GfAVJfyJW}HtrZMiuBt1qSH*nKr8k#)9)3oAvsR!=G%$OHHxJix# z`$2F}Eg1&P^;k`gFTijaMhbtT$PRr{&1L|MJ(80o@r^6!m(3>O2tJBcu5U>*QNDX zJYAltmjV2P8Z4O(@T$gd;j36Qh`(;0UpXLRBp}m)@pZ zHB#zsN+69D;)Y8hG2Z9cl)zCLv}fgjIW5ff@$?4%H=c5^pQmzMThJ1Vmy_~o?CE89 zg&VBLJhCN=^=C-N_P7wtFEDtA6UJxWgsw;E@Cal&z|g_=Ry0`wJ{0Jm5|tZg;Ij9* z)e-O==1~Fhy1ZA^%zFr>*b5E`JRNe?Fe6a4p5{iv2+PkX z+-2ucl{8}*KTHzLUW}V&HXn_zEn%jl*oZjDN5lYAG*~<^n+LEQT|iKp2dH;@wDN=) zgfVgC*y@WP!W33&=#CP&H!9m(?f2ciflqYTMya`bah3J{E(YDBJwKcz%8fH^Ixil{kwf!_59@ zd}a#xu`+vA(XD4RW(c3A@a73`4#C?~T{||LdkS8Ztg<%H;RHPL%DX<4 z?URFr@#ZQS(qL$Atnv9L+3cbhkZbq+xLp4>jk=LUDS51OhnMQXp|qiT&<^WWs0K6E z2l!4XzD18q$yj+U-xld!NB0%l0%Z^0Ug$5avv+ee*9l21^8vLT)P&^3D9Ra}V)~J& z%oT4-jL4SiU3RViBu&TUK}ds#WNs)G8O#9ix8(74J0LeHEzw#YTr-phW6uV`GPmm` z8aWJFA9^t0g=6GaE+z2@pT2=|+IHa*d8NQ;%|T>?@6qzT6ksS_A`(Xl zYW!BMcz;$*D_AON@vhC?-CB1$OD;bqYh_qEKl?!0>I+vK_J*&II;^vRk~@^20@E8+ z5-5wzMV#3?n%2C7=&$3(G?BeMy*gK&++Emy2Z}oF=uTiV|J+2_`0hDUo-#7M4(W`; zbWzwuiZ9H$N0UG$T^wPk#J)qvVq|`OnCr;V>G}Qo9eM=QR(t4>JxN#RJS3YYlEz93PwDrtCZcmRe;y6-p#u~z6XLpzQ+Xg~$oi@LzG_1%HTIY& z7zMSZNFkl9nrKCe7t@H)FuZc&-o@qg8@6!*$K;2ZBXoi z!kMq-kBlY|?}5hHIUQ-ZS(n)#-pR_NE8X+=h5K7lq!w@!^3&9d zlexk&5EBRQ%Z&^Ye#2m6$5$$k%pIN;wjok)K7YnIfCVqem+7@U@?Po(I50HB$5ZFb zbR-wjXvH%{rfXmqgO7y|{)S2ViHp~a_&$U`XRm)6KZt$`!w;HElF4L3VQwHKGZ4^y z&F#84o&TIe@Po7gUl-V~N(3Y?FV_n9THar!DDU8HDj7oM7E~vXOcGDJ&U&RS{`Se9 zLKHQ*MiHjBLh{6H>DCLS?tClo+CxvucV(+dVNi&}xraB3&GWVo#i1|OLphfJS5kA^StXqd~<&3i+!D4VC zU%1(uLvTLPxW+(`B(0cQH1M58uFoUV-f$rZ3c6`(=cnjJ5MXT#%T+-gs%Rh>8R&sSza=oV+ zXwmV&9l6e^&aW!WisYe&Ci!?}07uDbcxNOw{BYZgPa>i-xuExFMkvRdEImm8baU1& z8(iTm7fke4hz8O9-Jf1xV8NDLRVT#6RLz$WQ#@_@0?={nBp0rxH(FA1n_4K6VW)$( zqr)%{vdPw(ts{6)p@1`5c?PTon#-j|7i@c+IFJ&e?Qu5)L{ESKsC0R7dslh6-D{O{mJ{=UPe7O?A!U0LtGeE9O4d-raK|NCdh zA^@*ap`GuX%|Bvu#|P_nKYhI1FxC1I^y_<|u>AaZ@t*+Y&8WAGtuM#aKUX?$I%-vpI)Y(iZ@P#U!w_@G)KFsN88+lXXG~Zu>qBX zC+Icg5-+(=Zhmi~qGJJIDroybiYY*S{lNv$5&_MEmWa(YGN{^3)Aa9eL(!i;e0c$i N{(t=ORYa_s006_Kz}f%+ diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-data-final.log.gz deleted file mode 100644 index effa0a8c301bb993ef2c6adaddd660ba9156288a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20215 zcmV(oK=HpHiwFP!00002|IK}Amh(1}?f+hdSJ20t0cLR-2U!@gn9l!SOorc}H-pk*|W@yL#$(Kb{ z5+xd<88%ZXTcXjl1JAnCy(!v5;fk_49DGkibmQjGDyz_8yJ=^Xfa^Ni^WOjA_d_TS zV_g#|H~7Lg`?j39QQr;w(YJKIZoePTW9^2PzIN&Sx{*sy^YU@%($|BFlPx}~;Z&S_ z-O);DKSOZGS#-L#uJ)JW&=qytzKxwxJGrK;JuZ(s9(mS9-;PaL$laaJ?yYLduw}2# zw17iJx2GNIPM5X2&{v8RoykP>H8jdHym|V?LPEj!75)0nU*v~;NtL9$w%YA{9g0ik z>rx)eu6KJMwj7F1b>B9J>WH)HT<^}I81c@NBk`B8brFZQC;n6 z&yj{G3w&!UrRfi$I(KyET;0*_8_)EceZIlpT|IhvCj0h$b}t!u=k{-*cH#7r(Ou1~ z((O-l?E8U+dQLDJ@Ou6CC^3*^5Bz8vj16j&s!?cEpM=@OV<60K(Led&E@ z>v5=P*OboKyfy6yZ6bZD?+5(f4MQKW+TIUi-#|NR@awK^!Z){5FBFraPiU$1F>N0S zd!NWPPiJ*-RXvhysoNu|WVF1fH+8Q0j(vW=gP|RJqFGnW`$d-SJaK(+>AKxJY{dGl zBUO@>mneiqGKZlePFMCVt&!dv)s~N9H%+j&oGy?|D5WE(kh$-)(oL35Xbj zIQ8ZBE21~QYNPb3L}Za@rXy=%1kn5j`fxk3c-urd#LCidP2y7ixR9VdHoI)FmDwN` zuBhLSAJaVv>nI%Xg}$Q;NTam<5N6!yZ)aOI-8f{KUK~O^IlBDF&Q}lr;6UjuLbjf( z!2Yws_kG)gPZup|MC+zM`K?Tv(Z=WzR^_@e)p8f@k96Ko-iBk;L!mD`9{JDF_ZQMx zO4kki2eJ3l&33yw9m~;6s%sdZ(}mvGt~%2vJ5tl$Jh&Q-yY_y)S9&QMBrg+vA)%{rECt>Cq6aI7>hHheq<9-jl%{1q@<`37yAxkIEy6FLamW{OV zf5p+0`Qd0k3jSR|s$AF3KdVLSm==MJC##SBwI{gUI#LkceVuPzY`yBr)NNEXq&bo3 zB9YS%FQP&3G~kl(Krg02N0plb--fSVPPWMtm+M|m;&tcDV;Am#$Nz$JP$~_n4s5%> zaJSf4d*S%}=4Or!M+q4O&cx~!0sq;xU>LVwy5?IU`!d?F^iA9du$o0WU6h04nZqU9Y<(Hoz}G3JR$c4K0&L)#O#WkXNAcfx6-;M|j?KAX#bc`lPJryu`WllaG9 zArLW|F)#2Ml&HY(2(eLxM)>+>v#JkCrVPRsFwQHSZ$+eYL%aru8r&R1@U8 z2?YyQQZ>M68Q>mQzo|F@hYMZXZ7UU7sTZG>pR1q?X^Wi%%-9!F) z?CE|s?#gR`MU{N*F6rlLI?=J8^H8Gg97q+x4I1mHW))%V-z(f{xVid9g0}P^5QF^W z{eJ2RTs(v6AljzB6s0?|*Z5Nj`%j{lht%n^>#Ouh;JHAZghQ8)SoE~BceDX8z=;QP z1a5rFS!Wyio#Tx@RQ15#5X3Ddz@|TA=Q};8vug%oBbh=AQE#94l=w(A++Yf}+5!XZ z`xJWFHU~PM@!E!9>x4rqycf8}^ix+Nsi3NhT?-|dIR<@cB&+9HXzfduA>yk>WT6F1 z;&=m1cIPdyaveYoC)fGhbc7Mng5ZS%fsjMk!~hlE;G4)a8Xb3)XVcI0?R|H2$GY9o z&X8g9#x5`Ae=Q!<$EhODCmyMue-AM*c^k8{1?B+u7Kt_JGq#&+@-FO`v0qD|X^r`@ zZ(;IIHS9-apWAbOyTq5FnF8>WvimKFr+KC-aC0xFmtK!y@RMedJoK>(13j-Zr@D;6 zWb@FHy{_^!Y@{w{^MLjvWoRa?>F6haCF`OO$gq~}`67w}Ao3allsH-Lt>r+(fQvd{ z4u|TKAKtKrOUcgNsyDk!=YshKGU0zUV%&;Kfog6sDuw$nP#nviCAk#@9w860y%09;w?4Zp?NV7F){$5Rob+oGRkAV<4HmhQgl z_akYW)v;-NFJW}~T<^>pO10yBH+rBPkCHT>#P>cq4@ZPmIJHd8VK=NOGJsXs#7M-}3F=}AT3)5N5{NErF7YjJQf=>#Dr=0vN|tBTo1$T(Bj{}p ze#pdOlgk0eN8mKB^%w9P?&q5D0oliyNgTI-xg=tb#^f?zsPLH{t1thSsL7|DD`z>K zzPNU@U**VlDog01^alqm&wW-=pVbt*l?32-2mtM}7xgGkzVa=v-@b$eq#&0Urf;DlM&8LpII zDts%P=Mi!p|Jbk}GU_IxH?UT>`EagQ0lUZ!;+y3x93LB`US$}~sR{d0lwX91zuYbM z_+lRpNW;P=krX?Q!HfoblK=Yk*G-e>efjX!@q|-g%ThAG=w`lgHb=V`wUjKT13&&| zf7wh@PeK#s>6Bd^+C;)>E~(86U)fBPUZSS9?V*cK)=C(6bP+2A&d)nKic{4o=}i}- zIzvN*SjBzQIKL=a&{XN!SFATe){PArpO{c1zgYds9;!y+16fouGEMc2%UNG?jZO?! zMeFtJRW`e23;7jP&7K}sK)O#!T0_5iKvL|_?laqPtfv(F+PgkHFap*L8OC$A>Alfo zep5Yl@49M0al-aJq?4_{Kyyzu$5`NE@_=>S8lE_}*kNfbLft!w>-|9t;{i3vc0@%5G2dx1aeY@65|GhVFLgoU zyO*Ee*NDxgB&JAAfdrK)TZ`;F@xPSo2Is0j`cKD!g_ssAnU6zt9NW=Sf?fFzXY|Td z>^N18Egac=bMw2468a6bXoeSG}K+ujf8g z#03cvsj6!)c=_&0ESODCc7$_T7Yr>Zn$;>79!r<^|Imw1B9$~O4x0U(vQM+kGlZfB z4J7rrF6ZRt#zNvV7$1k7Eu?H?VL2^7ND?&~1@Yx`&SB z*7PE!%(CHQj0w-+!ng54Fj>n!8C?RMfz5?-8)})w+Ya-VJV1=bLB?rMsH-IjXG#kqr!0y_|?7y~u z@H^SWT*n5bcWuw!z2tjs2j!T~FUd!<8Wg))^Hi|gz_2<&v3H`IM>UTmZ#%##r=@cZ z%$cDdIdHe)=A>bCsKyrH*4#gFVO@u>bG;Iid#qG)T8WlJ6N_WDzXv_t(=FCavW0;w z$0R{W{%z$3Oj*MQ_$MkrXb_T&A6pi@Z#ljjl}%F#Q)K{@cUgwS4mpu{_)lH9Id^c5lC(%oLMp~zOuqTc4ny_&fkqoGI z*>m~}Jf5Odh}-)`)Vb1;b6KwNJ&b=JUBA0cC4M>W$B-hq$liZ5B!{qXd;c?J;Uc}$ z_FqN~ND%kMBrhX(ue80E)D)_VC2l`S-F&Nm2Te`4+q>|+^v^^82k(&>-Y=mgASC>y zy4oopQgoxAxlCWoz=FZWM-Rw~@^7S%#3f5xxO6{$7NnTC;otxIX|km0{5(;KM#sH> zspS818Y29WJVH-_fkf_cQGQ?|TvNs~cvwcCYX-tkk%tObk?eY@e!C);av#3^F{&p?IoXG4M@dp;vvawdN7HK?odBy=qG-GQ<2VR z15*N4*__0u_0vcUXA2-ENURbH#O=snx6hu)u~j!aZG%jTeBZLv^zj0qS8`d+(A6XK2Z}(}g`DwjU&z(WO`{;!) z4k#%R)|C9GhUgGe5Rf+sV7+)!{kcQBNr@7K3$(E+u(hN41beO)1y5yZ?)%qG=TRmX z3b}d9h!FD0-h-6B-;(|wV*{crHJ&$8;z z;n=~r_?cBcBKhJ4zRZ32ce)P$Z@3Nr2A5%0cj1z>8})|#&#(WK>c^(lN#fBQQEt<( zuQ$3)m*5J3+y#oaWd^8UumH9AnZ}*uPnvJSdJb$3*g~PP)ZjnbU6M%sugzogUSa2I zS`JIoy5bDCqqsGU(*Le0B4#hl-P!ALMR2xv-w5T0h(|BJ))^eI8T`;K<0vAd2~){F z0~Y^MX_KUd8u5E*iPzB-MI`x4{>4sSU9A;8M$=`z$fi;Jv5v8RR70FXJ3rXDg%`5rG%RNf7 zc0dXeZ@;;MeQuzJH?{CNZJ@T4Ep8>XCZQ`E&6NpV8|Yi1&}MrnqUVY|WASFTVHdIZ zO^fj_BK2y=$A(2e$dj9iy(o?8o+rgmAaXywr)dpIBN%{#z`KVZyh$P!-e^CL;cvWE z_pZDRg@GdWYKK&OC@4}5m3wvgy;Jnt#@^6UTY$3WPsD625Unib&$KIc#{-Vp4DSLVbrHE%>=&`Rk z!Q0%~$dg2B6KjO#Rlb(!wAQKl=1uYeXxTW*d zdwxO*5j-u6RmiA!Ef8P22Y?Tq+l9j`2GuPZuu%YE27z}=q!CptB5RpNvnw9Vk5(W zK6^E(#dr8yDnEIJ*D@R#3vQTeLcsMBxL8t^I1PF2G2D+W5ONs@B%Nop)+cSp`1wAy zDj8-~Qnyzr|bML|5t~8>w#7BW4g^sT4sb&Pmf{;fco78OFZXffk?1v!PWI?k>gP-WU&h zqLN|wL`q4^xxzuXRA)M$rI&dIjYJYgIyoq07k&`1^KKD+ap^63GWG+FYWX#KGWNr3 zWMndo|L`8U&vp0cdm7jGNa*CCko_Kk&&?|u;3hT8F4Fgd58Mco&jTHF$MdA~W}q|a zJxIgivq@32>YR)0niuhTyU4Vb!C1!F{h0I ztMif3HcYxx_!jD5%*t6>3iAyWd>paptL7bA|J_~4K<@s2UF+8wNLYh+xoEdJQWA{H z!2)8NwvCp`qVaQW(!;@*GWtb>uhlmf5m_v6zZ`v$20IRbkPlV!mN2}A#)q_)##6no zVH9a=jcV{j`v6O*;&iZhasQ5*_~qEF4;K7^ZXxIrRcPaf;yj|9$H(p~Yev=80r~%v zKSB2$8hm1gT}y*s5GK?%5({?F+A=-fy#9aXwlh7R5I6 z*`wG=*8|Lk=>6q*veZ5C2j9`r7G%2-PMq7`gXP$U(Ji&TSIS{ zG{g;~XmvguHSs=f7!Edz{GGl{XTIk%+AtvWErCqkFi4QDD%=wz&zs`L@2s5jA7{=@ zddS~($F%CIP?d8#=X^0BI;Np<=eD7=(I5ZS+Ixw@a{`NIuoU6b{&Vj=w{h;=env;O zFz!lsEW-Xo4=e58WXx5PV!8QF#Z9Vg1a9r5UG5~QvWzWH<~YuupRBTwDnp{e!Pog4KtgY2R4z?;z{4y$EDRX+m1AvF_yUsms_b~LgsFlcms z*Uw11vuQJK?j$rAF2~B1cBe~VFg?y5`KU@dI5fdb+R?NJ%YG(B)308jD*Vh*+1XsP zb;(+C;Y)VV^Sj5bO|sr%H^Z=gZ%Jy`6$hrS8u1l-G_H0#;ySx@@2$sWrJW4RwVTWF z1+cT~2-9AQ+UOKXnFfj!ncpkVsHoSD5+&pxwFV`>0E^Zl- zbWa{cm~o;_4(^Ed;yK1H<2KMiAF4iPooyMMF9|a~?zzhiIK#vk_3v9L1RmUWdyYlE zXheM-S;rj4GYMLHMx~#6B5$F7&RwAm64f z1L|j-Xc#gKr1BoEnn|v69q0Co^mJ_7j&BF&My>aKUeW*CYo9mk@#+Mx>L|t+`su;k zijDR%8Iwu;n6o`C_moIWBWO|_{25hR-?s|9T^&j08lRV>@j!osL?-!y5-aHfUiNLL zI^n{x&Hj|alqy)R>IY1Cz%im5N$1TpqUoyp5? z#!nP_#gk52z^aOrsz>w>joZV2 zL~rD{oTGGYTc^o+tV7@_gzJvg+)ZQZv&^q&At|h+vc5LpO^FkIS`&I4voA(o>Z;?{ zMO{S$BA@Em1%ejkg&LmyjBfQe6rxgps7Ic5*)o;0d!x5feO@)Cmcy&(*r(T2)Xjwe z5dD1TYsF;T(Yu4{7g2Eo+4w$W8hvbZyh7cQuIiVL9Zus=;>YZ2Qgzsq@CxjXW=%kI zlvbUv;46()T^6yCe7cBLd%?8|d(JmeU-LXG?TewblQv+4U$KLR(6)95`SvD{PdmrY zb~rw~Q8C|+PbO)xE6;7!e##;5tex|QlVSM|e~#_2eQXj0>o+hur*+B8HHilnsA|wB*RLF|>swgG zSC37ksYV%th8{QQx{vK$(QKhvuNhAHXoM1`nbQU8y*mre7#e~V7k91~4xShkhI>94 zNK+MHVgw+UXSG50ken%0B9XS*F^@>7AyAc%wFB`^YbOA*RRTbg$s~L`wdnhneoUY4 z_r4axcCrQQiXGh=cJKonn23`!oF!D%^eOGy{=`u9^cv216=XoXZ2MvyYhga+SBAPq z7&I?x^1VOLU|8J>zyjDL2lMwCNK#)sM+@(Rj+<`; zT{{esbY7^kn;OiP6fL$*?Od#H-U3TDtYHF zkQvH-qy%q3m*d|krpg{WoBV?ku|j{pXO{KfFTeku~Zqzici zagj=Hm8~8QHaK4pORbuHJ>n!`*i7+<^%Lp6)SAPSXPXiA^QmU0Zg=zzxzC(EMU%pu z4mHgX<@(beJ(iv?m=wYRN8~K)-Xx%)t|4}z!0GDs;dVZF_}m8MhLyiyb`4!)*s>&4 zM)f>n)8Y*iKF!GP{gX_o7t9I_NPPe?RT4cn@%S27{y{cCBOac}CB#kes*T%|T_pQ3 z&5_NcvmYEA71Oc8Ef6I8MfTEj!Y9_0`LY3_$n_k7V86u8jU&x}tXrc0F5QoSfNPoO ze9zX3a#VU5`gPIOBdH;m^RBJM@3e{eEm-DR!0Ta2u&g5DwWeP%uo-W@_@ra{!H-Rh z{zmkYTQOxo@VuhT{h1l5I&mh`vVvro=4F>ZFX3$>BO8SQlAIDIxQ#p(0v(B#1Mh3*lLzzU7hKi$mpt?m&~Q&+F!Z^ zRq+S0qvF8HaUwrBpl`FwE)q4QB z_9t;sc8{HHSx1xQ&a>e%=eSh%@*m8N1##H5X)!rRgLzCMmsQJ(r3bHhvb}7)~71PAY_XV{k$PVk8_My8++MOmJ+jNXj!XW0wTM>!?fM zSaZJ%SAQU3Yr0ymAjvLe11`TRG2|pP`o+O<9C0}1rg~S@_%+1^x0O|g=v9Ct^|3o) z-jMGVyylgYR1IjGBjdfQY5<|}q}%j@O$zKJrcuc(qqlpw6jA@RssXzfP)#hydunsc zVtdN@n8CW1&SZy`a(Bcjk&~IFRPI}(wfjf$@0GL>RjD{xF$$sS!ssWUUG>}tNfik8 z&Cp*?F`Kq?&C(gL_zB}qH#rg8WTB(jRwa}qoEn1=7*(XNaH%Bie_5RUc}KeeY-BZR z`?ejjpN2{(tu6=Xth%f?#UoMe!H3N19%zT^ynf zCsAkw0s5ECA@(5wr;wB2Cac7rT(dv5D@4Uy7DZPrny>U89M^Y>4^K+1xo+ju)}~ij z?!gc6D0?SqX9jSEzWul_^x6DI#GGLLEditE{QdU4!>A%VBMVp4WZ^7o&6;?hqE3gJ zthn}YAX((uYZB`eis{?i$q0Nn6H8mLa2mpl(8KM!ZX=2iV=+8BaiILym1gSk1@F}(H zIsY!?C(eIO?^*8>_fg(UH=TOiYpo`}3(g{c4jff-2Uthr(7iR^=BWDS!U znSim$evtlOmzg($|2z0vEd1ENMi#5;ujv0864i^kK-#ck z9Mu1w`ev7IEk`i$cg?Z9F(K4oJ9*dXJB~JOw0|8kqy{u0_IPtz40UqBzlg!!YDP(q zkjP{5?Sq&yf&_7*=UrQOnF;WL_#DmXIJPtgZOHZ-`qu4Dv*mA?n}i-o*!yOfRox9G zWbh3Vv+M~^hHHtnB%kX_o3>%hZQRtZ?Z)ebE_o&F<5{Jrezy-Iy8x4o>R@&)-Qn1U z{8o$Xd(XCtAUPoXuHKd2AN{A*Lu5Blo>*y8S;&&TX$7F-s#&tMK*Sr)b^<9c0AgBT zj=-ogl9wXAnxD zx{_68p%X*!#=pcTg^Q1u;H(j9mQ`t1Rr=J3nUB4+*+{^XYm#_%mIVTXXIdKj-~jxwxe%E0j**0SrQ`oC$hibv*6b zeoDg1x4%=3IjyO$J{diN16}MA7b{Zb#qjIv$SOy&pg$5xr!T#ZY_wjYa>pLhNwK4u zs1Xz zQV^II_;JbTIN}}$WcWFw5WXQdv;*A84+PNKyh&4-_}N_Nbo}Gj7d~3{yv8S#e2&l$ zV`|20f>5G1!BJdwPK9vYt!s5lVzA(jR}7gYDaYBj))Ku$qS7wgJUB-me)IU6I7Nc{n)|7$ zjuAs7hpHE30zB{q1>|$*gR_?Jx+5yL2F81qyP!}a5ERkI{)%vllaQ##%(xJMmFyNR zl-24a_>14F&agzn3JTWbfU^qU{r+j&nc0I$z%l<&3KEc4wDQHh;hjNH7R5^|j3ugh z8P*Y1H>(EC{!EkUOH{KqH#8pSA84Y&Ro0slzoouPYpJx#1TKjfwx`?Lb5y!2&7xel zdzuW<_>T;`{~UH_{(LeFAxnzD`(}GDzhAm6V||fRT&q=~t~9>XGISx=w;Jlm7Mcmc z2i@{$O=VUoq&n%<6`#~$Q-n?7ne}ROOo3J8aP!!56TQ+iZpf1yTC!&Cd;+H@*ICAv z8u5b&(Z#k9_l*&R=334k!Mfq}b!P7xD%u4$MqynjzRg%&Lr2vn#h3fqnZ9OxiEY1Z z_!C(w?GaUHQ4@L@EtC=@UU_%>m)!IoL;_rxWj4&FCQMN}`dEPxS0_vJ9<(hrcR*%( ztio#m!<0ErPsx76IX;1`Z0sKYgT?l#-Dg}93K5~>N;kv^lai6bfab;|v0$4^j8DKe zzYXxsrjyiITKmcmc+6u-`U}n}p~=K#Xuk2*Ig9NUHgAv%sb9#%)}N5)Mn`+gwuKS* znkC1@$gxWGQqovTJ`cIcn#WGbC&{cYKje=d=6^4{U0GvXn8z37wKhJ_Vi`kDd&W+- zA`ervonmiXuN$zMS!i2JRAvcL`^@%Aua8BV(@WSX1#LI?S!5=Sts7@joN;4`cs=%}Ww#dw*`ZCtj~z!tXzu0j{!c8s>*Y z*blVcIcdZ-ZMeFH`3W0q6Z0-ik3K~*O4!}&XV6t#Rs;}qlrXWO+q{6TG52tCoxfUY zK08HHUv3zv;MEB)ik%>27Bljw*JyTHcFPD5QsnUa~$+&qCR^Dx&-xPNB7x*_tLP`rn& zb`rC_yk;)b1ItJ8Z;=rXDZfxQFg*d$wA(1{Px+GS|B*g7?C_ z;Z62c83Em*Q%YuxlC;emo_uC=v$?us@9!IGZ`%&F(!xB3NQm zeYMPcbyZFn?t4~r>UQ2T8MLJle9LwRpAab}li7A`+FoFq^>ckMImQmf?H8>2YdFzK z5r;XqQ|*3k2F`AAC;y*eHTsNTIC;**MtSUvjIU+Go8_z#BH~c5pkmsxK3gM-?B0At z)Lf1UTQM-0fdmT0daKKB{D*(scz4U-Va0n$|9RR`mD4}7Qx^N_>5}h4`x$7Z_r`$8xIQ!g%}MEou|Kd!`W9Tr;@u{GQLBQhmXE%{ z#-a#?21loFD5(NsyWBZ9hh|(Yo5H+K23kq>9+^5wwr&qRE7+P@mu;z;#*1Wy#lH5g zxd|lfLpz*ruC_Cw*fjZRXI?l;$hXYws za?fz7jO&t%jD=K5)hphlkP-*NpD~16HbQ5E6E4YeZzSyyyC)^(z}0R%-N0-sOApGj zKmPK?Z#Q|;%P1uGG`U$^eT}=tH+I7XE6e-AUEQ7Bu>8v_RYJm z+b%7HGBw$7-k=aZ>eH}@&au^k3F#MZ`O-r>V#g4C$~E=}nGVgwJdk_u`YoG}x;Pun zEV)DN{%C7IY<*`K`|3=m{wBZuYs0XYvUgHSsx@}3u{f=H+Ere)bebN8!eMW6;DEz}itxBfX<;$J`QD<@_JDQgb)D+7x<+Dn zP++)dE!K=tdaU6Q?;V{(f7fw$(WvkSS7GZuE8>%ucjC29%*IL;IKv-!QyX*!8;d&w znaAXcHu?%J*k$Xzc+>0-A%?rYX;O5uZ|m{g2yinqB6xZmX|W~S$UKU%_Oea`o~)@| z%GEr$PSH#U?hTIpFQv;<)=4mo4V>LmGaE(T)=mSg1bI_b*_5H9d9(TE-4W{zcfBKA zI>0;C8(;iz7slw1378O2S?405ump8UJG^l9z3queL`>vB#O{;iw@!l8A3g-xiS&%D z$lkHj+F3uV71Xt7@)g2>_AlLy70_UIX#7}$-R&RXv=dVYs9WZBOXVga6}RUQ6%j4& zB+ErR^DZ=xzS;sV2ca9NV54Rou&ZZHB5Z>!UWuT7nx{*R19sxRTzUzF+ z)Y+v?Y?E)t`|!ArZKgeD#0GR=Y516>wF7_n$`hTitEy(A{tAkgMj#a18WIM%HsWdg z5z?M76F}8^LG5ru7IIe=PZMfkto49LC<~#qaWu*_6>oEy&*v)wCsJH5+-#yyXD}O^ z!e1W4Z>`?6hMWB2|K2M%Ju+zuzK+D?*}Zz-%*U-))=u!0IG$o@K*CDH>xc?sAF|3? z>r;s_LhZuIhrCh?bzBj=5q=!Oyo-gsQ$Y&6lB?P2%#~|Sja5)(C3HZiOUa1{03ZYuede0iMT0f{^dYqMP7Ya_Z7f-PotWa0}LI zs4`-_rfpF){SjRlv??tLTkJg80~(_OeKZ?=MLSMFOJe&sPESMHgg9=cgGSa3zq->R zwRQIm)7$c6)(O>)nI7?(uT+i)tt+$6B<(-VVxJ}N>*$$oeJvqgNiVIJHhGJ1NbR~G zAUjnbRDG-l^Kh_Mr_ zln13pIq7OeC)dT_+?{v7e;HYm>D&i7m@cf$>19;!VG0tU%$HG31NW>vX_MX`K55b_ zpj*(Ix!%O6ONaLt)0PHfdduKVidStT`8(TAd{ncNGO|A3$uFRVT95Eu7OiOylyouo zHARVZ!qIT<$EI0lEnbfv3%q=fEP%TBqus5UGq;f1`-DyNze#}}2kzua!*t@BuiLiU zFmHp;lLgeeADY>O*%xKY?q9Cdt9Iim*U}v|^H~k8SDfqEy`-DbN;Mz=-RLHPtOd^$z<-KBj5t~8=8de8t8qdm${N=DxPL47> z_As7Y$VJ9#$M~JwuJC;}AlZytrMK0T!l*`796>tvK3F@K{jh~x-;W)8MbS8-;B2{T zuh2cl!ZHft#ik)!4ty5pmUgOb$cjKo{%skMu+R16JleN$VM^qdQWL9|dHk!m(8r-Y z9M)4iCT&6^She}L|G2@TTQo8wP+)WTKXw<>M}!*CucRSr@WY0yeRlm@(x3=EulTiy zcWNN%7?O+bhK-a|0C3+W)3Zpk3~mbp<608ydopN>dw((4zc|J>j2~ayzNY=)o?yd= zCq@%{8r`PPuNz)-;-prTQBbR-Z_;+ok`m%!pHRxf$3%`;Pi4BliFn~D5R(b{$Je^4 zEBQ6_V@^UxNxjTHf5Uh#|8RTUW4>6z`@2nA%N->HaqXN)n2icV+H*+X97_>&mlEDD zeG53Boeybsmcs1s1+!#2id2K{Jq5|rWT%v96FH$FaR-2NAZlF=^-zH6kWp6pMUB6im zPj6f_A`_n>1lWD?`63dD48J;e6~AjAzZR)!l;lb1-C%ANc6G|q$w%-u{ReH0*vUMF z%2vBv9~@HBU)6sOqA-^cp+H)X<)I|%U+=95tZ`j9wQB~K2>!%HN4X4YlAkL!@6}>n zpS*uG#LKJPmALL7adHO9bYM_V}I z)U?U0L^vO`NtK9>*w_*8j$UHE?NlFsp1piV;=0Fzu&4av->N2??8B7l#V(G@oXpp@ zQIJ1bj`_L7a$vJzuuQaI%H@L+@~?;@d+xgK!^&e6fxfz`U3MXMgqlr)CgeYV|J{}B zENYu6mhVFB@zjmG8u{7W8kN*zVi6})#5+^?7Dgcq9w%J)Nku*ar^ib%0+N1;Q`=X6 zwTr1v6$Iy4N!}wCIPwD!ur9ViTzDuhslI`^ew1Kkh^$%AK?DIQ}fEaw(^kh9@QD!F` zNUIMZ(ygKx36|-8Ne%5vn$r3TH(lzevQib~*~FgDH12tn#7+6wT&cBQ-TDEf|7y5*2XGoKiTDD{w zIjI{%l@^g6%?0eB^x;6Hm6^#x4D+YfsoMiyW1%;40U3M!kS<)o(Yvy&pS2ClLo}Ff zE<&On!|8QLS!rY?wtCMsa93A9Ery=b67 zQ_vkOPL#$(PuCV+ZVv<+w&XZ8x0vM^911$oD{?JLrl7`24J0k^4}fH?9k?SU@Dq9- ztMH`zMYOt*1mXnmLjk8H(v+3pkt<3$7%qwgy6bTwZPHeEB}G!@5v%5aajX}uf(m#X z7W+vH{0ds?k|*%wau~RuIC0p$!xAs)zKnb)pgy_75N(QJAqc@^$dDhDR1a#()*0QL zI03Mj8dZwkqwiDwH4i&+#imb^)XN09d4$>t*H6$49dafBHeIKKq&FN~bTu*PezSp*u1&_&BY9BwYHS}c%laD?|U zX5IUDLAk(V3}&eAag`4jI8|1fPPPn2Tv0KkK6ov!!PS*|38N9FhUR>fi(L=jXSRvM zkR8tns+Qi-gF|y6VX!&V>B^pwAk<^T5u_Ea^y`fkk!6CNo1lo}T61Rl%cg<#c`JJ* zeH;67$6QsjSr%IRf}GfzVpzh0=MQrXEPX!MGjOGiMUn|LQagcTCJ99#&#5sj=zSsyI*D9SagBx3Q?N? zZ?sy(R!vNslkRJ3t5`KV)O9lTNRNwP%i^SAw`I3M>9!*a)4*^30+m-AM%$)rpy$;aL7kUvgm40Ls? z2nEIy%#Q=lL;mVJ* zk_4c-;FCgwR|R7wh3?pB(ATn}HMLL7ld-YBfvYtWy}WWA7;8P+cy@c_9SiXO{xtT@ zmh-cgm~!)CkSsRPHT;MOViSE5{EOR+rWwskI0hER9sZmefT!wsN=;X&>7~NTz!*c+ zP->HTuSLjdNNr&;`CNB1TO;AxQ^*MwHEM=@O*#=bBslx*q($j5=ct0bShSRPbc3XN z?|mx!Ff& zcoMh3*2xmI0kAGK14&NOu!xBj>OBgWJ`l2&H&nW#zMrH9( z?sg@s-0Nb5!~JI-^i(nWj@9hjmbGNbx{8){x&kiU(AUW>sH0VDM6bk+OK1h1H2DiK zPk^LQr_>dtwKl)*B;Gi*40(MwNDxwb7(BpIOut)8j{W!IPY{a^Ks4`&EDgeGG$2)m z7=c-mV|q<3Hq)$1tE1iMe@aKZF3&{e?&PEcWSn+$w^);l-qv^H^&v!^GgSsEmX}KIm53DM@P~%Mq?J-H{`^&PJ}3)6axoD0SEGNq(}YFe5>+ z_$P{hKW~IgnNZF{QNK4ad=3TK`{VTI=|$*|H7}<2FN@PJubQ}MnX_u^kYirJ}WHqN!GuMQ9vVI+-n5wC&Mw%9= zeH$8pC9N_3IFGi!j#+WO6R~IM*``%AYEA}4PiTbQ5)ULBi31b1j`=JvxkNg%K_;Q7V&1Ts{eMZO*yW;Mb6*zpl=rcq4~mS zZ_AOvNCke?=5Q}yHl6r%gyyV5u#XsH2~23NV8(u9(~0*#%qL#T^e5u@Sh!8qUT309 zM3lE^-Ob{JxM_F8Zo8-;T9q$O{%pTV+2GtuOrA!1G%x?_-)DlDw%`E|5Y8dQ}RF<$MF+;?W1VYu9 z%2$K5mp9E`U~k3hh)t~8u@Z0|**qYdBC9cMCJ{&UX{#&2QDZ9W4h1^NrOK+26OB^tim5%l%T za^rxkh(X$(MgvYfwHy=DpO?{Cm`YCays+UPMfmW(Lp;(CqByV?8dm$g(^@L2p8^~% zKOdwqh=69d;cODT@!;WznYym4Z+ey5eVN*#@(U28Li|SIa8cq6!Og0w{HCsU>@UhO zm$DRILE_Ay-;wf^p!Uc-`W(c7xsXtcjg=TDS1K=Zka@c%^WsI_GzYex$@k}Le*4(0 zQ`#18;ruL3qHNIw5x&112h^&-rPxRFPw>TC+kMb*R5uJf|weWDNdF)ZqM(Zx(50tzyu@k#u$;p`Vc)%N*2K*y@d_U)XH) zTT;T5#5oN&TN03D8>Z2rTejd`rM8jDO<#8``YC@OU3BFN65mAc?>kX-fO- z*FQKe&i;W-OjsqSU^LUcjvIN!IFa@$ymRR!RB__W=ESn|vEgWO^^J_fHx6edpFA!g z&p5uYD`_CNtgp#JLP$N-4iQmHQmm{WAgQlU2)>FQ=joC>{mZ|KLa3~nV5qBd!fbgN zy&9l#ns9lB+RthihlRsW5Ea+Si|>gLWPTyf0p5(IG($NRE+tulXDF+DDYRBf{7}1= zv(+t)bRC(tgVzeGjAW8b7;sj$F;Q4WAVQbKv>#%!j-+xjbG6Z;pUo*oTDO4ZrZki0 z=y%Ivf8mNXJ#&aoZt46!QMkX<+1`7OSx5rTWCTdgg5QMgjMA$x7l2h zsJH3)E#uYSE_z?L@Bqmp;57RB!l`T@3;O+iV~oNBw8aknb`cu7O%AZ@C4+ye(@ToT zk!y`0Ty!#T-9&|Vv4N3!{1G(afO9W66HC7pW~BsvOY?@0C4p4Z^HOM4N%ml;v^BWd zNIGUa@y$;<&udn>Wv}Mj;A@J!&7qibj1<^JqVk+YxRKt3rQr{?CVsIhpt zDhjV(IdDvv(Zy5TI_(eX$5gvLdmSaeaA?oiyZVx6)+ZO>|7bMYl4;ZjU&u3X@U;52 z#tX5I47mngdT?j&kgWmhD9c57I$rXD+L}2>oM!=jH{$Z;PPrFlOgL-%>R83xBb(Mo~^F5&BR(q*3WV-EFqHt=@?Ym*tp z^`K)oeVcFd-H_r`A%+;Pea-6A^q&JUDwl%>*-0gL%u&}D5{g2Ofi10y6rnLJ(4C(S zD=I2B*?M>47lNf0vJHKOe4yNGJ_P}9C^p@>@y!WU^J5Q70 zX#1+3)k*~dOwwEtqW|aTZ@>QfYm8VEM?%LERO6n;x4sIe2Cf%@8~NBoi42qqDoQx@ zRr8j#qU1ht>XnvYoa+L|Fx8gqAWXY1aG0XSI0$Ewz0eP;!<1B z#b*4^vPF-(!G}%j?SA`Lyzj*S{%C7I6u%X}{wulg>n{oNq2B(HJgGM>wC}_Hz;R>`Xs!tUV*Kj)6HeZ$?mGv^Std9A(E_qN%WD1rS~B2ULl zk&Pu;#$Qy&rsdnkpqn9sw+pqI3Xdm%aWMng`m>v2HxzhJjjP@Sew$S3f$ zDjH)Ce;t@b5A9U^`CLA5-#FNYhN=e>48=q#Il0tMIx4>NS)8jdhGnBn$y(zsi8{o= z7Y){3=xR0s2zZy*_1%vxH(3Jxa7ss~cG7)Zz-3CjxK*_}l z5)ug01_eEv>yK>2nS)^x6HvT?426ervn507%#v16_%kz$^0?xJr156aMfqg{l@1F)#)f==uYYa-B?vXiLr#xUH6aP8u+s9~UOgFBhEqHW z`%2SRD!oYrD&cQNhs&r}*a2}J7D@Ql{v^(o(}cKcVaw3@eaJ$BEeA!;76kG`jJR`* ztGbOuLl{Ef4nud7H`Fp%jFxGmeJJ}`@7*d(M~M904Sn^wX(+mwlQOhyB*D!~7iFD$ zH;gk0>3hg99Z4$m+T;p4v|cbL&X)yz%O+#{EdSgv(DPkhQSpi9dxD20SSFHt%JAh8 zo`4Jav>-G{J!q~%#*RC@F@ak=*Gwd)f~^S*F(fW#dHCGA9gXMN_{v0j*SBvz&38eF zR;;NhDMsC{^~LJkG7-N88?&~<;M4oIyFdUlPUz;87?kHs=vM1HbA4SCt$7oO=tvWp zxPf98gPn8x8h_Uzv~|6TB9VM9MVFlTZrmOhnHiFH3k{n$DfxyGx$noaabW3w&C!m0Ph1(8LKuUW zSjDvu;smJ5Ts6KEZ?vvJ2JIjLxomW#vyaIu1zlBul*d3mnp7osJxVf*z$y$!aot$S zGbo29s_lHz3(JKR-#-_of?#CImfgsjOe9#%<4WBt_-E@S@wh?4b_T%J!iE?pcr~{i zAO}p2f@KkpAI}rAKO0#E67Dg6#Qb0oSot>HiB*8G$rVVkYtGd$XwY4ug1My7lx%JU z+LQ@Ut(GsCyGWi+=G{@@WH+KF2ND*PW(*KT2z@tk08iOQOJBG!?~&_9&jDIT2l@3IDfg);Zs zX=j0f(ml~FP^~m}!j|kE#_v(+SizPrCiELt)I$b#9DFU9A&ilvKsMYQO*_t&*LXgH zg~s^RViLUM4zq+Plf2Z#(VXDlL|mG3GEA?H$C*K=OKxt=t>(h6fw_eideN){`_anP}xr*tT)ZV7`S$*z+7nH!>|!9 z-IurMeo0eR?lKhXA2zl+!y8U=n z{UGhvCghXV{ay_WU4^vJRb)&MHI%nB`I~oSY_|05Y)spM=*0XtAzIAis;N$D#SIpVfogo{TjKo@F^*j#kTM9-!PU7gL9rV-ly$OCZNBz&!o32ei~6zyDFyCbMIgU z8x40jri**Yrv2$C=1HF*9O@hmoI*KpC6*OT6BxX30CW znYku4M9k$Naboq!jg-Sf=47Tv;I+9Vwkgu-^GOUx^?l~h77W`s>8>pt!$2Wfq*@`> z`lFsHSYvXIK;$P3^j^?yX|H!+$c?|G>r>ijZ#Py9RFPoHPqOF%5K~?qjTN)RvP4l2 zNB8#6b=58GI$8n~@@1kZPZqHFe6+E6ig+qhz&ovn-{}wk{mXB^{ri7@;q=Aw|NZjo zFJjQ+zc^(^{!{BimT$5lsGG<}v^6!6ifi$N$3O&V;#Z(4nn`eZYy z{SS|`&TIfbeenE<4j%ND@&JkA-VX?OIm9G>^e^b{K`iws7rX!;pdMkl=D8dGU!(;+x6=D zvaQ?AqA2JMsGgCEX#tnXwgWbt$I;Te#BP_6*t4K}KO(KSB-}iEd5l?_!9Ss9341Nn zui#?#JjhNHMop7tMt1{C*jpXr9Qi&(I)x)z&U0LEK<6wf>_B~PbocfU UI^mVsY`G|20Y^Tl9MA#)0JYDy^#A|> diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-serving.log.gz deleted file mode 100644 index 597452b687dcd774cc328713219d9fe51f827ee7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 531 zcmV+u0_^=CiwFP!00002|E*L@kJ}&+zUNnP+&%b_WLHtUr&Xn%`#4q=OU&3t!N4*@ zqV(5yu-#1}ZK|m11%csv%m-s3#3w%lV~y)X;~hnBWw6SD*w^KX|VVro}l#exzXz?vMm+fURR$UNtS3wurN_<$ChA$YZW< z3-UFGmB~a|mcs9*y)z-??9{;UtlpYVMxR)Xq6sLe=T+CgdFYA4HKNJ!!B8(dMukjy z?mwuAfTumHwqWsZY?A!sUuumzpr)4^EZ-@n?ZG(h50lyK^@7I?!AAlun*q5%MKGtw zH6}+B>2*XKfls*R$kjx+MT2~hTx0Nt*>4417>mkv@Q$&ok+jL$lZxC~`%kz($zFi5 zS2^6lYmnNrJLgF@{b+IYt4}xbnv|p5Mdt+8?B|nGHI$UKM}7-70+#bO42Eo^$^b@Lc#rPrRCc V5Zw9ea>%W=;+_oF4;~> z@WD94Theep{lAA0{a`G?*U|Zq%-;MY{S6Yv4W%nyf^l%&h^poj8a|KCO?^&aE)DZY zUAvN&4rFT)`K4FZTyV;57Bg9?X+3W88GgeRy2HM05$i$sUEdGO#Uh@Cu4k#mbVBlB zCnpCb`BkhtB#kDQOgcWX%rqH0+Vn0zh&)+9 zX41D%!GJk~Q)M>J$GO}S@_4RuMu~C_nf^B4&LQS>>PoZ?Fa91+^Tv3ZLgA|Fx>~h^ zTHR3JUc^(Mq~_tKRkR+pRM`%rr$v{8x7I&o(~LUXG~}twI&XXSW{V7KA_q@&gOA`3 j6yc@sTi>H9>-x&6^pio$dW2j z$+BNR%J!NB*drKr5hzpNgQR33#5+F(V~y)X;~hnBWw6SD*e|QwYAJ5&;iwg<`swr2 zhmQ&3vD~U=2Ts@3!N(m%0IUg+O85p%BI8{NE2uBn7KJ!h%7p((=0r7Wi1)@B>_Lm? zeE3;}=v!k!d`Zp+GK2XJ`l}!nJD?j}2;;>4rh0RaBU*e)&Si%gj5Q0$xeg0)ltHPS zma{jB5chyp#8UyNu|(p0If>L7wyEPaT+LTLG|g&rzgj)4??BzPYEcw%Hg>z7ycK!P z)onq($B?x*?%*qZ%pPpXEJvi8V#0S^%IFLuw6xkr3AHAWifZ~%pAj`{r)a4dYHN8Zgp V2!8qNYPrFp_yJ2w>fY7`}pp|?l13$ zbEl$u|Ih#V=YRg!r2YBrUbPR}cJJRkIsXv7*5Q5BA->hmx}BnN_O1S;-?v`Dr|;(# ze|xhlLAB0C@A_u!mDSs!m8)Gr!UZdVnU^9MyZHp)GSXU^KWn7cv=7R=TAFY{|)|k7ozj( zs2gpM#_I6pOL)++JvH6_%NNzQlUMEe%NP2={6QmUYrLCm7rhxn6HoAu7B&|b>kkjZ=NXrlK(Vm1vUK?7 z;8vm=n}_DXz*Q;uAALtRw+#@kRUW7_hXb5pf97LsAk?mtZ!f4w_2Wr3 zADRyzYh4h0tw&%3)kp1tg3P04ZfL9;bf|EE0?EAn&UoF%{=9zeqYDj)sUE01;9)v# zXDS6jtd*~~eCGfG)ppA5r8#c{Pr}I;6jwgTjSD55c8xz43?d z-1vj?psgBEW=Eixz4r0E^&B97Vfp9A*w$+mbfdi9y7e#(By_i}0xJJ;fWvz|C&npK2@w0RfvEOiZ;fu8dPVTF)Z<4`rMW4N zQ)~te7Fk<$Bac9j9a%3(Ezul|_N-h<6=~Q2SWcmV=cWZZ={2B%L_yTsSl&I!P}MMM z@(xI8!X3aj2lb!@XH_?20l1w&UeQ4XLLf-W+s$ZhKwj+^9w)ApN947Np;zIg?+7=r*mWMBz|c}nKp>(?*P{g0U;}_$ zrC?Ko(Wa`9ye16=rYpuF9`u=)8n80~Z22=fPN!*5Y)T^~-T79@YDDZ2hk&dA zF-JEMDgy>XFbOyV&XGs)>A{mFoi%w&AEQOhpH{u%pmnDu<^|u4bjs+W>Q{hCDnfyS z*k{!MM1!%+zWM6*Vg0JvDiOP7b1eDRzA=g)z~*xUNJu?tjxeTp`1;i`8j4Rdm|oUJ zMFW50-GH%8CMP8%%zlcx2DGFyBltMv)3=&Dc#h6iJXY4tDEQFgf-^PrHQXlo87ryx z$B+i?j~ZPm+o2!R^06v`V)KKU5#$e`<Gj|+Fh*$0pR#>DMRz!?hj!F4s}=H# z4xnO_@e5d6zjAM2dw4nY2c}G-ilpxl8u;@gr~~vnb=*LBgYplv1ZaDQS{kX1lyn3o z7YZ^|aU&!V&-g>MbI2eWfXviU*9`^)Q12<4jSfH1^bsDR&k;owodozdon#FZECKQ%yiePA}KqKj~(WnB7;^YRl3=WcRqq>UaSORUKHjVN|Rk zHW~mNNih_^9!3{w^GD3KV9pk8=<}?Md=}AL0Y=z%FzCmIWP5^wNq`CoM~`omZz7&e zVWI2ilbQnMjMs+8Qiw4viBNP9X8`O#Pew>_&8UBsQ{J!u*IZRVU$DM%NG1~@q8O7_QIE1rL zm?I{&w℞sg<7e#0X1{XNg4fdIJrErkCmcl(D4r=Mg!^>mwLBBzzR6miXuO5R@Ju z0Wdxkxl##Gpe%UO=6H%=oi1>C0i1sR@5|!~m`_YU&e6T}%N@cAWM-RmnA20%G7v1E0n1U=L_Z%Uu$Xmd}2%gCv9b zx$QvlCcgs=a1C?{xIQ9GtDoSQ(}?-g3g<+}TZoJW3h<{paBz!KZ0TE0K`qFK-##Wv z;)Ag!>XL8(@FP(!;1GkgpkxFA<^Em`zMZ^BwajbGWl9=MTI5TVF&qkRPR^S@9l0O| z&Gqa(&QKVzQ$KD-g=iSJn;z8>%)P;&!W6-*@Emn<4P6mH{M!im@G$71@If}%v zr0@04OcOCBoVkI{93KS@I(eM)2%H4A+4NCk7Xt}tkY^$|tDappEKWdd3S7+md4z+33< z2G%=FI!zY8p?V+FOYKCljhfJtc%w`KegYU3>>tPqUNV>_nOnQ*NE9jpn^c&vQ>`{a_Z5dmFwoufwpzd7?kt@UJ8Zj;u2S0E? zE*sUkcdAPU>K^4Nr9OLccr51dg#G%)RxmxSmq$UYQfdsC!#Yh?WcA@IU9;slF@;q!A z>W&lh*d5Yg)M;H=?{}(?ECt~3{~nE+3*$KOuoih@{-I96aEQ;r_hhUzNFpI|UyCR` zoH5F<)PcBx%Q?M?*kePt@c$_^uSV6vHBthE(=6S}^XA5p+zz&^GXH8dYBmuWFX1F9 zk3f5mEZKZfXb@^U7y7OZTO@KIF`X0(G92Oyja^OyF4vqF3rk^x5It}dTDC7pUMvTS zt8OKPZ<7KQ(n~yvB2C1&ygCchpp2HTk5;0kIZ-7dW)f+y_#? z^^dFB?&6LyMJYb|g(X&FZT6AbA;tgzeOY7SMAb!QTad_O(kpAh(3VZt=a395#fIc;I}kR=o4Jaa#YGWIl{_o<3oJLV6EH_O zCpFj%RAVEF75)rV{V?dxz(w}w8ZfNJ0vG)p$q3cYK*@%>p3*5h(7D(%7dAly^Ae$z z$qxzIVMdC+TuW^SK)@;AJM|Ir$`sTF94#vDyIlKta#$qME0FUCiLIC*aOFisl zTdlB+&D9E>?xRW)E>#$#W(9W-#4zXob3Bm(iQ^(e_y>r*$YriR3ZpE{JAKe+w7$SN zoW@kS%$Z-RX-15~BWp>KHryHt`zctm?ZRcWh)VK$2X_r#+OF&AGBVmq3+` z5iC~jx06C_u^TDn?B{wZX&{j&@J$me)}d-$g8c@F9y6N4c4`l%^*)fV&hs5G_P|9m z=u&n@9G8`p`n^cE2%OKK{wP9Idwo)mW>x9JcR&eg(kKy(HglaOyLfm&PH--4ti3uJ zNMlgf6WWOqswu7QTxre=ZYeMc(}=~Z*pbRdY_L*V{7TMeq|$SZ2Zt+Ic_#LW=aUho zWB835;VwOhP#Z`pHlb;Jt$xz zxMfcH4?##uAV|AF?iJWrI`y4C6za*9&`;K}J@C+gb=-lbER2C$n_Aatu7bTA?9?8X zua+2OX${6ws81fv6M%@VTi72v)9so=U-AYEUgnTAz=oz(;P-6oOEQ7w_uWpJTDXu& zxzY^#@>u8I52M9XTtF>ma3ZT|1j9qCq1qkm=eD0ZGGc65ZwfNb-jRk*F^jT% zq#~%4@lBmqOG^}PFP|j{g=pG%Zy6@l=q$q$aQ^U7w`&iiDO+$Zsw!v81sxx7V5c!| z4GpG(O12O{HozGS8txbelx#Rwbd9bz?RhJZpfPVo^DPo?XkoLSlGD=kv6Q%Cx=zNac9{sw&7_@4#BGV|SVF;)+PQ6DjD3bS3R; zE5W7GU&Mg4p=}H-a>9uNT^v>bFVQ?=3M4X-O6H^ssdH?RiI(HH5+*Op3}=5T_Xj`G zz^fzpLO%70kwQohRJgaxE-`HHn`6u$-5q9TnS62pY;oS8*amLZs~&SgoFJ8N7T zf5;Smw2Dd+`MQAS{CHBf?vYF?r^|DZ^5H`L)6Eg!d^nC_1aceWlCe8K1Ug?MI>h8% zWV)bD)B+>AjN@i=KRG$T6#*TovT~w+82OH+kKC^EZPT7809z3tM=NHCIM~ZFn67|a z2O?`ESv8ToCPOIIuGeBXd2W`2l;BIUiW1;6@5zr>*pTQ;o3`MDdEk@emxn0`J+d9Z z?I)CW5%-jhPY%3Q>Hf$D1{YXoBQrmlML~+|V|nq0q~41}g#`1@I$l(+=fuhKOv&Q- zv$m9}3(ZOOhsKR5R9n9z>biAtUBCA$k9BF>rRfIZZdC`1Mm9*iv!#|3Ad$PGEM4g3 zyq0)ccb-@6CnIZa}VK5wUN?9|?240R}Th>TeWL;*1q-YiMl7<;&Br$iurdn2VjM1#e zS@e})QrZ#oE;7Q0j%Y&Fb4?Z_xMbX=*F5l)c0&O4r-QZJo4mb2rt4lipn%yzi5vCt<^6RC^S84jw2kfQQKHx zYVMLhF3Nb_0x1S{j*ld0GSDQHz;NF5ILdsRMaJAM=<#E7B;#IYAIDD@c&l)_bRLc# zcbaG1AQs!TtJ82(WY<}Ube*C5qY&j}rR$-~7iU6_h0r5u2+|S_@Y7usjZ~?CCV)&C zR4CwA1V?3}SO}g@pMdIxgV%owikXX-YEJGN#H80W+!c00Wgf+cB@1ao z^&n%>tPB9fg7PO5(1dxe2mn;}3joH}1CUykeg!$EVPe8@nLd&)3i2g#P7}MlgaF&hP3c2fQDa2ESFHxcK#&*NlGyjTK%*b z=8xT~>*|#hPD4ASb>kW5AZdk5WykB)kV7}O7`qW2T(qe;Av_PpO2Z~L0w@kLp!H8& z#>F?{dhcB@5#CEhyxc^YUgj~`AnhRvN1S)l*z?Q5T!)*2^hob}hePbrxYlxLVH(`b?^~%GM@sX8WlXSyj;Z|LbhQGhbBHQ66!C>( z2{nc+tHI%5a0=K;-a^DgCA>-S8EKO02Ihbm56(n?0;?%Iy(n^f0h9((73vDZ&8df( zzPg&SAr%#v;0lwk1oUdr3>jZ`MM)qZV)f&pvmv9-@}M=E@g!rrDC{Xasf5s8-q#z$ zMdL!s>n?8>eqC5RJW=fn*YvT*S`rukPOB`>7Uc#oa?9kED&V(0h%~1RQ5~ekPkaO+ zbT-oH8n>K*aGfzv%_B0NjYC5`v~6i;*-s=f%^DDqbQMxxU3*>5n&SgtHy2^Tc8OuB zo^(_9-%U(2%w_og9)R5Z!aY;?R^Hz5i!un@>ZXl6*D!V6(dJX3YGr2Q!rh$5Rx*T> z3U7kEO?!Q#I%0dS4BtFQ@N(}+MwzjtraxE6C1lfh53flfC8d!1Bi6R%%FBEiIK97@ z#>WT?USLajXBku8I&CV>{B=i_oo5SAhRU$S>)6;|UX=J+S4JwLv59EV^9XRV0_jEU z$NA-pT#Ct0YyYU$v&zj4rOTMBW#*>P#5cUxkhjIvtd%#C#uBBH46*Q}vL#LKN>i~K zv1hNheUB>=6EG;+_iJmrmsrlc{8^d^*ICU@nn%-&2OEvN zwq3#t`~m_pe0dJlfeT?)N%$M+;Q>uWl!c4Ix#FMtK?>v~A2dI6tU+p(JFdmU@K>0>LnwIodUM@iRWj z!fOtB6jFWWYc|^h4~;@|V5RoQKyNXMGm`>a8a$7W`y^B6%N(8C*t(iubW!wN_JbiY#H7E7}z-Pj<_g)L9khf$_{MG!;LO@!TRwZzI!KnZJ-k z*YyCaZ4htaQEI2crA*2$-8P^v-)E9SP>D-YxSu5o6{@oqH2sR#s-z;Z7eg#_<7${l zC325gGL&)rve=-szs+<@)m%)M{6b_9P~+(wX!`V(yg)@wuGrNyQijfh3BJGw?{=R* zf7*>I1l|2(_hH8-EdB+YfVjm!2dyyD2Y>(mw?B3q0Mp$L;QV8U$UtvsXDlrB?vH;9 PZ{GYruE_&LjgSBUx8tUp diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-types-final.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup-types-final.log.gz deleted file mode 100644 index 2bd498a737cda6a4187b7688fa2ea07c6df813e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2812 zcmV-DzkXiUf{mmEYU~KxA_!#V@-p*fLYvLzZ?@}|R%XAEUiKm!d-9vd``Zt<_nTkt zx??MYy!-RtfBxg2k=*fSE1L&0?cME@bq~Q2`MchI{`L2poibA2_}1SwP6k{a#l3&G z!Joe?qx^x|&0qL7{I%(w-6>6*|3qW^pgQ$~+W&5XbbAWF`1c!SHXrV9fBWRq)0=-q zV_3^*RA2#q<6U#7)%Mo;QBjVx91L2HL z_Fe=ln$AzI_R`7DZ*FgIH}>Hb;cQD5OI|!^EgZ=fKV>27WMl=4a&Uzg3V-dB5f93= zLbYUq3dd`NO{|^Tc$H;s5s)o%gnQH@34XxI<|w4@4|1KX{sYtVz(2T1o$ZNg3_CLY9XIPb9Hf*T|3^YZH4UBMBGrb4!hF;UG&;pT)COaP=*FA>^3@7geH{nyNZ6ZO}Unlc|yIz*a**ZviQpk?e`82&6u%fr}iw*wDze&vmEk>=y5Ro#0j{9&k}MS zeWNC@XB)IM>(hz7G~_Ry#^#NIfqUehhBM{fqoAa6(x6xY11JW@kjuHD*2=AV<#^}3 z$~b&MLiIx}ztUKv8mUEcbQL*$LbYp!)uPyo_}i~sc*%PpgT6uIY=(K zI0UDvqWhWt*pfrDVypTX#t6$N5yS3_KrwJo4$)%Ns190sYZyB9!Q%VYb`t#NbePXytx(6uc_zmWzFoi{OC*x&*ITySIj|@DMP6i64(xTDUFYrRo)Wlp& zI2)IKIyHQ78W#mkAzy@H9y(nqdZ=%JqX!8V{%D$npRsqQ{5*@DJwVrIUs+Zu`GosP zQ~d!SC^#C=if^7Pa8H_l&+L^I?H=C3^sm9U+i2Hzp9UZdy;oM2){oHKlqNA^J2mXlLDwZ1&&Q${6nevj!5 zNToX(t7B(ag>3TzTxZRhbVS%M7M-&&fi`mW<3#7K~52@bPH?R9c1vCP!$Q;Q|wQN$#7u`3piZ$x=a3 zg>kxcoqshKa#*ZMGQ_xegO?m|UnQWRiEJ%=4s=5XIOC7XTuZS&>tFy3tlk}`c>*y& zK)QkB^+42%;WAD+p$9M7KQ9kMZ?4+4E+x$?C_d40=a6FAt$sF9Xloy0zdqzc>#US+ z!Y50cPP8&{B;w3TMEnJ4o#>ixX24d3xg*Qo6Q?zId-my?9_6W%EJ}FHb23DxyJ@iF z)NQkp-+5Kc)NV_e(E%K3B1!|x?2vqGCfozRK4ISZuCx4mZoac^Y!KragLmM6AhC7$ zQhu1iA){lZn*%iu3vPxO@XQcHod-&-C>h__14`*Q%8XM7D0(b3lo>sHCT$Se6GP#i z?#YT-Z$Euxo9Do!a#1}}sPiF@0GFc>U2dwxIW}FkQlEy;MyDwR3aB4AC$+D{+0!m@ z|8_HWAWLbZ!&;ZG#!OiByj;W2CINCWRUzC*&w~Nmh!Dd7P?}6aRy?zZ&p=|w2FO-A zedIf9j;#=bkuGZsE4n_Ejq)`ThP8QGkCzePWOeRHlto;}C2tw!^P-1c+E8qHB^?fw zn?xaOK#j;Agd5-s6^s2QJ5D*-lV)v@ms7`y7@EwAb z_;N!0KJv?2F85`GubULbYn+R+CJR4NnS!6vVt%u^CVKEL_tH!$Et(VWzyKnd1Ppv~ z;GVraYAY`rIruqe%SfXfFSi`G#zs1mH1{v)#jgVVz@|$x?wRAa7s%we{?YUw4x@AocBT**Pc|br%^kNfc zc8go(W;=&}m{PwVL^w$C)QOQMHSaQfQJ*S8ch7qXUmZI6j%@cXxv6MmKWgAIk~ zZlls|=3_jnPL4}BezJM~xJvqzpe)xF7rDYkN7QaI$&0k-?zy^87RBqt9a2vjS){Md zoQ^&iw?=lfuwbPNm`kz!gPId2nBRHhD8YTIiWqfTb;2^O{eo049iQx))c@s@=c31q z>^F*J(MntF!u+7u&q!@QO#gD!q(a z%53g$??3u? O@BRl3J@H!+C;$K_!+oj% diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md index 109cb6826..a17d120b9 100644 --- a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md +++ b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md @@ -25,7 +25,7 @@ Foundation checks passed locally: | Restart and new-version warm-up process | 1 passed | | Strict Clippy, all targets for the three crates | passed | -The accompanying `cleanup-*.log.gz` files record these checks. The restart process test covers same-version recovery without re-ingestion and new-version cold start followed by fresh input. Downstream verification checks the restacked Level 1/2 plans and current storage schema. +Generated logs are kept outside version control. The restart process test covers same-version recovery without re-ingestion and new-version cold start followed by fresh input. Downstream verification checks the restacked Level 1/2 plans and current storage schema. No manual deployment verification or human approval is claimed. diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/clippy.log.gz deleted file mode 100644 index 9037562b0c3431fe4970e66669ef272f6c065090..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 207 zcmV;=05Ja_iwFP!00002|E-TPZvrt4g?E0%8)3K$sHlh$kUAH2tf~?&KCmtqM@|lO z{Cb))RNa|8`1$*u0buK;={THlwVcarMbWY+lm#TWT6+&A|z J)x0|a000NQYCQk| diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/identity-integration-before-fix.log.gz deleted file mode 100644 index 96ceb7e1df76a54f88b920838620f771c7741347..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11720 zcmV;(EjQ91iwFP!00002|J8llZsf?7;Jdzp(=Ss6OjSv0)lCgQ^m;tNU=}+b+xuh~ zEG8M5$rh8rW{{Gy2K(=OZgC-#nG8~DV{E{7NyyGy*@N$Ro1&CGg*^% z$3~gt@%sJE`|IS*hr_YRy6nTxfByN8|JAGZ_->Qs52`LcynoW|L)R*0J~Zv;yRYxI zWt~;;OkqCcZPvlBJ-D}5Nw`j~>#o&R+EiJsUbs{UYP1YOdyO?b99kgxx3VtHUKPnV zIA!-OX=1xS zOq+kG%FX-M_#MK+?@R}qd&hu26l&A&;D4JCvG1}Uv(KBmPuHJsKi_`&oNwP=U9|%L z=H@OT_*eM9NtWmRp|7$|x2b7Vet%DYyuZ(NUw7%DtW)!#x_qBLmEAtwD_G-@H74zJ znjcK?+a_zXgGt`Mf1l`wDBuDZj$zUg{;Qwr^r5Vav@BHJmECa(uwbX*{+8@&{D-vA z%A~dK(nhsi-)_=IS7m-ofe3c0ZQ8Q#;vnPQ@#X;FLsg}%$_n@wZ!ZppYFqeKQ5vAR zBb&d{ie|kex%ej-ECHtS>)C3k42ji=1V}k zI;sMHc+n0#$OZcdU>*Ec;D@-co9uvJ?i(N|m9{eoK2Ovude@3BDDa|2Mz zE)73Z0`IQVV|Ji-R_kuSn?=oSjG)sKJ8a?G zt5nCogex+zr-h&$8v$}Opj@WrSm%TjxJW#x!dn1-Gl`EKAhO|G*SICZ39|m&;sbC{ z0?EL4`Sk!xtWe`6xvb$v%szwXm=Yuq#_(lS2$wCwCu9)(2aBN^hO$970g%#@lpzulJN(P7wfNhZAvbIzaiv?lB9iUd6Hvf4Dd9ZJws3(1=sg7Hqy z2-`oFsshWy|K*%z6$Mtr^XmdN4WJA;T3O-yd|lvz@7Dp!h9Qc?jAYDSJ+qcNp(k7b zRHNQ6w4%t4X4c`Y;S9E!uiB)uPS8dd=Jpz3_eVqgcu2n`ng1PXO(21~GlF$mWwLgx z<&uZepf%yNrb0C!!hwC1R!s*b(dfR-$zIsPUON-L|J4fm^yZB^WHsu!22}8@>rtm5 zT^tczFk|@@{ah$NH;Tn3AdIwHQU_yq4hE?bZeppJezoEfM-R;j+44K{tfD1eGy@sP z7F7K97`}@Rm}|$XIo=iujnw@bDB?s{h$9K3sxAD$PZoC$Ht!*8k5Sv&w!lysab}<+ zS6V;xO%x0!XQX=z2(^c-F=RoZ{KdnK_R}g*)D1_g>7vi!HU}zk3pzSzh1OUqQH45`6>Aa)B+z`X@`sT+HGL@+YE7`g zN@YkLsCa<7yNAzNg9hvWGo)l7r{EqCmDaNzt zDnh#il=xnhBh>lvS(7^ucjH8P=qnS$W3W2Ee_q&GCMPxve`?ZlIS;LGS#m0TSfO`S zHDq_@71<$hyz)D2k7N{S?eNQLWM#tBpt}Q{sAmUj(*t<`EKng>=&wiY78ERUx8N%B zCx4|Q&4^k3idbewpFS4s}9IZ$B%Iu zG}w!LC^+n)>BO_VWOL4G{S~9+j$2W2xuaGTicz& zt--A#4-)A-0B8zx!n${=25v!*!jhvk@=A0`pE~gf3ThEUSB0~=`6oo+uF{(!r`!We z@`7fPY6hER7en}B)4&B}S5}S2`#~=XTSR58vYk2$c3Yx#*&PE{2CFB;yqrQtuHF$R z9HP+5f*O8-Hk$V|TfSXc&j>T(*%PxJAT}<*aHc452~8Kv-lBO(;w($uVM`Z`jEP$g}rz7FggvOs!6ALbHHQI7?ja()j6l zvEHI*t3?$f&)S;SIsbra^%U|S_Zoo`jD0s2w`o>j%y?wMhtPz`5W9fwK6JGZqL-Zz z%zKo`aJvm^OvM?cW*$@1YU1Q(SngZ)tF{@4k;n~tnzUFHFtZ!vpXpCvWq*E&Y@D##7!K;fKnVm4EY$5wJ2O- zyuzLR(xWUTkug{Zl!t2M0;|yzLh+=bX?Z~aGxt|s#S0z+B{^`=m*hdeaZWn*Eb_yj zD(X`0b$VKGf{%zQeQL=m%>IlhkbB*UdbsJ!s$d-iO~D}ELe%i2S?3nIHE|n8ag2`| zq*F_A*FXzE#*JJtOOe%C5Z>7twSx0X!$K8n$y?)TmmKcl+nf-{=RiBJNEmuDHY$=a zBQ8`^=_9E(7{gkEn?S7^TbgIJk#k$!lg#tDGj@NW9Rl`}SC~di#yB+WZ$*}1l+-G# zq|ka{g-SX5yShkJTM-$&bBbqJx|!zGXTXiQh#9HYV{{X80$DTm@0{Isoizqu&`H3U zo|N~}EQ997)g)XAM5ASGN;K&#bTN#S0m*m|zj#l{TTNz+Dt=0?6AV1@za-L*|2%;9 z)XTq_-vP}59siVUZGu`^C)e+`g_3pZlCtRhJmRxK=44a#JXm3H!vlsxw&(xQrjx~MT3FGU%vcZ zOfO>gh;}gr<wZ2xNB8f7J1YfHNqcw9yva)93yQD1QR3rwh z{}#YiS&nb&NLiT32GS>0?)KdrR8an01fH!B2_d7kjQig9p*sxy6X>sC&t=-$ED`yB3`{^Zh@`$iS26kWfX0=o z#J7hTa04j1flUP54E95@Brs3LHqqsers%iY)g&L#J^hr8!V?lyWyd6F8vfRY9{%*# z9`))(qv?qWA-KHukdqh$*cnT!H_XiE{6BM_hAd&Vmef3dCF zfhJ)10UNS*X^mKLTht?%W2?uKXaEE`GHO7)O{(Et!p1r5z~ZY=ndkC87o+R~kjA1S4Z%TL#Be7=$Zb2n2~0v=Pi6 zf|sPdkrn+2N->UI;vgf4op4m`#csq7<8jrTB7e}2$a2vrc8reTP3O_f>I_Ca;5lVh z<2JOPnX@t}XH zM@a>W!p^1aOk{q|G5|LEm8kHJ>WHy96~elnZBC2zF$#kS+#R8uSSGJcPbrWnAoas) zVI2yKjV_SSXQAIH3b5=1{YnFXm;xOGfO4kib%&*@(cNNw#&2YdmzZzc&9fTHnhWNB zqXD&07aFo7@48n$u+Le`= zcO0M*8*P#`U*t<1W+1J>A`}@M>UTtE~UAj}p#8;C@P%|z;J=Opr`aB0FEbCP;1>68Ve^-)<1fFskm4ErYH zWxm_&pcv5>93XEisfB>kpZS!5mu+0D8O@F{W=6h;VW~bDYZ!bZmLIJY8q=%v^Ysn< z_crYmyf94F#+DkG9wSlQr>s2$>cx`#bxcSUuU~tOS$NK7A)mNZSeEhb4e#3ViJw6V z556gm4RV2~Bx6V4F8f5m3Q_qnuhe8-rg*2V>bMZLjq^AXLG>2{SQu%6cQp}4K?Mh->jVZWUa}tkPV1s8A*fq|+0z+J7HqVu+JsBnO@~j zy1)f081o72b$24qg2Y7-_${FIkV5qB31Y|E^p+9U$1vO+uN-_p&@knWAfhHRG#zcDy#QvE7yM*!;(1~9T zlRU7ulZ!i9-LZHvaDf!~^sta}8_npP)&{B?xmYOpD(!D1^Ow+!DB1N{TP#uFDChc| ziMrygIe#D9hT|)%RJKKy4fm?r-6AzPWCqT!&-BZJ#4Hmao@n`1kt$qk>%b1=1$i5? z`XPR&BBjg2LCgWphp?H&WL!zYJX*38jNmerGiwag8q<-*11!yJ2U{xX2F&UW`3p;Q zw`%x>)99B2Ra6d62%tl3$r(w^9<47ts(Eir-dVm;{M~ZY&NkoBE_EWvgxX2^& z$>qRUu=Apnc2Uir_Z${$HtYR?MTSY`vPW?xuYo9en=8+M5j5pZ++1HjSeNL{N9(#< z5AI-r?MsZKdK9k#DdjVg56O=Se|H9!YJ+4PZZASj5*4P^)gp2;A3Q0A?)!$<{N8t99NR=lTJ4Ve%YXC%dzSM72np~r&s%I3O=WNT@21dIRCODYr0X~mXik~fl))m;0!mq;+L6ax*@1}U{lHb=-4!eC^~I5k6HEWF6NLerwseS!X%kgDOHmH@({ z2V$-d>S+LnLau#lVs4d;Q%YSI4$lM}yt5mVXXA-pgAurQNS-MzT6Yn|? zOCqN_T%187t5bYjNi)QkCy`k7+|X+bOD`{5d?9c#zpc(KO~-74k|e!Jw3)r3=!6Lr zr-(Hyr*%An=@RqIrF2|kRW@z#Ttl^K@b0-VuMQr83)<{q;m|!k?~S>50byrGx4Psu z&fuN`u-K>>sRd0-tePKIth6>w!Dfb29`jhN0BP;aszs2-cDE_fjV;ivh{nhn{&PKWw6caBU4q_t!vMu9_#H}b z-uzY*mkH~gz+|n{r@g9aAJo+nTLc|mupl{;NUfLdD%e>L)KOp-PnfZ`#WK+NFx`{W zJCG4V4ymT^M-gW_x7qO4OZV`9c)RtwmY0J+N_c*BZs|g6k7jtU-}tscxvz>mj=@V+eH& zjdi)Ot+G@0C}EgW0CDG7V744S>8Kk}Rml765bt)Jj_zhX?y+jlWr+HfMi@%5y+!6i zsfTeY&GMz}j`?Ba*sT+@LS?}f2Vpvq_E;jeq~Y`nE}EbUMGj%1DWt0H3XnqIoQv-W zD8%@YLwfq8_s^0A1Q!maPLg3MxBKPhh)yDUX!hM5=9jb_qIB8!uDO!bM)_hFlAPyF z!ZT2vDD+@5Gj3ls2V@|wSA4%I^M?tk%)$p%Gu!nl8Z60R=9i9(ZjSUa;v*5KVEGOL zBUg+~(&WzDm8XC_OQFOVQm+sdCQiP(Hj|i~vPsn~X0L-aNUars81WaE$ziClme{0o z;3Kp|Y~33{ao+c`*{0n4Hzv3$4FXQPs=I=I$`RF+Q$@Kvwyp-$=0{F8S(iKF01a~b zT)Q{EJ8a{F(b$t-Q?ZD5gimBk<`v&0C^qhIOxcTZ%E@W|mGW;%YH0f8x5? zg+(qL3k`N%=^cLgWms{ctpQ8tV);w-vlKj5=Q_wDR(8>WCrXc^z?Dna_x4H96idYA zaApK&1Fk1ZAN^x$Cn|z1ZYZFAN<=n;FM2`5O|73M@RM>-scF+s5-snWXt12pWS!XHVJw1n2dqc-Vp1dew^+N7ne& zOKI+L1$^cy^)N8c7OoGaNI2Yn{k*4QcBPXMgnL+fSP#4$w7)ybQ8etG@VTb=7fcy` z31oay6ygBRWkX7_zL2N|Y{i=#fv9uv*ibuJzQCzyxFp;sKE3v}%`N=*T=Eqj;@Bw# zuML1#1wI>R^kd;UW~00u{3uPP)7=`8a{^WN$F4%V>%cSl@+E@NCkU~WJX0<*nCNA` zY7zfx{Bkre)jmx++6u3MF92Z23N{QG^s^3gXJ>L6*PX{Y-8gTCQ=ly-UvnY4bn1I5 zWbks}Q!iPXxrFh8-(;a+c3EsNX4QgL()w;+C0)^lc;)(a?4b}iR?fV90VhoSte)CD zsn&-(R%t)uWl(sDd;jc0&IdpTfK5nFa-fEmBVW^J-} zN(ZZ71)`jsJLi?N{Sl&oy8?w`%JO{)WSehVu9c=9lx^3=-N0WWp*>8VMld+|3n8M` z7u-C@T`KIWbvEH?8QkCE{!wK;*|a%eLF5Z}Z2|ulyRNCkQ5rJV+Rer@F9B-0qEL^c z#~Y3kroX>%g~zNtzXJTRToRFSrNbPga2bVE|60hf0U1e=>kO_mUnN@uRy{Edm-U6y zvhC9)oQ*haoor37fkJD?y6?{>5Y0jJg3^tM_X#*FVLJbU1!?1|%iskSb7--<@-^!5 zeOApn2h5H!p|lp}y2jN``YrF;GQODGSCjDk7h7b^E;P7sELIJ=YT7z735+g00yB8P zvTGKvp9 zoEJP8=V5t_38dBU1b{hmvtur8;(?o=K#9`}(KST;V8&L7HLS~^xjjx$h;PWvxfe?z ztKe3HS9sy^x!2ES7UMEDzsIngY5XZW^_iLRuqV}G8<*@_*9B8jJET?({kv9DV3X_? zyH=Q2I~u7T$Q1EMDJPclzVewQT?ZELc^NrckvkZ9=s0MJr`yQD5${u62ar1?y#%7G zraDkI6(%G3R-L^Gh}6z|Ti9xVDvh~XmKu9lrGb1sraNtr= z1Sx=W!)M$8W|5(T*#1+iZ7syBy*08;f=OWGHU_4W@iIkea!wzb?R7X!+$UWsS`dG3 zM9Jhdy14F5fGR{iClm}WB^=9U7S}(MV}POBE(ubN8Ps*g_N}q`60R9VI5OQ8a*r-- zxaD!Dh5*sy$+zLmn5p`f^;nkX3dl$(oDt-sxW0fi@}(}8q?v~;HMnVl7>)g+Ho%LcaSa<%QyDX zZVWH*!q-pck@BC>C+hPr#MJN!kN!{8v|%~?u@&cw<2TzN3**|qLvrUT|)Tkg;ZO-JLotuiJZc1@LW z0VA?2Fq#{B^G;Te4l-C$M&?cHm0N;YGP#kf(1Z&^w{(e$$GrPTF8AJ z2p%B8thHPhwC|^^Ex2!N)^19WT>#JZMK9EAUA@Q|m+lx&?w;y_Gtc>4`8~#OTxU7o zb8k}BS#tuPK^>&}qRqzoKbO-^DYBFNf{Lm*szdYJNt5EA@kX4QI3MMi?+!?P0=PI+ z_RU+G>JAu#nPJm5UykN{%Y~U90~`XU(Fi>rKx*N~&T=bv5xq`twcip6=bJ*Z?z^Uo zi5H!yyV{<27t;@8&JiR}(B`e126PANfb7m`0W2kat1!FyVW-nkvFGt{SZkY4o7wTp z**bBLKpudGnoroub+%w>QvP9938%;vV0s2eaeXa?DOn1J2l;$rZcN_~8nd0UBjS$1 zoQ{cM%ii<|%~id%Dg6`EZl_muohmA*&Do0^-xil9fe^sj zDmXasK}&cmI(YG0C_p{fXv~pfiyN7%FQx4?S;}^vL7;HX-;A@K8pV?o`VDGbayjzW zv0v8mSp&ktfM;X4u z_OOeXToT4UAZ2Z5kt@FUFBkn=PDzh_vP1m@SZ`Q))+ZtZ*;dwd8Y)4yP8{;c@LrOnHXkTOTfd%uNwQS;F3< z>6W*Y9_oYsww8=%<8k&GcT$Xg^j)?)(T;g`=II#P%d6Ap*;qJ^S7@5%Q_SWA!T+L=O1^ zw3hY%WoQJo9VXiHu%OGZtSBjKv0LpxX`G_P4Wec?V?QtT+0x>{w$}>s?DA^nTy}cB zI~S@U=SRLRo~FeCwL*-tDRXW?;vL(8XQ`1_$a zA#rjJFvyig?hGye1+<5g(cw!+WPzBkvhTWvyoQuKMD2Z!`;)3J2 z8{UKY^4(6>?K4{}?sB2XSYJ4dlKhI4W95D3gLy?Zl;8({Ey~SBI<_eu`($5wRHVkQ>hL`b^ z$odW`wUa?Jgq^vn=mm47ZML`MYpoy|crN8u(TssdOM#Tu&b5)w_L<-YEBPN8`@5{) zDQE4|d*~Mz4 z*ℑdj@F~&W&Mar!}41FXKZ-@u-X&FrCjojCz`v0$=njx$!;EMSzRp9MUH}h<%;U zRE<5MPjInABigW21^Bz$9d-fxa{O>$%ZkX8x=wC&@R^>;_DJZPro|{$W zY`Y0XGG+{$^*Fl-I5n=oR8>2TFlIXDMCa_Fo^6nbW9L^61el1mWeGX8Cn*j6|K`qP7*`(czdZ$wl_00`ha4o1qcG^;>)1Z$dd(tn+ zsc9??^-nvxFUZ{>B^~2h?qmpPd7h^upeeh8C9$)$FoDC<$0EUtq|Q1WHGF3oV)_ks zyyLt{zIKwk7>vMqRg@huF>7wwlMm`K1~CN@#+k&XgQ=NXcgso653#efSJsdEL78;e zdDBit7_8tGiqr8}mK|ZHuS|`T*UeZgYwYSrPOS4Yvd&Q%XXcD~xYGiu41>n4I8(t( zc^uq!d1B&)aoni)JY}wXCU;@f!LO@jbestpTSL&d^WbU4HVBGlXhdg^=|zG$s^Oqc z3ffYIN6?CNRMN6N3AL|L@YfohjwXF|49BrZ4Wi_8n;q6ZvT0=F-V0}$OTG`&qr;-h zzatl{3VN*vQ$8*@;qd|8qF^KZQ9KPPX#-dunF^!aT9Z=a zSA8&T{=x3!gWbma*4+R2<@5FZmmlx1a!MRbzWs?t7N92?hZ?+j`z-+lA*+*be`wop z3F(jb$)Dw2z4;-?GjX2I#2Mmtw$klS3HA80KmVANCUIZ<;q4W$3IN;ob%J~zcJY7z z{Fn59{rvm?_{*Pu{{0_*ck?Y#^2!iA=0K21Ms7KLvnt}Ut;bW* zP$TIS)(cOwN+zfZ1teq43IUvBPh?{DtzuV~eG)F%Nj zXFp4+Pxr}x(bM_w#0JstlOKM4hXnY;za%_w4}KFg1Aj+MEV0kr>d6Uk8tWcSf)N`w z0q%$7|IJ?x&+Hn`0ai74ws)oye~Z=ErPr}^0l9#$x9cw8`XgU}FXs7+ZvG>&iq8Km zyVI!$liZfx&a3ahUf;U+bNBJfN4~x#43e(&j{y12 zzq1PV=7&GB6iZx!Gdimh3~qDxCTwzPMH!PqBB_c$ynQ==ac+e37Swpq0%~8jik3ji z!FHPWA}*{-+Qcuuq4i7gReZVqvhFHAeZ3i6#eX)Ot2w-i=CmVS&P711l_Gd(8tercPnXi$bMMec*p7A11?mngvTKPc>Z(3$$ANDDv1l6 zB{l>FAEyFVw49?jP-#rB($Cj7@ZVeC-Q6k|Vw_XU`M~G4^jcP}BDZsWvyOakKEv&R zs#WU*0ViH>-0C(N6bHV+bhy8MJpAqLee%~Igd2pHa-Y2LD*iTq6D#IwFTIB4OUNC3 z{IpKp{rdGE|AoBLh}ETc^*YPS{e4<@e|L8%0^kj)=Z5_lnhL_;dPu24AJziGo^6=0 znx5w%TQdH<5)Gd|-hcW+tb`mIz(IP@`jC%2sAG)KTh_r~xzt+(--^AJ*!c1G`u^t2 zKlY!J%USuXjYwLtp*kW$4{edcISX#Ji)#+@a*H<_;+)Mf2k3PSJX+=NBQ>t1mXDdeME%|;cQ}8zevy z=^%iF0q)`X^rN3HW(E)>vC*?yr0dUclBTxmuAjcVyuAF+qB^|h<>}=;$y@yIv1(q9 zd6B)~gO{p&*>`!7zNC4R9onk-_p){6GKW|N6hXtoiuYJ~_N*W%|>P?^W~KHd*F=s+-?``OkkH^D-&^WFolfh_P&H-|MyGXRJ_Ok zZ&y}zn_u$3v-JOaX_MxZwg2Y+{UtA7zFuCw{p$GFFaN76OU$w>^A-cR7uOtqD)Rl0 zjSCYc`u}n*&GBC{&`)W$?@si8>v%%{N`6j$-~aO4m*2nr{_Q`0KOF!4haVap|LfP^ zUwHf<@PA*D!=bx$MMB5Eb9Hw3;}8Duk3SAoSGK!LUhdp$)*jBg_q;vt&Kc3TB(>YM z)$VX{(O>JNNiOc?#~*)usb1#}9uGNZAZ&^Zk-6P#QS6#5N$J02kU5jliO`<9T~7bM zdM|hMXwqGtW@VeVAJYaKey~d`nps)3yE<#yuG#PEs>lx?JGxJ&tf`y4Y$q-?)hWsn zn~c7bzhyi8ph;-Nq^Qryw6U_Lp+BX$qj&N_n=kuUH`&g$S^ewoJ!x=Rc5U{3;_S|} z_vggz4s;+zb@IEmJK~!%@vMByimJ|bSh|}!@JHGte6%^8Yxa?)_{X&0%GYU$Ro7y5 zE^FRsovyp=j>#o2CY~w%vCZp(HmBRyRgJIYRmpcXDe}av8YQf6##k=7V5F|5`=9Na zj{ZB%4pmBvE3y+V1Krh<-V*`l7bWd`f3VZ3 zp+k{7Qdbg{oMBZqdaiA{R1o{j8JE3u#>`#);*x3V#w8}BXn#7#DNe#>M zOR(R|xyb#OgPl5pqYX|@nV)rXa>U-4xgCfp(M^QdEwe^O&+m+D`(PtyO-@t!JA(}P z)6o~I3z%^HlRTX|nsL~A*ZuYj_+#cAU5)IoE-75hc=7qvLF1p%{Z8?=E)e-`egJRh z+mQ{K`JGA9$?d8pKamiO?-P#ta8AG>i3m~6b@d}sD5Y(Cqr=S0x@-N&c}dr?&|7Y& z-utTDTAX1*Bg@z6vrCd=2?_CwU7DPxt|a?j(|+ge;IwGq-A8g^zMY@Ss>#v`lW&su zUDqD}V@J%Zp`AU@g`)eFUucWzUf`115A~SS3iIv_P3N?$_TctB8JZ&LfHYcC?0Pc4 zddr$)LGLgx!`s7tkc8b;$6ej+3)1Nz2}yb!^Y1gMT#|ys-v!UDq1%_0G+m}lw=l_z z-Jz)5U{|v5*`X6|g2yyG)az67Ck(DoWUc&>7UpP9y`fKZ&1zXog`e=8Av6v)Uz%f1 z8$mZs9w9N)0;)4h_KNPSJ12CEJ06L|IsF(a<-GZOE=u>pk*d9OALW6seUQ-1Zbz^D z<3$K@nWB7PJ5-V+Oz8jc5z)CPEt4hPoPJ3X6|8YG%K~Riqglu!4sTfH@S!TY3u%%W zY3Per#H8XC(bmz`aRo899j&75E*TyR=sKDL4YtQ@l4i(*M~Ov3k#=A=5T3e_w8JaM z`vaBUR$|+3Ug#-|qc06z*SzJp1&59#&PA>wh%z^!yHmQlYTC!^QZl*#KI*WOPA)TF z>+qwco0Vn-S$Hjr0eUicdvXn`hS=2(ggR0T_W7x+y8e-1l>G-SexDVrJh*Vj=zmB{ zKOVCt8!B<*?9-Fm_JCe0~Qg;UL z`r6J5rkoyWYf|nb&kC~X!ryOMhtp2oWyO*HA!CtZ1uguNGz;b@Yo#5YJi|%Dx;o)` z9rFTjX4}D)n`|?vSjdp#2}`VV_FYu|5ryE+KqZVQ<%Ah~I^?|s_dr6^JS>`|H}&8} zQHHa{u;&lQ?7h#Ob9T_QP>)Dxwl~D|y>S<7=XVArgE&+1Vh7KETCncH-SlEE|690z z+OTk%_d;EENlI7>j#S~K?V#SUPK;AZg^~OP2P9?EUb}kI7;-moiEJQ<$E0>_=t1;i zMl-ICm%XNZMoOc~dDb0S6Uy*1SfwAmqD=;sxob^RRWYd7{ZO4r$q1T!P`XJ2?o{Q2 z!i+Oy4;p6URhrmOqzdnH>Os{S+ksqO`bXB9Ee8#FK&_%V({1kQs7Cm+v&;9Tw2hXdjC#!IgntMz+nFzGoIOnS!`1w6Aoi>XtB5TJ^C0`=O`7U#El1 zzI#IZjy8@aO`=m|{PNR3l7hjGAW+ed!mS@D(W2FjUik5edbcvWk5zNKaxFL6PF-LZ zu*c#38XuYrNtq4J@=~|zX!cxU!U!k2)1+Dlsfn|0iVu8q#xUNU=Gh{=((O4DA)!hb zUEG}<8&M+4=d779w)b1gASBkirnCjqsnMz8b*duX8_=chN`*URC0$UyZ+7p^0L$St zzjiP3C=khgN0GPbdwMvbw^>fmck7>%7c}#?4`mISjJ)BXW+2ZNn17>q}$a&v+Qo ze$|8{vfh*leoyw)P1rfl!AtrB90$QaZ``&puZ{4fhhwVKYUdZ5J6-R9WA}b^WzN{4 z%8l@Yi7x>Po+Bx6N$yO8z6QA{oUA~Ecb!PO9O;zUe&#oF>JZ_DBvNZ%4v&mXxMh}ecrTut%C-DI z>Pq6_-yw&{e0!ynsOsC7zRccZ_TjHS4eIP_+1~Gx6hY~MZ5)s0=Ok}0C{qobWeo)7 zA!$mc8U8Jv=+iOGg4hgsOjqA!aaf;P5`}#^@{` zPB2`|5E;`U{?3|;?hO09Gkqi?Cy#8M^~V(NgvE}7e~@LUk0?8l47aC)UF|;|cy%+Z zHBU{f3^zu7)&6Q-#mn4Z1=zp|a?F|YllM5=sXEzT4rvL+zztQxf57przI8cmCegk3 z*SpV<_|~fGr)DJ`$)6phI*YFnW4k;hJ9swfWu$zt6nUf>aqDSA*qn z3b-7$rj4$qR^~`v|3QEFkrUSH#JMc}ua_?`2s6olX-paac_9^})BhI6N{2}=<-cBz zmXw#5FJFHA^3DCQRkmF(mGNKMufKlzCHwaE+b_xCKeP0gDBGVKCS^ zIKF$o|C539U@(0eU{9Cxa(@g?n$ZLhCssb)qE}KGNpF1cQ(Eoow=ZA(a$h-bjTk&V zbUMteA!#;Y<)^BkM+5HYHAl?@z5l!LI{cbxAp=W5N7^0e5Mwv9n#tx>=$axDye9of zTR5w?f^X@D)dVm6S6d4;N)#!;Y|K;Se}> z!w~h@D5E357D&+O_-7@Eb^@28 z6mh^7aereon$918Tq@e3stG>R6H+k!yEs+5WBE1+r;PtHOX=>=ZO({|lftt{inyfZ zR*1=v&DTg=;bZAw$PgqcIuam^6D58eIwxEPLDNmSv!CDmuxJa_(;wd4M&^(2aHX2#r zqDJ*buWDLi#C2)49GxQbWN6#m|LP82nN!Jf20rEYWDpb5n5Q@!VkJ2@f=a*K8jvW(|X{eEPs9c=?c*6*~ zaT_=ztFG9wQSQEd|IQjCZo}MZoO8z>V;@6+7m4;4IV<#3hx+c_YaY0~Ival>%@F=U zIH}5s7k#KbZL&&p`qKU2$cAV22th@fz#&Pm^1P-G-()8ev`(J6V>6JCc`}DnKWtSZ zierWxeEHt!BoUE_3@9Z+$k!-=#U+2Z#|@O>?wBdZ2?eG*2R=1*%EyjVDcY)jjbsOH zX`YJX@m9oO(i?}Z8Kp|6^jg{++%r7+rT6_Uce#Y6h}LdTtQh*t)X~l#`-Ze2Bp^rJ z2O-8@V3-+Ew?`G*BVm21-f-KYnflwrl5IS7V)C8zwKw?zX}!$Bk`odITdrf;v7U6) z9iO>=)@%6e!>aM3B&(h-s^0?Ur&CAI)#&m0S|a1cxx$r8;)7}tXDX(S${8f||1)@{ z`>vgkN0LMV_MN0sSNHTA7OO}mtGEKF$YDEZo~+7V{%i{8GbSf8OGE8|wZevs(7i!%lB$SY+(r0o{i+Q~94s6eU@ z2>(%uYr}cD)HYo?kA3zJI*L^LFdl?HDep+u*C+#ha_3 zkEio4QZzQ0R+6UdLI$TRx;m=3t*U|(+-J|eze8}UvTEP2UuEqA-dSdnQ-_v|hxKPA z*bdov=KOpgP&P_76I4cS^DxMc~Cfbv|X%2k9mfcpX_-nE^ zu6l?A@{q;)gBwYZ_s^LA;568F(7{W3RzBG>Tu+>io!MKSy}P+Q>4tD8o$4grj4?9lv`o_ym1mo zteiNz5TA6jltMPUL#kbret|8Q*olCB(Y-+F757=K4%kJ`*alur6iE~n+YReA z7g(hb^N9JR`J3dL()RiP=I}o)9jQEWW^OhcQi6YV6|$m2IX-;BS^$_&Kk_5I%A)>z z-GO#)52oR37b10F@{^UEXE;tH$>rZbn70b#$oic;b^MN0HppCindS!oyMANu;Wre=e}QG8c9sKh?_~IyIZ2L%Xz4h14Rf-kcC;} zjrhsa5BUdL0GX5>^8IJZzpHV&GI=A-NjFuw&CKWG2W9aj1z_0g>{2y8Nm0@xyyyZ) zZt`?G4Jek;YhPzpa)a#p%jt@1oYy>7e06UtyUU)HT*odO&Ch^=3?=%uRAUHoNgP1(cqqsZeb3WDUeCRulXowF zVbvV9R8Q+lXp`+KgeNt98&s8uHl7m!dfZb}&1E94)w+l!TYX5H6i$CFgyEX2aA9w~ zqMi=pS~h%XvqK)uk?X2SGF5wMo%j%lGdYlG+`>UKuE~ZCAp3G;Ul=Q+^al2H*U!kR zUpRVM#i%sSN8Q(ss~}o&eGS}%2p^ARX3ZFvQ?nJyjNNSg?U_1kd#EDAM06<_l*&z! z4^x%P2ZZAW7tn`Tu?1RQ&qXoA0+Wd&2ftsMAoX1b)F{Z35j7?A8I$GIiLYP2yqYcY z^;dI^Eg$Y;gPn5(fx|mqGNcxsXo{lTeEe+&+cJ-?#22{)LLH=9X)9t0@u^#@&YCEj zMNn)QsWL+XU8zc05`rG7IzdeN;j+{Rq1TuOJk&7*DxqOT$@KD>xOdH>T9D?^cSx-RN0@|8LN$<=s>^Ih2XIugA71w#F zp~G6QqLR(pKBW9{qLdEDnz^U2rj zy*Cbr0=mXcIaQVCT;OEsaZo(fwL%Y`mC?BF@V+S$56%XsWMCq*S%`RdCnqJ^jXGvo zI&~+9Q;wnQ{9V)}w;8il->$_s6uipv!!!#`1FGTzhjSv*GbYe5?%^4RTqh)VYZbIc zV=Q;?7N*$v@P75ef1DlV*SN02L^AFr{&V@@xNm*joe1E~hmw|$-@=rT z$&&$ku;deikbcX`-TOH!c^(X`36!GC3W97g^X*q88GyTyvP^9M9odja6n8!zUhZV< zL}{`J_A_{oJu`CjhTZW)T$k$km>y77bbCso)j3~2fy=v#NFU<9&G*{N~n1A`Ky7_ zUsTYXcLFqCOX)Y>7F^uG0Y zXZ~_i40c*%m_lZ+IBFhM?__^0*e6+WTri~Ga7obc$l~#ten_@Lb%hZ$hk)Vb*r9o` zh7xf9M-msn;XY^K)=Die^T^;kRNZTr@uKjW|_IIMK*Ob;6GqWOUNx z5S7v73uw>GY9qWLtMC_)i5c9=rd=m3#83b3use@;MgKf??(f8qC4TMGXM=7WO_&^6vmiLhJmW^zVgYwm1BbhGsY-cH^R#&VkuS%=usjKOAe4l(2IN7p#AGs(yiPF{0?l%pLcJ<_8;~3&>3Ae%iY_Kzu z`!dkNvI882yixnxD&3j|NEGDpCA$5p#;^G`s0J2rQ_-HW1U6u0bMp&#PO^&RpqIFo zKU#Eu;OBbM_2H>ou7xpU%>|ffbKhWTo|~!G!J>qu+@8Ihfmm~USEjGyl6(&kss8=a zU)PVEmt!O6>-XMs`N=naf_@`hECRCH^#CH>cC)tm1_$Ro?lPSK_;FC0!j|7wHKvd4 z5i2}?WVnZSulsZo>UA5we=Q~6X#52?W@tBfq!N#S#?@;Tr;c-(gXL$@slG|+z=Fxg zzP-@X@l0pXFV9KN~mcP zl6CP3@?+PZPt;Wc0Gxx`--*%6m0AA29W$BBH<1P za@+^k-aapFQ#$5RluRBe92sqTxRMYb24k-3@vI+w?1gC^VjMSl3ZbYI+ti@ZB7!Ex z15jAbM|3CjxtxwI3WK9`aXo>+Z+#n|(!FO*2>UEjvW8?rAKcr7xOGC95QkfRJJk*5 zpyf+YGh9@$Fn?fZqK$Y1-cE1>q&82%@XH@!namqpH-^*f$~7bUgzj|A7qDhz%Yx09 zO1UtUA#e#`_!`L>#+2>~y@jtF5$axaI1*+lfS%^sO~!r3je@RvHEDN2vGK}T+rZD5 zb;%W&3Ig>#v?bS*DM_E&%| za0di|@oKMn-cB9R_H5pdtGaiCwO1b;r*{nJ+hc_t(~%ZmUmDEc0&5*<0eau`fHSCP zZdZTfd)=&ZRHV)u&ChYOk3I8ai5Mx+o8s->RCk?-4&83A$ z_F(G|$Dz>3i(N1Ci?V~pUy+H4^#p~0E(Pesnq+UEQ_XJWKu7~U2o50)SxY-`rw zE0->u$%I6mfHc{;=muG9x2ZM!*QpHw6hJ(aKrZby%!&~q=2kWbqu!85w0ID?YTVMO zWJCy(yqrLzxvsO9!Bh8$tC}mMN!Rd6o9qWDdIOx6$r;_lM%PLSYf!Xv${pp@k?P z#3B9=5N-USTtPlGBSiay|Sb z8VTEkTillmO0Y85ri{a`6BE`XojEy03!Jj0e=4_Q5swU-bA+5nL$$MBs%EQ7ycF-g z;&&CGT8K$hg@3xU=O03+Rpp;pNU>C)i@Bp(SEtYgoLQ)JkQi;yKH)@t5y|k(f=5qy zr$Wod-x=$}i=0CulU4`dE96I{Y8SlYP|JQboCiFM$(&HTaz#D$JUEZmdkKUKSOWB^ z{B*O|?p+y+OOYn-ysr?1iRx2~_h8pTA;S$&$j1jM)$Yu@8C*oLaRO4+XkKD}f3LcN z%ExdnhU7K1LPAzw_=w>o=AYj$c7CI_+vpS}3suolzt>HaV^L4WK@TY=G%0|va`yzu zvfU}$W{?tGj#Ed+H(_G35<1+Pef&7q2B1WPNJ6i{vG&Ep1HrSrV>uQ;nk&HBupFw~`N(Z1)ypAD-r6b2x|8R{Y zjR|&QNbhtX1q3^3_BjbO8qdSur?<}~QWplNKk|uB2da$VgOU*>xsEk#R1ZhhG?43h^C>4CC2R2Unk$6FG(%PUn_vShJ%tE19mu9qxe7N3 zD;t6YE35Je%@Y>mfiMb`^<+^KQt9g#g90Y4UDf0+BMGD1@`h`YiXf6HC}|?&jdtJm z>sCemI03jD+RR$)eMIeCMtVvI6E}A5R7a~a8R_~v4KUAGOaPTI6C*%@jj9MYdK7r= zKkBPevw;#`j5RUxq!blw|Itpm#c>+EaoZrz#m-GED;SkJGslcHrw?s2V~y<9tv|z) zb^~A)t%bDkwqm%>%G}B7cf2`g>#oxz^*KV=4Unzy<+uq=L_)ieU_3UQ;b>=j)kwgr{9M&D z&u$fON${7H^Gv^JTp!N24PWye=5Q|JkQ&D=y|*WAa=H?q(*N+fNvS!vr5##uzv2m6 z8$d&Z5rh+km?P6!h!bYu5KKPd80+-&QM;&ci>+|ZGVJ6320?g|j zw<>H>@G>+mw0?EX6%^&Iu^9Y7cU1SXP5`l()X8C}2y(h)2<2N9k#ruEtS@28cYejd zGJ;^dUP4A|*&j)#Y$hN!tz?_1Fab z0~)Mi#)eJ*rM3f^wQCRkqaO1a=axiSvA$M7y4jDwBzqV!@qqN`8ns@^2g{4H+F4(d zMmh(C@Nw=&((P{#<$t%&lGgQxojXDeznrDlHCI_OQ%BJ3-m1CEy_hu#s;>t&o7CFSxX339*hn+E?cGUb6c_Vs0u?D|Yn^b6IbXV7aR3c8)i7&O{!n zrg0XDVqB}Fr-zd&&<1zvBo9SG$8w-YN@`atJyS@%yM%i*`lcQnyxQq%K*t4Dyv_Z( z^efQw`%{k^Dzl1_&-p>#W*12w>|x+P^oB&H6Qe6b@fxbt0mxn=j_3TyDt2Qi&=!x> z7~~#?(CAPz78Lb|ez2VT!}MTcJQtAklEbj&#gd^|#l%RX(e0}dtwA}Wdjr?b?aS& zs#spCP_@S5!Mb}e8T-E0LMv#9;haS|kcZ;IK6d4633OJ!E@+Pn`U?2i4IiYGt(twL zpv@!j&$sV`=V3GMMBt1p%$NKUUa=X!^Ic-P@*ptUjKUbw80;k2Yd8mQosmMsOTisr zwU@}AJNDwjq2fpLghI3CJE-XQV&pMaWMtE`dzjHeJ*@nVZjASLsXYS_%zZ!#0=V$p zwlxpWIdNR2X5v7M@_>gwXWIgTeEVMctPvhiVf*QIlzh}}%YezaB+NRtnVrF#df z>7SLq>s3uJoCW^9B<_1>a!E)j$QbMX&1TFw&5|kSRo)S~dso7Hnsit4zpV*$ml!00 zhnHl}$c(s5?7HJNm*Q$kx)zjL?SjSbp+{xH*GKDIm+gN4@|FJow*W<8Ytd)N5IZ-( zB^PQ0>~A?G-6``97baz_O;`7~_ed@7^CJE}NmBG~?dXCC?z7y3d!4Lq-6?UY7kT*_ zd+0fH4UZmjet~A>-%@R_3MII6|+`pH6VQ_XbF;-`(E|JRkzcBr{|XW7N6Js8I|*fE5ImG zZbD}4!vtYH^(afod+po=PyLnb_QBoi>sR*<3Y{`csnC|CS-Ag7^37XLR20$+ zKdD4XvAh!l! zB!b?Ql5p#Rtm($--LX6*V)ox>?Teh;_0@H+PVQ}VVI$m7-*9luBD}{Ykrdk%VWY&n zjf@cUJqRLPl=3!P!w6MK5@2TKk(2x_ud8_z3)|xLv)jS5{w!M-8%hLaP`SK!4D*j8t zl>DDohrwS^vHd~|`L7pbN|QO2zU1Z0w;#X#=6?9$1^@4Vb;4)RNQGzq;+n%xMZW*B zaWDV&)8zw^*`G+P|BEiz;q_k?p8?dP9={qlSA zAIE+2??3#|==fj1{`SiY6Oe_ac!v#h7a%NO2!@EeDO@xO0n)`Ev4`W#}0T>Oi37*Q2Pf(p{oWLQ-z-i zYNnH?jLhB@63D0;CCNSv^>0PSm*YHT^fG&gyXPrmv)}zWn}VE)G&Nw*NhI=A^9!Ov z0y994P27KyA##wJovMx~Wzhl!rn!pO{=h4`kxHxXqxU9nv&c&ru;HX2uuMwT=mNgL z7L+_MVqdRv5n0-yI+graop$4QA^10xgZ5tEsEX8hL>Z@{@p2j8BDef^r$ zt7{>srqXbwxPk;pOnyFED01g5@b9`fLSzASe5#L(k$&2?(bRqvLUJJ>eqC3k%hv9u z`eU`jsK{?k%p>+%ji4ILS`sP5tT1`OFWP`>bK>1;Q`NQ9xdG2BaoKW0*rZ*-H6u!> z1{@jIg#BDTF#akhfV^HZ{H3DMk^O|GmBKfKw`rGJUklFcayez`NRQ935LltW8gUZZ zK(ztmUvP&-5r>%N`lEs#s5%=$dM!8Qs!DHUtUD)lHA1~n^&nyMh_|^!&YJG_8TV1T z5~FNp=&fC}H1w5w-qK`bHA%oH5PntvKO2Q`d3Zz?i)1XP*Xuba0<^17O@i(|SgGsv zB3{QRW@PwjbmZu7vT7(R)_~(ebf@|CkX% z1Y%_gUBRmEX#qErm}ZkPz!1iFNgJ*Y%)mxS56CcO7tW$< z<7DG+1;?|hzw+K)c+UDK^ z{p7d&g6!T!?{=>kPs} zEE|94NGz3{PEoOq!Y~}qpcM#d_Jf6#T9O(C={Lna+K)H@moZjlUjSPN42ME=7WjZn zxa6(+_qoi)J?ZfH9X)C663JkUmiN5N$Cl3%$LC9}IXpKx!Bp8LOiNj#WhX?*L{h-P z2(5-^aod5hd}%Bnp2VLkH+O7!rYd@Brit_jSPs|lOMSt1|8bbG!f1-%!kSLL?_1g) za>p|SK*U;dh2(aG5yTAIVvQl@AH}nMksKG@Sq(c9ipJ9AW(sp(H>PWrLKOH{h1YrU zE~pC2NENJKv)#grkMocPeXd{kx{-6$v(rKIP;1j{T zsQ*;}#zB@r%4hx&f37ypdvYs;GnbMXZ4&D@*=5isjthDi1OX8ej>zl%wio^Ux)7>6B=GSZ<#c^qrtNol?#go z1%5-1f5A;GE904xG48!jCHrjIdHIk{hqVl@;GpzsWIm(0?Df*v#rREQK7NHZPzpmMB>%?=v;2I!4io8H?g1sm8 zuG;gLwO1AW1Fasc{L|8177*hyMS*`6JSsr-cO~mdhRCh0!+Beg2;^u!s4(1$oG%Uk zo_ijp85GS;U)f_avN!5p-Px{cN{;V)vT{Kj3S;yI5g88Esk=ko96HpgDB?=V;B@$W z`I?551`?~^-reU7i21g`Efo7qdvbM+;315aaLqU&J8Nxy?dpupOTBe;GAtOi@5r+A z87_lu!J6~Jmrs$*<) z2!37-$4aoIrQ>I)kIdqRK#K90#7&NomYM2z7q3L`)30BCmffd!DkHjG08a1cMTZ(W z`KA~EmR2cd-A71c)(p{*81Es|rPSLrF>jsd2rAOA;Mix9_c}sv1|A+#69VY{mV!NF zg$9E9RJm-)DY63u-1>-I1_mMCSEtUI0JLrY<;#}^qiZ$&!0M6~A(rXB{gb7ep)a;C zj7Bk`CLU4`#+TG%E-q4km%!Pr85Sd|zfKC4qNj{1M?O$*Nyhq~UZ5>1tc=cwz%4e%KlN&6te9;T9 zzU)htu6s!X(Iu_ASxq=|>8Uoru8#LIL+|di$fzrYUT?HdTX>_tNvD&sh|%L6S6|{g z+ye(>bpVzF=~7vQSOUj%$_-4D!$T&VHQu=rm(>=@2U<*pN?--b)yJ*|a1<~f@pF$Z zr{Z1^@)LkVa;z{?M8szbkR^Tm+>nsNB*;p0JMQfE@X<=vCF(cm{)gc`#a=~&?{Lmx zJ>=HO&#!B`CQ}j}(W65^t;rZAhpRTDr7s&^I~@6HYU$*x$x($t-7efEE&>d*wFFto zyKEKVUUO>Gx1>BQ(u<6>4OKyVL;GYrVQKO*PNc#jAnfduU9a*Td6Xprjacq73LYv* z%Z;JF9Nj5o&%d~evrBVgd{GH7-=MOP1k49#urP{P#mj^PSE`Y#znb|LXqo> zWjM#R?=sx9y2-gNh|$$i(Vx3lewh;=<~=rz1b5NB^EY(!M5eKI8WdDo?OQ(Dw0!ER z?KwOWq2nXBn7vYh58<=ts>>yB=bww_nibZy`uskBC52tHDP_kqp7!R2u`h zqI$O+WIAX|0^aSjbIvVZC1z(i!kR^WEUKzmLqF7Ij|0cV_ddpo<1FfWktN*s=J@Oq zk+EuijN*{aT-uRa6e$uzFzXEzRv1Q7hQRtIYfcZvtAtP#DYzMA?R(4wK}zVCgt_-@ zkU%wkcwx>QT)=NiN3TWln&j=X(c5uKgN|= zW#N=GdP7gPEWFa>`5WyeQ8^Fm6MBl|t*ieeFF#!M+$p)0k?GMR!B7Zhf~Q=QEUJL?uS2EqMn? zF`f>Wm2(;5lDsBCXb?-ul2bIdQsE{o z*X7Zn7z6NO2VlsX%P;7&eU&A2d4r@yRmp)T&!Nu6j-2Mv!}+jVc}kjn z@>qS@mi9Qb$W~;pzZtjMJ5_4zYxV396&6U9(l$ zLNXVk@|n+B(IoYKLsB8k?Q`@#lez^z{t23z&v&?LEAce?pZGGwLGct?0;|GbD)PN@ zRK;uhoXd1)23h%*6*MR*IDi*P|3UgtuqoG*f%Zy@6XjOg5TcNe&+m?$=WYc(Xs`RJ!1iP?TkalAm*{v?C5&5OTrb@?>BJ zhD@C3{Mc^gSB+WMkd89qMkbNd9co&QO!+S;gtuW$0T}EmVQ$lDM3%S3o+C(CiG<;J zHZc4rOyu5g?po!aI+anQ_ETrD+6di%{KmG#7Bhltp!f&(WwrS@`K_KREe8USKfhc4 zmAqF0mgi^t3C!cflDK}Il$yPg)-(0o3FMngtsR!N&s*-;u+Yqc0~d|rY*=z-&6B57 zzc<3BTjY$%HHiLGL%0b32yUH3`esjRy2tv!?VrVmqgYOM7t{1}=!#V@fZHO;wx|0f zANfDvAD?7OU)-sNM*lgh93%PU1wPGH_D{OW{@c08?lZKpR9iqLFn=RL$XtE1R_yrn zGdE_KW+>8EoL3o8*|&o|7S;P$Wn67xzhoZO^Yu-Z!E7 zKv`O-u1@#LDR?qKM*M-(H}Xre#1H!4z^eXlVNuVuRGwf-ub#qU3wkx>WtQ_cWH(eP z*3M+1rM#N@DwgwfWAq<1ME|>5eN(JrbDD|)#u)G%3t$L==w~zLfWjT$k_}adW7L!G zJ4Yl`fb#dCGqNHvH~27@qJkO9W&uC~zbUsS$ZZ9+WVW&HesN>LPBUiz>VS}3%u1A) zGy^Ht8-8;E17AT6uX^T#UO{cHh*(Q%Nsi5CG#BRBte|f<^|PqW6F6&03IBV2Pny#w z?dR-7qUp4r?+~>043FbR0H%b9nYX#%_ADaqkxWxp}fkosO2FBBL znO9j$Wwdk2ClFla#-2iSb078^qD%a{i|F~QStsQ+2+5?7+D!k^VX0#W(p6W7+yspGDui;*)8BuY>oJ#>I= z-RG_9Ne^VBgwqL>Ou_LOozG!r!*oN>M@0k^JICp#&m|hx8k!J(23=svAaDi{D|vhL z9Fw;sFF1n07TdNn*fO7T0y&>Z$xp>IFpaxL5VQ00D(6rx(V*Gtz-Q0^*2JoW2m|QE z>3#;?Rhr}^s++5ltEevFuu}*gxVJ`9c62y(Cn6u33olGKkW6WB;Wo@%aqXxEqLknW@6#_2l?f?x|N%?(K{Br`Wy zvXG365x_oy9GO!&TFHu56`-uHK|7y9e|CW8F`b3(Z^o_l4~X5J1AO1>FD$v`i|8x} z@-Ct?_bn|VGWR7dB7%#J3=jJ3*`#LQ;bjY2$QT|HTlHSxhUt$9EF}SFB~^)$^p+mO z_1J<~wc&uI^K=H0U)2m&pT$-s180h}chU2^L$Qi;f_n7RyqSh-c}*@aZ=ME69Jf7= zV2B3)F+=_K?yB2h8EX&zYQX0m{9OU%pJD2(TMU#obisu?BmX<&HA17@a*=cI+6p`~ z?-!=22%7D9E3AbpdeV)9w9tI9kyDfJBxx$GEMzau?GxZzd;Nj(Z0lJ>l410V%axJ) zbQz7;M2NZp)UFCCcD~n<<^;fjv&2oc$wn;JmR8bTrai4BdaX^ZXqz|ao~D4H$l$d1 z&7E65dyNYwJ--c58fzW?*nlSOD0A1MUNR zWRj1fO49A=_~b>#P~VP>HV4w3!naUFW2WKUI+_Pm@Uh3b&&xL$6<2p*19I2*tFOLq zK*G9N%S9U|tVslK3l`v;g{?G%W{tnqCXEh0m(kA}e5p!1i^yzwdw29%N^;i%m)_;& zYsBzM8XwYHn$$E|0r!md)X3Y6XdmH|$vKD4U)&|M@DGV|Tr%`isjDKVcMV`>?}6nw zuC!L+4`%F>_3#!n@IYMk{mSu+^j&ywyu zX;iC|i@9{Q$dB3K<4|N?d%`bP%ur(OGjeAp9Xq#cU_FxmxMF^aO8}pdKlT-K)ccpm z{l3LV)mHqUK4EoIO~aju!;+G})3@o39r)R=7!X>Z0EVs@#7VamZXPpn7-|0j^)?F0 ze$~;_N~K($PQ{VGGax#Tq3h(jp|q!T5(Ams& z`!qT2+~G{@A@2NS+KZ$jbBC3j+auWs>^jqKagxfK`Zhij!WB}4PPeI~A2j`zf%=&_ zHR#5I)xx)qIk5eyA90tDv_h|##VZA`Epm?a^g5C44(RHf?CEOl8B|aD5gn{aU#`96 z(&dz;jg24qW2A|VHeA!r=iD5qHQ18HgjKU=?#NE-g3U-)hj&@Zj(&bh+=)5lCHn4t zZsGCP=MTrIc$_};o~Lw&&~h^B_Ea4${hgHNY$+X%N`N2aCs^!Ktw(!FIx;$Avz5V@ zXJR+-8+0&7?9tWLDJrjRmD)h3Na~t<=VIwN-=#E#qifwm;vB6FEEpXW!D2dL4$(`} z1~}Y(6V#Wk*0V@lQd|`qUV1979@>|n>z8B~*Nj+JGKjFqNWW^66C9yijGg*zpo2c< zjnBYaGdN!oW_;XBmRmo%z!>%KYbiJ$+;)44p*?FveI1*Q*>)EafAoyfEHo8fLxI`M zj(cqqQ{5@rwR7us*BHYztd)Bs`31+1V#@*fHmw;@KjWm!tt`7JqRLsFmSaksOgi9_ ze8?_{x*yyIb5|7uE!aqv;L_jqclYMlb^;Fv=ccPc3nW#fz~?C`15mXfI*I8cI4Lcs z)26D`6j(jBZ%kE0nCNJ$JuOY=*7#A!aZgI6Wx_oPS!EWp{hvTPM7L;$93=Km%;&MU zd$>?*tQ)q<`KG#w;Juy1QGzy!aEi8D!=b49A1a%Tz9?-0gnNYF+n(hFH?N<^+C z$87GhilTalhEJD)`|WXOcNc3?v-negbdj2eMA{&!DPuYm_x$wTM<+7W;rM0l8#s#Z z!n>T$VE=5~TPsd>R!wLE{Fq2E)zlr89N;dZ=X#p|{A1Hd(Way3xTt`%iwV2=!8cM%q+Ms^2 zr*v#gIqSW(0A4Y>kp0MGMy+#&gDe_U^DU#0lCR!QYJObpHq#JdNfT@5@OGam96R1zD%P!hS;FWB4)O4;cTe1D|?^R~*>5;76997P_ zYk4~Xu=nBAh*@v6NcV~y7~y;=Gc3uFu9is|??Bfj&)1tq>~U3t&q=dzxGSSvlznK% zONLYS=5e9vpy^zjEV)RxZ1mgOeI%D+(djf5{24vgB9dRjzD{GoD-`(1szb*&?)T;` z(@-B8Sqn&5nr^~R^ctu`UGs{9OM6xZz?7w#7Af+0w&6N~gjzp&I z1KItr=L#Lu;*aUm!yzkV1B3BcM8_WM8!{DY*=!!&8cWtdZHLPNbA1L#|Em(1jgKTNKCtW%#r5`)5w& z!nICjRgw`HVDN(%e$qa9pUSQfl&oW(6)7xjM2me!4I;{a zI>-~GgSxk+Zv`3mS((;V&P7oj37?P4zAD5pT(w3C3e!y=V93M*U1fI$M<2Kzg5!C4 zC^}pbSc@T`w0t5>CO``CDB3Q2BPEnh71f@;ArFPK+-Opm)3Kl#0*fMS{K zMdm^Vbq<*n=4EFZaFHUMn>=%$GCw5QLA}toR3;_IOnh`hW-i(2^cf7;78%BglLUrs zCuxzXrb#|}Hw+0eG(D1t`N7Vy$RtHZp(LL&6D#}hH(aKsv?C@dE)n7Gc`?k=+iE&K zmyA^Smak65NnCVa+B0!X;EGn7f?DJkIwvyr^71Kj$+rZtel7l4ZSkqjj4i&{dnpx{3-bLPQn zyI0o9YNgYUI3QP(dp%?+Y)mk$Y1$>Y`gcD>aFJxZ8Yw?q0S%+12mlc{YuN)^*k2u26>lTC+~!3kPM2nkTWh)7^y)d+mw(zVghsi#*5Sa} z-j{5yHiu|*d8*eQ^$48_sWnvLf#7r^6c|aOZIfIgZ`_w{ACOf^f?Z)nbYYNoObdd` z%-Gp8AMHLb*`YzdvpsK3^{Se5Gm1Cww8yN%0ZU8{p>xUT;^%!??to(df*Ogo;HUY zM^qB)Ps+o&8rB|7r;W%_&Uu_(;S1MMKLMM~8}2Eo?k)Dz&|l6zwYg5pxijG1bKPE_ zc4P~Xy^b1c-C)UWHufWs!br2>+FoL^>F$zU_Qd#s6Rsw1Q&mfEL`W~NT1vC(eHdH* zyhkjPQy}@C7WhbJ%+g<7Ay?I_Tiy%FI0O{5DK~L10z8eTdcQ8`SaCRMRu|O=8|&|| zhsv>9YagfhgoSI6LGyd8<`t?Yr#X)#{H$2LD&+<$ZR)hyOkBe&O*QaDBYbZ*95qhAZ#l3FZ#C$w%o&& z2s%EmC+;xB*%Reyv&*8!Xt13uNE|zne0@OqG3m(;7FV6PsFutwqKjtdS5JpFF(V-% zb#9K?7q@lB-(>Q-Z0)E1f>iC3EExlHr)u96|&2|=akS=HQn?dtwFm^ zEM7ssE6L<)^J6~xq)3M^5%!P#*FImOm@ED~T7gWWmt!i37{}7J`0e{zQ<$<2B2F^K4YGp0n4EH8` zf*i0GK;#*opH}Ie6Sv`H#Pkuo5Lu{pXbP|(v@diQP90eg*3|Gh^sQr}RvHYLjYR24 z_ogFC7MP)rCLw$ji0F8L?DW0D6m;7gJ+kPAub;qgAcC$g5i)$U?^bNF*?Qx<2dp^9 zH;{pMB#snnH1bg6KTUTgo1>^M!isyo?iy^^CY!$XmUo4zq~TE z#ag3GLnKqVS~cMGDn32bqv(k+giz}VrcdB(Z~`{Gy?8Y0b0EokFp zYIbmKK~~6AM{}YfMFv-NpSMlYpz~x(uYOZX?R$k&bAxEvE4;ZJtz|Idt)+*2TNEg6 zUF5PpR)YR)yQO-$sA{p?Qr0N8dJKI+5e8ednVZn@{yyXH;$Xvj!vjpLuka?T(lo93 zg*Wujnz&gz+sg&X-(r&m0>&vUjU9>tq(wL9y0P{`B0jO3(N@uXK8?euN1RzSi>|zo z1cHgP1#y@KMW&kxj{|g9>47ubBpM|UCrb$qZb$q0}$Dstp!qNn@Yqo4WW@Q^2f$m+*tT+U-Y@LQEi4{cogohv|8 zeM%K3Hv;rKCAHgXPfl6cF>p}Ffhk{|CDLWMD_vRSuQB?@R?|RqC=Q3MXYSo~Tcps5 zn+0`&_(!i3k|fMn0oJB7Kz0IOcsyumz^e4A0N9L75Gg8}ec9%92jUfU6b`X4j7tqW zv$bJaH!IECo3zU+O4+fUj2Q7|YYf%ZM6b!eGr9qXZQ185c^1kf+b;{R>MPQiL7!DT!Su*a=Y7?L+{VqG zUZngFL0UCw#Logo?WWtUiWhQr@V5Nr-&i1ApDLT>{$fR+BY?p;&8~GDuM(-5SS22C z${0Z6QUUz_Wkc0V$ss?hCk3Mjrrqj|uhL+lZPg;ywe49(9f{{QTij-4te>*yPu7U2 zm$vCqjCIa3Na}h!RReATW<6KHN_vOcaiEGLBv)m<=ToF(?UyFOahQ^>Cm7NII9fg% zX8JG;#^gNfsjJ5x%XgC#N^n|+jFvM{ObqDD-p1ODFc;Clo$Wajrw7!Pz#y5?bc?rGwtBc4>IH@onB;MMeXhFjDxtkImudC3k4$_A=Bq?%1bOmdm}Vv& zD%aWK(^kd5j8h)le}+}sYQvA?j7p3^n(ULYh}g?J?s6*PS!7M#7`sg*6kCs z2<6UCxQ92E%zcy&mK_NL@;Nm92D+&0cGQz?tO551WGy35$w%3bPv7PD2F`#+c00F5 zzA(){klkKp+botr+O)@Qu+w;$T~#ZLLx0^c*EgYUEnJx~c%5i7t}|erJ$A4a2kmWN zpPlkTo^VLEYkEk}B4dfKzT)OdbXe9%AD5<*T7QPrZ@VUY0=1Av)nu278}2pQ!~FWQ z34jsHrePj1MB_l~jT0A=vT{9retW{k$i&2O;fcraJP~ty{tW7zt66}ij>;J}2g@hW zHCFDu;?C+= z*2Wa_cm30|q#?>9yM}$)0@I5`5O~yRr85h$B9Oim842w2UYcS+^siVN;*&^&QAgu7;Nx8Tmkf4}zk z$)8E5$Wy=Nk+(*NLqX}}gSbo3#*STZ9NJ*`YFBmW@-@->cAt=B14htAY1MJ301%CJ zwI$MGCh1X|75Q!G?DO+>$2m$l^C(0wB)VB#yo=OVjJYlcGK+ zEi0-zE=4dE+|b3KY>pxH zuUNx7nGR)O9ss&Znl+n`awi+jtgT~_{8<%QyY`)7?B1D%?h{{sSuyNK+9Z`$4lvzgZcuW5@uXI|F&MeEq)FH0S1-pOGX<*~PuPo}SZk+^sY~S@ z0vE$8;B`DL&LcBXThz|9*(O~;-UM~JX&2S#+y07-H&R`C^Elm~o}Qv(nmNV+jes)z z3{nh=0L=t|IJ3OJo0hqrp239a?@G- z&s&~^_be@{w-qC(mh|{^QKxAust@%GknbZreWaB()J_gnAABd|tG+CXx8i|-tN_S1 z)U#IKrH_9xgg2YK{@}zHRyqPTpt_B;Ii2dl*+rG0xsBzB zZ%Nyv6}N12);~(2`<_M>&~v+Azkl~@?P|fUeDZ3eC#*?{W5sZbDcaM5VrE;p7n~^$ zjaq7&Z2A+43a%nD9V=@HiZNV~xbs$v9ogRNin9o6=y|Kvlrf{eDWN0Uh)PP6LpPZ; zE?(P|Qj@XmsKw$aq#e(Nw@U<{_md~UhBo7xa6xg^w4<2F(+G0|4oO595EgV^gdmHA zks#0AB@vAt2fHaPh5b~nhg$@DJ?hoN{a*8S>kE6mxrRv_F^G*RqL4)Ot{i=>+Xx;& zWSb?r^ItaxFN7XOf8|!7O;+>e8EN*u>FM^?EP`8)+Hk)u|7)>QIDrkuaXh}-+9_}| zVK;RHtXaXC6bwMAmLmtfDMb}*Fn%Lcq)(wI4_jaF$MZ-E9m6*Et)}2VZ>tf2u}z7a zvfO{p&!ZQ&@n)ldW$g3lnVw`Y5Y zz$dJY8KmhG;KxN$Z%$Saz|)UXx~#YPGLe2|wub?V0zg#)bCBGeCM4Px`f?$2uP zR*wn;UXz_jnsx?;-j(Gt#g%*h93tad{=IYOuH{~hG)ygS@uI5g74tW|Iax@dGuLb; z?9DE%*#F68X8FErvn{BZzo<=Kg&!R<4vtB3!Sao}Nr<0=LNr-}u!9g=`1tRWf)9zd zPiQ#BAl-;QE+11=TWz;eCDAecu%oYypD#bM0drJ|`f)_ z**0tB3{4ieV#o^>sZ7<{e*aeq0d#G3JU-1xk-m@W>qI=sFM6fl(G2Ze`_uD(>_5y1 z5-I?8mKK~YYgfFiOVYfW5aOOaW^!HVDJ5%|D32UE*Pz)QvjK61w50C1=t(6jo-0?h z$m@}!OC;Ji9~NW~f8mOic+PStXx7}auYJ#m&-rdR!vcY2!)w-`RH`z>WFB?gSwm%yK_Q1 z&tY-o{MOP=lFqvIju`}S`e$;$%i*gmlC@r?6~27Lx-&sQ4v<^ zfD+oxIYr}ASIk)qq%$cX7e0bE(sCSj=YnRs22Dx0~gKA2V z@V6yJOq##Gl0~cR$|G|y`+iP3$2sUr6A?Vfo({=Z%A7o82{f#NMhjg*BR{+b6_d4% zwBu0CY7;6_je3Wof_~+rkFS;M$hZ~cbvCBW0A0C7-frPwz>UNO!}C>q)sSJe&C*0Q zY%(jXE0fxtt9HB94D-U!t%#kLAwsa&xhGi(fwgsV8TuPYPX+xgxLUy9#O;XbmAK^W zO;NYU^6ZZrR7ftqb1z2}Wau#qD9 zdqj~q*RO1`A|36-n)j-+6X)AIO(tQWJg(nw6SWPTgD-sqk)p6WS55wRRciMngjKlP z^JG%A_U!u~en zsX9t_g2b8)9UvWPyRwKTSKi6m+&c_-GB{U!xmb@kUmU~6`KIr4^)aQJ)fN3glGT&As3lr&c0^Yt%N~DEa&G2I z69YWoXeW^6ghkB^Sr|UGQo1_f8VCvo;w)wRY^r5dDR7Zsjc!O&aA70scP*cV0Rp!_z+M0Q#%s?rvcvAL_wCh&)+$wi$|V*5bw$E<_3I6hUc6Viy^-po4Uduo zpibCR!f}jEciUvA^p(ue!@?%3&l&2(yn@!pWT67_r`AS119;0!r|Aqb#*?5(aN$go zq-k-h>)SR&ojJTjA1FF^esFbH(9Jb|Mh~V<+6B7ot%2(z|7_WRlP1T*#la!ygY+ae zqSxw)uv4SLpGiiUcFrtyljiwZUP6Qtyn===#Rw6)yCQ2WFY!!8<@_fhv< z#Fhbpl^k1d3pn$(3sS_=<<&IycSHx+irei!JhGsRVxj^rqcGhP^jk=fY#iheaJb!< z?B-52EF2d5LwEQUtc3;l^X}44T(Hd99-^$Gy&)%;MooY>NC}JTLN#aZg2q^!R4Kit z#3RQVm{)z+l{DJlX+-xjM@yQVRNzI428mY8;zO!N@VwUgk7drJ_YA7^{7qu0@SBs@{Uizd!>F1_)hT#pszTRQ)3J=yT4i@$<*R%BW zA!RvS7Y2UKKA<&fPmHVLE1W%jW$C5m?4d##yWpJ|me6)fkdRfvt?TKlsv;tsWzR(- z-^i0)NOh2b8d-tkz4#VB>t3?e<~}=Ntmf8IP!bFDu?(M8E8AZO;M(9RLnB=UA}Es}F^+zLuffSo)4(4A(2$^zM2M%vxtD22~A_5#v3l^W|? zD+A747%|bLs7}5#Cb~8Jhk#0*X6`@)x!-bC=ZjSkbxQk1v7mrc$>;q8{D*;FBqLRv z5i>!&CejoHr@o%;w%bAp?^uZg-vZ`ZR^}r$*<7z!Lql1(JPF~*tfGeQ3?$Hwbba4| zO!i$HAfRd>N_LjJzX5@=;-x?wi=nSvt6+RJW}R5r29*50V{QA;E=(918JP^a#yfx` zUp@9nmY}V8b)g%Ox~S$RH~sg z>&w3Bd6Ovy2;i=E6oxNLQ?76=h=w05SaB@)tH_jB@rTjz8yNmBQ~i+M>Ohwrxxwfq{I1R!EaHZVu;`q)WH|pD0KAA`s153U!BQ}(@{X>Fcsfs`O8bI^+WF&` zT)RyFJX!IDU*_OKELx|1C>SPk;ZGeh;T&rd!02~kF#wK7kvyX#3q5vpv+8AE*oJOb3KJL3XP#7-Z#;Z%xjr6++7oh1D1p6{hr9^Nkk39B zk7<0Tn{2ip^)4yR-WHva`@hKRJP}*$JTbJ{s?SNeJEC%3ZwL@q{bwa07KzMvbF})oHD2t@(eN+$9{7wQ3*VQ*BYT!njg!E$w%d zpfLJ{-BU6;QLM>tk`AuS-Zd+d^?&W1`Y^T#b6bP%ehO974-Ii95{Uw7nx_Rf>U3}l z?obMACw_aVxD|k3f?GQWIPQ3PTvpR7zudDXlW>tP6490gd!oxkE52CXyF8;P8e@3S z-uxBIE=ps(8C#v}bqV{9kL4NURX?RUAx7at`H8bM@Y(c_v}h-JlUN4s7Oq; z$?)j550jX)kS3OP44x^8+ji*J;(OH{kJhirXo#;9M^1rb%&rxjG>)4j>+B@s-aX^Y z;mAFXY!?e!^GAamG8MRp6S=O*)-D=FMH(T~v^$HR2S##T#hP;AG6UuE5CsLAkH5%O0*Vom!zyfrP-`n_4^Cu zSRcZQ8ZKhIoZQW2TYz%{)U+yY-Wt%bzT`!T)QD71@uIIuTi|{3yQq<8uF;5` zwUh5;sDSYu9iQU_mm=6qQdmGx0RiBqCcUy%zI9aC9$0ic@%S2%|UBo*7YxDiwk zOS^PgyJ{6aT+YmgcSjGN&w_Vq%`t}yc9YT?8sR3JiT%uL%%g&jBltzf3#;^S~J(=41Q>oSOG@O-TxNdI>F65|2$K+=`r8UCG z+F%N2;M>p1-OBNM3|dW?c`{(ZDzQFfGZvgmhHPS!__L72r&4auP3EBqmhY>QgH}?o z?%PM+VhiXa&(P#h!4dVFt0`Q$p%c#EtcqD3Qg4mk&n#dj8T_M1*h6IZRa)zPnw24P zi)0vIVj*cJsB&26VX z%_6ROQdmZKJ}!xi=x<1l1v_dIZ1qMoo7oukAt_-L;=j`q^h6ozi4%(j$Ak6s%TsCF zQFV&`Ly|8fmtJ8F4OCb_p$0KW72ph6^)OHGA^Ag5!5uF5fivHW4^Coxf7D;> z;n14K%(=#hUI2p6X-uB;12{8DzvU*S$UUT~F{o%$>qgS=Lt$oAZLL!aI9zMDbMVlj zfFCJZF;b}{L~x;#wa8gX+9w0?&4~KOYc{(jcW$@XK1JS3w+qQciVGrsHO}f=NiP6E zWXD3&vRIbu7uOE|E^VHQdrll2 z5T%B9mPrg*bB9}UB{9JIl3&A=|BL#+xtQ0p18 z)xdL4_L3!tD#JS31?a-TJ2&A9ci$UaXd zOPa^SvpyKm^6>=QraQ1gq;;lKP|;`6i-~xrtE@_wxe+la*mfq2CCizbzNAn}J&rYA zzI+Sij)|l!v|G(V{`Hm+0eBEBs>%I>H=@CPNJskzMMt~oG2W&)<942gukS!q!URoWx3GTaT6SsUOQnb2BVU9<`iVBWH zv}UIGg&>!?&4%tF2Pk{Grr_-$O--Za-c*piIoZ*{%2aDy;hsKji%97^^RLZ?y*rje5mF~ zuW)5KwKxst7rvO$%_N&Te@&!Rov)+&3rSY%s420h-^~D&S=8qQeNG|U4M~Mov!}0L z)(>wBA%+6ihfwRSW-$%;)#|Um_yZv({byBV?e6RD%P-N1Tvkz-a1u}l@++v%IfP!P z5qe!k;PqQ{dtbTGl}sH5*V|_eiJxz~Z~yt*ViLrfOhrv4`MWy$$}y!H+NR`X+g;lz zLweGU`SKU44O~s^EEI^@R}Pgp!u3%|{cfmjVnAQ6ryP{OPbbyu;;SE-?(csNR#aqf z+6yl)j};2Lu3|assMZWOp@|oQYND;E zAodrxT?7DKimZ}M3NkFo(ElPol@;GE#)x!Rey#M(tzXSP0Re#_qGf3G2U;Klt^qRY< z@b(UYy*45d_AlUCyT~>Zm4Q2bqE9LFuCaT}E{v{4xrIO%=P((<9)1UaRZcs2fKL|k zr-20`febF#xbE+4Lqpxh=Ay(XCMWDGkxs5}{KPN0>)dAb6;h7M_6_VM;%icDm#Ftz zYE!uegVV=n&A$Git6d6!;Z<_oqaTOdy>1`Hi2RPs8q5+uCJd#>=-e%n&v%SvNU>Cs zzG)X5^iJP3LpaV!L;2m=ojUM7vpMOUMQo;H@#RTF7g_Qa@OwtPAx#=bLulI7q+vhJ zm~&v-VEmO9g}U4fQm{)-RmGQ#Td-Rvpe92*&L#83?*fFrAwjumLA}YZXWvw>SsX}2 z=t6Xjs60yC5ygk4Z!TVs?tIi04S!}?n4Yr5SUX7(5Kr!RHJLP7X*0X4z0zlj?XjS5 zLz2k6;c#5^VDv3B#&Pbnt!+wL>?IqFn9kEb(a*<6xXZyvvb| z-M2|&8Kw`v7ENL7N?R-6ggHyJvp8|uCGuD9|Jwdb@Qxz37ayfsC=!2UP5;qKZwxEL zDV&OZrL1yQIZ1C+!u`=n6vIes+j@{1Rxw(S*65>&FT2vO9c$O~gO9dR+zpA9?V|c1 z`;&Mx&jEwiGFDuj9o!})xNbqA;sw+F=yNbQsGJXVu({jer=gA99x?FbHzYG2e}5kn>W(KEt1OCDtZ%8$-HCQ3SURDCFXJaP-QBWc{DYt zp@Y6B#jvb0j44==Rj3`fF}Zm4(Vu6xl)GC!+kV@*cVZD*U@Ylj5UvX+8hu0FB-a3t zk#kiUx`Dl^?@$#VFm;f(U8e%$~iTl!7%xpG%<$r|a9B$NS=& ztxZ|omV0GHu3Mpad>GBPlQZ)V&`e_mcG&eB5oHXip}AK3ENZrt^!ib5Yt$+hHr^G^ zN*f5V6DLo%D(cnlKgRNaqu~O6Szl? zc3evP@EC$x#7M5D`mwplSUI&;M|-}!KLaDP@-yY^SXxTX2hlKqjx_0AA$KV?WP z61slEYD1*D@0gn1nCm{81S>Yoy?fZ%);VK-JBSA$A%Z6+yEpJNEfl5Ip7$wbY{-&Z zZ!z?A1TCXmt7ledA?2z&a#IknY&7^o0VWzsf-Ufn6a{UbYtBm>*7F(@N;)EFB`LMg0)D z8;{SCUfJylSwT^HsYm4&FoIwo1l++JE&L|5Xw&}{8SjV%qs%x;X3hre?t>xb&g$sY zCKmzVs@nPvY7VeH@f|z4oLvQJj$_R^8Kx|1YQOPROuI;&i7vJi{aPoyaBF z!x)3Uaqdx|aTb9D-4Ie@c|-vRS+WEf88*1=x%-`Tx(iXj&5<1alT)6cbFqF7;i;~I zPMUV_W3|st#sVtP)s9U?27m`2NdDeN|Bb;%HP=xqV#%_hMEUQ+EX} zx=M4*OKm1&4GM-$vDU#?u`ZBBg3zO!8RIiROW7K>tK{JFtLDdB*^wrd;_ZzTObw(g zRno58aC%^J*YOP!ykuj)B$N^ri@Y}*EZIuOf8v>>Q|m;G?(9ly)+ z&}~zcXXK?s_?M>@+~V<3g%-U2?(4IULIV}$D%XPNiGCAy zmPC36>XA4ia=2|~&ukESWI&O>rPmyr9~y<&cR9mwL;(6hM9f4Fd&ES&790RKBNZp; zVPctU&yW~VR;Ga0TC{)AAAWrK`t`S8UN}=R{jZlVFJcJbzc?pI{!?pD}c8 z!hC%elrpkocWDR^#5?dF2mtWG`e9FQf}^HX#?dp3WD&{-@DBP+)_@_#A$eXZ0e$py zAX|3psDnyNzzBz~*)|2JD*rPLTbM(AVj2PR%H?QB-BROsB zXkt6CPO^NV+dS17yi4q@3RXM(PpQXLs|hd?fWUr@;CpsHB%0|rnjerf zPnfRa2FSs7hV5-!(eO1pS4h?rl?Yjl+S)?<1p3?oGJ95iHgZTc?6jLO zim)4L%F;Fcqy3!VVse!{okbmY=#=;j#h31-qnB86kgzX7s=yeL4_FSQ2l3;xlQhVg zlI3Fta;t74p?(Q3MOnYhSm3zR2a><#j6V^`&TP|rS!krBpkqXH2OW%BF4Q`s$w>+8 kbVeQXd48%)?w}NPCvWE)DOas)x^1kgKbV6qpsoV|02s*iGXMYp diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/restart-process.log.gz deleted file mode 100644 index b1adb8a3ef80fd0229ee09294505ebe679278c79..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 404 zcmV;F0c-vriwFP!00002|E-cuZ`&{ohVT9rJT?d0an~##J#_;ArcJEBg_{N19aJW3)T_#66suM z)B89j=jUp}Cbe=-$ndnsIO#SfX;p3-_bS^v7A2HavIz>7PA#$B)d$%tK z7mP32a@~^=O<(0!zx(tt2u0r}Zvy_K)vr>2*vxAG!{e&67{CV?96zCjNBv59ghX+f z@z)$;5;wUYXup9N8c}X|1uj4{!Ft0>H+;?^%y@J2Wk}y=lD1cHBd$|;Ikzg>BXf(l zSP@^z6{uuU=<;GvTkP_vXZkCx5;t#mdpqp!Zg-om>9(t)pckNKK`Q10Y$nrrY&lPp zp;w8+AtAA2LHl||T5m|WJvn)bSz7O&QL}`j7V0WE-Mgx)ARdd0xOQYO2TfFJnhZ1A ydsxF@bc{>n#t`WePG~qUakHIwSyb49`rPZ=%a52RT)o+GQTzlW(YEvk0{{TqXuw$j diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz b/docs/evaluation/pr749-final-sds-2026-09-28/serving.log.gz deleted file mode 100644 index 1445bf431be238efb94e606a1b7b4759ba77f5f5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 542 zcmV+(0^$81iwFP!00002|D9AzkJB&^zV}!7xO+&_7ph=S3le7@M})jiJV~rLw%0Rm zRDM0!GK@(mnt+me-gW(N>Ygb4%a$a+_?KGa0MztSgXNiGswd;LpBDPe ze#K+P;3I*S&4{v%z$C6Yagh=3L68rU{RM9v-3{Wo5}b#`#V3Z&-mehW9M3s3hI^$^ zawQO04;aER19+!ORIY=+s9>*hZ$_^{YR`$zlWh7qdxq*uLtm3}lrOQU1f)MH$NYnPffIUr9>e#of zM`CBTQ`G(XU55?|<6#m=&4cAQ$vNlVbL;@XnQ2;8E4={jcpLe3iw;(4oW&{ZFTHPF zkt7S{mt7SL(!{-57PU;`J7aIXMRbY9 ziaU(Phk4X+rIRO6&uBuSrSV8lt?}lSM3#G6eunu1W8{9-(Y({bE2ASpM73G$s?)t( zcZ*&uYgGjm2mhhs=~t(fG#< zanU_cBC+ZgG`8E}i{|N3HK~Tf(J-6v>M$EHW;gg51auY#Ahe?B$%;bbUAaKAmWPw5 zv>0r6rLe}ia)s7~)t0Yo!)3{3D9LDu;}|ZcpT3-bWNgk=-C1-6V-a0%fbP6Bori$N zm|t4t682sVy9fJ|{BU@b4xr^)k8P6( z!o2{LOw*xX0QDU1XfMI+>f7~A`C)qY`Q~bRc7Br00QE}IHf&IPSaGX(MOmTWj#Q39 zwFV9$gWht1Y`eE1E3a+=yZ4^}yV@5U2V+R7A@-eO#~4~`Y|#g9gZDm^XfZSu8yY({gSn!8_`J49@dZ1 zBL?oV6#=vTuFSL`!+)qWfp_(Q-MDU6?H`Y?^BEk-P#91`BXMrPd3z^?aG8vCku*I~ bULgwKW88!k$UsrBvIxlwz)jjb1pxp6ILk}F diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-data-plane.log.gz deleted file mode 100644 index fc9448e37e18360c571637f30f3d1c3b7c6181a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20271 zcmV($K;yq3iwFP!00002|JA)olj}&5CV07>t_}i@~F&4wdnoq^X&jn(oWDZ{Pm6=G>N5Ss%X@d7o!(mDldu z>-V32{Pg|XfBVPz1AqO;|NOuI=l}744zBz7`!3(VxVreqPj5~4(s$1Je{^n_d-wN$ z|MH)IKj&rrk8z#<_}e%7zyH0gOMi03xBt~U-~X>~ZPy&i%6z zTt$TDYWgEC=dJ8d*~wMyVE_ue&yQ{yoV;!6mJJLoSJ4&ay$i5XEc@0qwCQ}mAI?LS z_f5x0TeI?+_I4=gMw`BB6+62R`bu`96O6Yp1%`KBNk4pXAM!({P=zK>tjc$;^4Uk} zszT^$*X4WX*RFnOI(lze^c8n5317s<&VN+d9|J8>Wcb!vNaG)Td2We#^Qt8>9?tZe z{Wcd-6#m80FB0Cy_=nfL8k{@>hz(!m{sI$oeJGE3*=^qCXP*sVwb6riAAXZV zrZa*yae&) zx328nZ@($4veDF&%i zHTxXMSdKp6DXWN-GiBOEo1M!Vpn%fl3zOc+%2g0tBF%hX5_2xPhRA{*GDskURkEJY zKw-2bAtWavBaYQGYKVQ2X^}Nm5q?f;dD~7@#_k{PKz}+h$pKdc_3h+{1c}gTbwAK0 zVY`32u2F#mYFv*?Xk)4rG)+&+e~y#Y!FYL&$j_kQukvh2J;{$`GM>R1e0i)%7%7Jm zhTV%OSA-TK16DvL)=E4A>Y7T{oDB@#5q&^3A{#s54S1FcHSLFPe?3z7XTK7uQ>gPq z&_H>mXPYo{UDtGglQqO2*RQ{Cx^8HDddkOq|G~nlD@gTq^kdj>O;;XEy2p$y`o%~S zM5#T37e-6Bpn2IMa79-|msl6g8zGdG1d`-4k7uyxh-JO+*)C+`Pp)3SA3mm?`jrsM zx*htoONuG2TMo}^?eYhFp=(J@(<)8Z`w2?)6!HnIQFO$nX|woSqp+4gE+-&*2QA63 zToesf@P_u-(c|#^67j2j-sZcqBAss;l19IT@d~0-&c8grcZVx@?RVEM(HNt0b@) zhinNc7S*~Y+2V15ywJul=*re9wxUgtnfF5ERf5*YLLrkUXRCxR%{;UN>sL?|r?^+9 z@H!QdWfP2&Qr;M8@Wh@QcywsLJ$<+l7Hir8TMD$Xb|V;s;Es-T!1H~0TdSdzHK~5Y zvKiXOH1`-;)2%7kw7BkEWL#`(i?ImP&J~MY7hV1)zB~EE!@v%fX^{U%;EBZd)t6ZUVrIQF!;?W>lyW$K*o;UUNjmgb>;5TE^IoI z0R_uv6%41`XBvyknm5SYaRp(2$q(1C`{onN4PAyAr$JN;8N(W1*scne@v zlcDCY7PBH+rOl75dm(D;I+IYJ9-9w?f6FPW<|xjmu6cEJeqkflWZf2wn~(T&m48IK z=QNOrWO`D7@Ct@1C}5c%y4Mnjhts61CE%EI)YrZDaMeHbW!>L_IkEQpp@U3UE1M}6 zN;))(vd`q;uq4@!EAG4ggC5Eyl>`ljF z%3FX(@qLSK8BB$JRk|lS^z{21$=SUFaXWhc`x6M|^s(zyph34b*5>zlog8*kS07oC zAK7F5t|Y@7O621YM$gC-L^iA6-%hD&Wlx(JLm}(-Kxi2*4KOzCIL5(m2d|NSniq(; zmsPfFU|}#@pf3$%GCm8^J=?;uYIX#OtdkeEy_E|$xQw?`N&HvT?~Z>Do;A99(%iKY z(+M#^M-RJ#jy*Z;nb-27)z4hX^Onrj!g$%9li7wBa+Aw&NTz#`| zD&@d_C3ImrcVOR>*b`xmuVJ(YSf{xIhV?_olrJJf)D*}OVgU>2PIRaWfwczGp2Kal zqn)$3;$>*upYnEPmzY$@s85!oPy@-y!EGM$%7FNaIM(w zKEg?y1uXmWi3v4r|7Up<6!TqP{mOz=2 zVGosm0XVkQL9PeH*YOw)XTLYQ2x-40dg?8wuJKcWHRL(xO8($iQPqeY-#WJY6-#s_ z>tFrd`xpC)Wq!K*H_zxb3$v@(<@bX)E%bqNJ8;q{V2gQcsh(nSegrU_KMCbMFC;z+ zw+V;i{rY90WgEhutUu?P6$AhJ{!hu9Ag(KWf+ll>dfT}o6-zh9XZP{DztR|IVg|TY z_aTh4SZRC9isw$<8FV?FCCxlTc|H%!Xpo&_!EZqnlFK`@W6^oz5jTQu?8M% z@o$p-x_{w#UIbRM)@VnjuD+F|d2mPou@4W-`gJ5P&O*^*wA3eWNniC%Bw96#-4KAU zJcH@FHfwfrZBeFmmD3X?9l3U$o}0ak#uwByj{LrNoS6$y8PX?+6_=bo4^;$CilX!{ zjLdtuT=uN|TOQ8NN#w-zJ<~+5+O$=;JUf=8fT1FR)*r7dobMw2A7oc{Qp6+vaQXRd zW!Pj$BExiH@hi{=IeXWUO&m6|YX?C{pYgNaJAfsdrKyj42`}#4$WYR zx_R-6D8edz0=d+p-&&gILV8$*$`fcA`m^?o%v=Y?b)YNMfL8lLXntg@&+A zj~6q+cps5yz5tGdiP>o801B5y!KngErCevnT`_!?cp~dgjn~8-Qj?m)XUXl)ZMxTg z)9a4{jdTLsI#R!KVb8fI8o6<2+&K**aJ|OQI?&EJVq>A~_YAF$wGZYAbe#P4k`_32 zxYwjlq#v};rO#)u^Wuzx?;`V6^Ja+)bnzCX%y#ZX@QFMm7GN<<&Bi)ZB-x(9S0*F1 z!rc}37S|eT2uNk=C6h$%U2^OusNkQS?|{5{MQ3E^6mDN}*a2TAAm{eN(VCxk%Cq)O zHJod|!4~uZDyBUf7I$2UgzWSWc_4Gi zqsl`jpk%zsO(7Mw>z7}J*j-w`lbOY)t|Pq{Hg)eS94mAhaUPH!T8Kg;@tXy!$Jrf^ z;-y!~0jsSVHan{F;PxMTWSnY<9KU#WvBp7y|KyCcP7Rlgs^=wO1+ng$^R=y{_A8-V;NY)-shNxW+&|Zg(!l>hIuZ}eCa|PY2!E+6 zcglyfxz5)R(XFNBZ$AWO2i?kn{LJ0T-4?|=Q}tHF|9`1L>~fgs@i zuH;|rsf`jJ92-9zfdA?va0ex)(Kr|foB*+cJ(Z;ff0rJg$heEZf3U+j^s zC#)&@PYwDafFga!B$BrArn+;BjEe#VTOZKIO3&7g=8EjOrWQPvrMc_2+s?ft29(e; z$_NlbFRxzd6`#)D%5--&7_zylwARohM^StYTyb7V3vm&u>EUkVK>rJz=aa1Zvp=>l zF23fJ_dsrZz|A~@f2B9@zlJC9FYp5X|IY&`4+_;_Y!B7VorcI8pFEUmb6-H@zhEn0 zxq}2=!bu)Nu3(C@0!3~_fAL=g*l>Q^t04)@CtBM=HFdgwQriQglR>y}`bK%qmpEkq z3z+x+B~1Id7Rm+|^y)1vwwzZ(US=_;0h>^vSi6&j7V>K7+pwIkl&zba=6|7CKEwJo zkFA(o+^(Gi5j^24`q_qi*xW6E0yNnU2B4r+Kxx7(kf{igWxkL#GNqNXu z9(GAKQ!Bw{33kLPT86cYil<}>YdDW)+B)+7mU=> z9I%&e^Zx6%63N|dS=j`HkeXdh=$f$z{K7PWqFCnG);b|=D*gokS9#K>08Q>jUjuZB zi+K^eK$>KSjH5#apAMYAYZR~SqHl{8$i5Z*=?xe@m-{&GM4x-CpeU)>h| zWfmY$Vf{Mj8xdQgRMrWQLM^&!;a9*#4*|r8 z(6&laX0xHrU-Po!kN{ghX?Kt^Q*#2ikfO;?B`^aGqef+i^D;K)K&~wK+Ws{&LSIbsGUBY3f0NnE&g}e%Fg;wcClOT$NGq45eKD9mEq)ORj zfXP**rAOV0fVS4lR_*~U7e~IoFuJ{R59^h~)f-UeT+!N?Au&Jhwtffni|*?Ha^L|F zcvj<*T#6|hNTi@n0hLmhnF23Wy-BE&x(A7De>Wmg55y-8voyxxI~YgYRC6_(&*6{j zTPIh+Gk3Sp;R)M-zU;>v=ff-rDJ_IFH^8$H%v{I+LNGpV_Voqe$ebxLIYhLYvCGzN zleA6X&ki;`hqKU?3xjS-h?Xrs46;hFwv4hzk4h3N$FZ6$!P3%a37Tl1zwjV}5lVvP zNm$4Tq(-NjWI4>@@=WwJ_oYmL5#=TXI4K|meR7llfhn)Tapa+Tty@q;Mr803NW+>a23M%UjcqT%Sm?dg<9H- z4J=h-48nAsW5o?V(LOdz6s$>39HF=R9PDsn*4#4Ta=!77}6uyP>0FxwhD+HcE!N;MIH=4fH zm+hxhfxB6gAa}2&OlO|Q-X4LlDc3@1*J(Zy&+!T1gke}KMc=ITQz7Z@;&VCvti>Pm z_Z7prFiObF*i#)|LptkbN0==uwh$3Ea_0WJ-5IF~qf^eJc-hV!n$G6!;Pu@lrY*)R z^SmW#Ifqrt2|3+EkN)PqtY0GGQ4=d$vVnO>na*S&tqzS#U6@?*Y*n3fv>Pz*=6ZNw zF|_Olwt+0G8>zaw;Nf37uhg0^D%GsnD{+pRxDj383VKl5D&x7iGD@Rw_&-Cgc-|5@Lr}q5fQ_Zs&>Pi$59XN5>ZgcYJrcd^d zJeg^&c|Tdi%XkwWSdnV_uubw}?}W|uP|-bct1>43W|pZHDS32mUA-U%w_)Jux}|h( z9{<$_SBWlt0up3cD2pxkzITr2KDYUP!eBT*?27!D`TdFbQ#@zM^jb+7=Ak$xPj|8v zm;!aEnxvc-VLZtM2bdH+(*vpL2i@hIJ^joyGGY&qS(r(NJ+S!KkHAdHuE6A!#V6JI zKQ>2Y|8Zyfk4^*Anc6&er?;*h%K?*=sY)LFb1@Yy+hgK&zN2@ui#KLN^h%upSS-YG zd;0L)2aa=n-pYb55~ClZQ8LZJa>dD}aZBM;d0q3vFOTrLNtqa-l1vy(3TGpbZ*ZdC z_7Ib5!?FaxZ8a5pFxc?RT)~sQnf>6kjB~8U(U*SV5rH18d2@qR6LiN~2ZU95+p=K+ ztb?+a!W8kT^$DaB6IvlvFx3<+W)S8Oy(BMbc|5Wb(3hSzut+=)pw^a`k(8^K4z4f# zl3;PoighE42#bsqygol-fhHw@VvNT zdx{HY){6Q%lCIegXA*xkhDME3jMqS54wvyTPh#hpHl6l<9d?Z|OzZ9O*-GKS@uNt& zAXC$t1@$vd&h!ZuQhATTx+K@R_H;cVJzU$+u|P8=yVIYDK0P#ZG84r^yDz)_055^o zt<^jg6QAFk3U==pJc9Igf+^;aJDtg9(@S${-E|a40;cJ~br8nv%>_j|M}^M|Qgooq zK`$V&0{@rHwJy4*Rm*PX+UD4h8k*8uTJIY~hpN3*1GR*wDq{+;a>4xImY(Ac_@Ng= z7Ew5o%I2KM8nS{ePOG^vs;YT|&PzgtX@G~T`&g}+^uT@`h~A4zWZ{_8Rf5)^;JXt$`ZknGgDc_Yd?PI=o;*AL-^6Bs)Cp=@F% zJJTlkM6Q8E#)vjkjs~%+dKas9)3oJFAUH^ALrU;^qX@=`GK|kBOFYsgbI;j@yNRoIC46p*A1tp7!NnDXbSru6rjUFFj5iYo_W6yaHT1hm?F8pjM zb&X*X{1zCrrf^F!$kZFXa~)7V+4p#tqQba!zL{upwLG^)_>@cD+05e=H$#^T?{5_M z3gyIwP|%7E2E@tO+R~#&Xo9EY&Vnol(R7EEo+T>~*1cPMB#@ab0lx-1S}_Cd2hI`E zen5-v0p078-Rg)fRU~-)aX+Dp>od%$;M>u2UbeN1#fVuizmSWS{0nl$gi{yH9&lBE z49zkr6FkL=zxg`ReREMXIB2L#mbsn1A1njX=+ zdtY4xQ~rY4h%TV-c=IU9c{p>Wbd*D4P6g^C4v<-w&2T(KBYU~ zpE#ONui%W=KsxtkCk?UdnWbc(#i=!*fM7ubPrf;#A7`knMl3fokABff%7lZW@5hKS z-dIVxOqeLX@h)VD{iu+@&f$n%6v;fZt{z(RHC>tkqn6=7x6yB^`j zyvW-)N+)vaOpENAzNbgay-@hP(T_D~mlpHrBA;_|WdY6Cbx6{YG@8og2X+>SnaAEC zS+K~Do1&wZ{7A{3vh;+3r6j7n|FEW@^7}2fa^^_5S%aj#*Pin`O5q5(C;ah*B59sO z)&esJ;XVDT+mcu!5&E6%CKE|2eH{E=a3>(LS=Re%0Qz8WjM0GQ8|j=>%frED=n0B< z*xa#db|4)lWlk@nO<_-migt*G10fmafoR@fQ*h27QQD}uiFy;~H9ZFHFRpGSp3VY> z!UIFraN6gDRS%W4$~u&;D&W&ZD0)N@{rB%4Vggyn{&j0a)kMayc1ipU>L>=}uuBPh zaxeDoK~~cSy8;_jFF{C}#WY8DjpxPhq}SKt0}5P190@7z@ZJ{pWJd`e#yzq*b#{Hu zHq1z2a2$BaT@hS5&bYyu64O1}E#)1DCfIXvMf_wN3{^w$U(phSV=bXjg-QNPc5{}- z_fV=e^=4mb`h?`*07v4y4#D*=+Zg?g;6;}=#`i@snt=YA9jRh-CabaZ90am`*`1Ev zaH8%e1eryQX7vc?KbPhi7*6gFex*q2}i31QFz^exqS&7S%Y7!aIEzB_#e)2Bz? zZSExyo_3QK5$U@8lw!EL|QUUTGx5O4*o4vzX!# z?kYJ6GZ@Pw1?VSWt!2ma?9>^;t}3MPhUDNjujlT7ap{Mh9yGFI>!M_VE?<#ETH`#@ zNf&83TsloSD6;VvJfpK~n)H(90YOSqA;;V6M(RW^re(?WgqvZ|E6`K8wrW1uR(*rn zR^KZo53!T4!UHFKDsW2)Aj_^~mmT9J)%nF0IbJzZp-u40XHoPnl=im!Wbnlruzt#> zbehkJ>tAJ_18@_ywq^@^t4Bfy43-PgL(XN4;$xEPcrFAvHI<1>i|#KZ5U#RhPRJfv zwVf;!cH#~YWih4>oOAzTnFDZ7;ll$rwzxC3ar!pKj2|nSwpSAJW8}{}q%?3cFzc%o zJ`&8aS)`N;vn|yP9dJ~V+icA6FideS5fj~EwYVQRy;WiB7jC+lJE=5N;&n8T4pn~4 zn!|zkvt#!`s7c7iZ>N125;SMcFghR?CfLFGq=ARuvjdkMmdC|tKMTIqz!v0*h(*NO zt84vwfYMB!aU2iC;Z%_crsdW8&5so(x zn;E+T{`lwuv1tYOO?VxeCj_bN4SCK(n`|Fs@HIu^4LMLN0dK+8zlE4jIT}= zb-8m(Jr0guX_Di|lDl-b!Jz{oxdX{LzHnhf@cXG)9}6xMQvobz`YT8Ht+m{pWY(Eb zO&x=vyl(Ll6htw|gDjOHSk?h11;8j@QBtdqM@?IIH^}AhZUIzU-W?jG4}h1KwU%sg z+A{mt3J`LV=3pTQbf%W=B57)p{kf+|VdI%x2nlewW+mKJz&aU-H*-uiTB4>DKvX(3xK9$Fyz zCjg3CfJN9%0w)Bcjg^ZN5DyS9GPm)wNlvB7Jn@t7=m~Vvbu?-UUy+opk{l3SgvO(> z(^L955mnsIUtS0#bu2_y51VbS0LbbG&Y{9kekKY?AY=V3RH2XYKLneS9sZ(qpq!Q4S1R*`$SIGf zUDJ(OCO#GHqpJsQqB3v-MW}5PS$B;I^zD3cTc~B~J&v^VsFxv;9DJ;jN_(e=&x-RS zr6A51S*HCzTpmdSOxs{02)dLOk71#c3QXNZ5ov=GF~(GIv;$ye4J9=+&BlzGkt;m~ zk5M^_YSn?F=+^ZB;sgepC<^|Rw~q53^hv;U!s8@0ZRez&+a7I*lHFxHR&~?4y7Ha7 zIB-@I`L#oLzK21bVawN^^}DeEO(01SAdP!&2=M8&6E#3 z7zXpkN4_ng-1yE_!9fBaxCR7#!7XgW4~l;H=Y%r%3S8gxh!>`1xVKnkzha=}=yOCk zxb?5^{RT&5J%PPGa#+Mv47ifGGA+BB!xi<8;@}$x)OH^7Tw9xc?nGR&{!UsFSAtiBmdG;i zQbUnqj4^DP`#UYzXWYe2i?9=lb{wV!21osl{`;nLSc0=i0{Lbaaw6lRIfAN!#7zrJ zo`_6cv6@M)s+0jM52xv@hsXD&Ip(IpaHU|M5Kxcd*mV6c(j>+poeZA_7-qr&X<>aC zg&10HB%^28=PFBoOXk|iU=NMhEZ)oe*Kr!XXL}l zQ5c8~aAU2Nm0P5C=@&lN5!Qm|Q9FBcSi>qyh~fxlIE*|CDfNsdmC5HmHCRgyQSg@| zyEY)I2=rSAFG(lZrbi@6La=u~kfv9=G>5fGsLTw44w8)1a6c`bf;-xLIv%Xm&Ip%i zMeyl3Iov5I%l)Lfpk=3uaymPAZg?J>q$xCaKAX^JW8pRgC^vX&q&Wk3-|vTR zgF~AZi#Qoh_R47tdENL6V>mrvW7c9a`H|1WEnOl8zWol`lFMn(@f-ytY%XsnG0+KO z=-7!Won;Zthu4tK+||!h3YMMOM5P1|V>G9;$0QNtN?u6+I^lm|#DnZ&Lh|xbX|nsc zCq|You4>f91Zxao!5`BAP~%0GKIQ(Tu6CV@A%E9DZ3VI9h$2tt76(>0)IOQHAu7N1 zFcJc|4WA@A8!Jnu%}8p1m>SSZO>`w@FcJN6UJmLVL4y1b4(y+reY(>%I>yRQd1qSf z?SS|MXJcn_F8`bEG}7JCZbqg~@`dUDyp^?22kkErK)3vS?wh$|v>#Vj^&9oCK!nJ*N{RJza>Q8h@#7@r5SazTArT<%# zkoXqIo_cn<{XP1R&65rn5@u+P%5^!*kax?iak8`0ux_Zuv8b!?Fk{j~s`(97>8O zU&N;XZSB}I%kf)oRqmSs=`Jp99{Se#FGI5r`ig9br1-ZG?R@l`_*`YhIPyf1~j|Fgy-=kUs zRG!v^;{_yJlG|GCr@G{(oG_e?OMLlM+`c5jbd>CT(7}N>hg`D-OWQHdBQDNtU%9-V z7$BW=m*$#Arz|=LCZ9temiDuc-MFR+1gJ`A#f}?+4qd+g7?ooE176~RlP-}bF*8kK z5%%IT`S>h19ANq3ko6=@giw;LUt|vn_viSd>od1*b&{j>g1F90ebI007St&ZYGI1~ zBXt83gI%}!m;#25uq{X7IM^tScR7)STZE(nW1;rGX|oq8R>XzyXIvPQ4A9vsZHk5% zk|YKKds4xUKGC%!3QTsg_@XSU?oXHfxJtj72O*-wv;!ejYDnEW%vFu0D<NgaH3vT<(TkV?p)V@tOcXFDi*ilHuLOg4Sv>@^l2@8OSA?@HH{3C zTD2yei=|(^igY_Vkos4BWT5Zy2}x01CrTk&A+3QpasUCy+zvT&-+~ z=K19y;X(#jrQL)LguC1X*Gur`I^%zD`3H|(2x+PicuDsn;k#(Yg3gcO%5tMWk+9gv zfq>oS+ppckP)jZi9P!ut2?Abg8A(&ukqk4PLf+bM>OfVITc7+U@+W6X(&CLS~c?4&JE-fsy`(AiXwAxKIR?`&GUUK9w8jRr&I2fR&50G&vG{NULL znL3===F>?ucROA!1r-l!Mmufo8hPuwT8y-sC;Q@mUdw!Z&oVQ78HmwGe8EbXecFtt zo%AN+$c!Zsi3suA0hK0~$cJle>Lu3jRqju2L%5oSWA$5Ur4k&sp|K+zLo%*Cv|k^K z+2C4TEFvBs^?>P>l5pYSKrZ6)t)!4T$4aQyC8Ip}eulUa6r5ulZ@^l5M)bS@BF}k{ zVCWS@X)BK~NR@MAzteEeMc)BUZG1Ymcep1LR}6Ct!#iz|*8@hGgEHb9t6KKuqIdHc?!~D^c2=qaxUw;9H#5Q*VH$6dI=>3Zelea7alqbUr63N zH4;g85)XA|p~NC^J9wsUUhsfBa z9HcvW@lnfs>%L4T-0lpoF<#HpcfDz8`dT-G3RhqDulIM%Sm$(xp_PdIo%*koaAj1H zl*O~_GF+?-wO#}MN-5R9gfW!^8G$-pp%eqd=F)D5<e<4L#55qh(rgHdam3ZiY22xfU9^OzZuv#tAG_ZT zLtQVkZbQUHd(#EqO`pGr7O{66n3BMHIzUSHY-o`9p*KxD3Qd=62PqH3>VY=VvZ=Xy zInY9pdIO5Dr_%e-EF{o)Ry6FS=TgLSHzYZj)sC9lv+yx}jxTuDXkkH=*A}vg&~PKo0-M?!&~LKtth)O!)M!UqN{1ynBh}>|n$Y)3W$*deVk`M23045|XkR z9%pD`oV046uY03iH?-_c_a%4UtK5~d@3wi$EC^gNBCAW~F5>Flhk5kH9lhcN;kE56 z!Z;6k>fanOnlSw5I{kgw@|tlbRi)hRw2TG}t>-K;AYS$Xq%?d?M6nG5rWX=PXx;+R zfKR`^7TR2>0DU*41ay?d#?PQ`H~ig^v$;77-Y#o^mE(rv$wXZc?FgFV-%lV5Grc{_Jk;3LPhtc zEfe=LKoL*oudn3DDp1PE#&PebeDItyyfmsDiU@Q`ZfRiNHqyyDR-2-Ou7FYQUq(%Z zrF${36}4LHk3{&zrKt2>`7&dA4i19I0Sc(MtsoYIh=xSE_g<9ABGmairlkdtg5!px zAP++8)+1PZ$&Se{X^tk^UgxZst0oin)FeF3b;=Ir9I1tm& zEsc(*9S)VWV+vRg9uu#@xnkmC)4<$m+(C{Wjm-O-tA8n6*EaMAN!8A~D_`C?2h~!w zbpjD?LmD2y>0hUUdVytkz`!F*Tz|{Uo?U#00gd_YP7~SSxOLNL#~f*`n((mFk~T7P z6Rmq5B#%)(CcD{n;&8a_OqB|Vxv^iRjUpNA=XB>8ldg->1E`T6KMZcGo$OfNp&wNo z`WXozHWq9rn}NkVY@R7YpKw7386r~K6K!LyjD&@y9o#i8A!UQv<@Zfao@V5UetzBy zJF=ELRh0TD$sW8WC5DZO@{#cuA5N-t47pOIKrQ;tunQbId?p%IO|a6R#jD~u+-P0ZjHN>nO3<#?F+83 z*@fm#R>_Q;eZxrLp1Zp&y<1hg!KC5M?Lri^bFb2s0Ceol=-T7P43aHk3Q7TUzT8@IIuEj_ufwBb>P={H}qFlqSLr`dQkHDiq9fZ_PbtaiB~3 zxPj)a)=W5SbP?>A*ienvh-}(qV{5TziS~)A1GG8mdSXiH*vIEkXNX@@x2C4_li`jevPUdK^fqkK3q*z<5OtGM0z4M1l7v{Cf zq6*#zSyyDiZm_nq#Hn+*{!;AU-p<9`Fa4>?x@4#EBxans7euzHKLjHr7J|_vn1Xl> z$TU<#Ymc*$y=sLXI!o-Ak`>qMSex22r$}ntSN=wN!TIcplJ2YeNCBfp1pt~ZuqNl; zd1Cu9I%Pq80)26O!W3xn;&c2bYNw_W@sOiahNyEXffgONEm4OtM~WZBWS+B8anJf; zhd;;a)``UG#X$IRh&mj*>{u2a%!-#WR$gvq_1(Qo)mFzBi4Z9kj3Zgtyx8omunwSz_Io5-Y+t_!j1N-8Kbq(Vm*Veq*8V+oi?n+$; z+y^vz9c6J-2MCb#m8DPWDw8uZZ|~3Go4zs{Nx@l%dBoa`l(@yQ)~K{)6>Dmou!^C! zd4P*mq)Of`!h@#@pNk>j(4t4DjVlpFkUZ(h*Zu2>#Bq3-lfyMYuN!URxO?KQMj;R{tS&XIz)s!emZx zK=-cw;E`q?<>ShGPXM597>N+2q8KY9tGE#sKWCsc0i-~8to85vxoH%SS0;lJ_iz==nIR3-fznkb_PmGCYtT*j7?&c0Gu?Yh_r$g%c z=A^}5;l{e>Gehy?;C3FTD4`B9N9bXEtOde<)^EQJGhX!4WAJ!FT4Wf zpU#b*-{t0d<4|=+^F|yk=2#I)z3Q>uUB81h@15#jMEG5ToRYBV-B3sYm1fJ?H^XZ% z#T9}6g--29*1P5!x`>Xs`msbZ#LSf5YfN70jj9`egegqwPcp6Gz8bJdljF+A@=r87 zMSMhUx8Prv8a$O_MEBxYINmh_7Qwdr=V2iiYmyHAmnu(>zq^d1Q>=1wWqvJ4|DWf1(3q*J=faw zbNak~3UoXi7%KN@2_!hLS=19Ssufv-X=@9>RYM5@=?G9krQMQDLb1fwG7*iS?J^HG z9jDg_hfwHwx6k$`c9JCoD2DL}5xv1{0aHYf(?Hbkjr^CxG4}qrG;@3r_a^u(68qxG zMmW(Vd+cO+7R}x_4aYYmr8+p1VO0b;-G@jlPZrq4b-?XQwW>5rw`9LFV}2&p@t^cGj>1YR*8-D^sgW* zVwkqQXSmf=eA2RBqAa!g1x^&aVz#u*3=Wy{~G^y%)tzl8IpM1G@WS;C_be zceQv~cl+`aD8Yj8I23v{6m$^`9v7}d|N0K26GC+4MzOw^Kmwab1M1;fG~H!vgj0Tq z-OI=-E$dJd&1)flE$o|k>b~K2>)~0ge)eyu@!+-*2Aqs{R130cMRZfUH_i8C{-7f` z*Mik#PEFDi8OYB+1Qu2QS3N(j0bGlAjb4wty2$sI`pg~sLpzND6A$e0f8|BV^A#j6 z*Ex{fWJwY&PFH@mbb5KqQEMp5-cCb#hXXyC})i|{ye3WCkBl%s@5Vvf`88+=K-x6=v;>rtM zI?Yad5yFE~a}pH?MIsKZRHnkH(!AHX1at`&yf@0p4;_&u!AEin#l2@%RN=Tna*l`i z;;JFSMl*ne+>(MEMqo$}14}*hu3s5{2qngmuRG-FbEKO2yxs9K676BvH81cFkHN0au~TCoIWyGxUwxS2HyOAxN{I-9@jaZCX(L1}ZXJE3 z&dV^nPNs(?*~5K^9M+b0c=(NW;Y1@`JP76!@|&vMvEL`fUW!759O)0%Y#c&2>Ubji zNP>J6{72Bj{2Xq+D-nwgz-)Y|moJ6ZzR_}4u;7QC<>+C?wZ?qpXS}6((r*D+07Uv) z#W_&WBJ`7a?Z&4mq|mg$eYt2t-cHr1@*hmTfQX~m-WqjVTV zroHR*o2wyQ#{QW+S+xqq2n(<5=KmaxnmFK;YxKq=W?Dmg>6?ETHzw4%{G({GSZ#3-8Y;=Z>2{G z$a_Kn^g?a`%t$JG3c#vpzk(dC8K=IuKp&P0Kpdx+oX~_YWfJlB4}dFDzW|95>*B}JVTLX#fXDw z{wsj`FC&&qsL-9i1R!%Md4%vU%!TO-f0YL_KyN!kiDgohBsg|3X{)g2%9(HOrO^$v z41AQ0cLQGBTDyi5#MgmoS~$U=Pf3c%*Z|Eubt8mD0HRUZHy}9dwsQT@7bLjYNOI+6 z?1!^zs^AvPwcA?zOpd`J1>eh<@+?#_ngWsrARb+9A!lt=rR~a9giobM5ZIPA0EDI9==Q+4QyNOE`m_MTx=Mrq;M7fpl#$F2UP-#98YZCg!wadTB zRGBK@vzJg(-TLN?Gd}aB>*`3vx~g2e6m4k6!LgJ;BWcz><^D9j;eDdr-uXhBg}o!J zHWglob(BxF@LbfMUG7O4fN_-MsvfQveW0dh$`z*>K+jg3f7~%;r3k4w8n6#(3o^S} z|B8rVKAF`&U!K*$vFQ`uK;I2}R(-S>b4su2S@2>Y!RIQc(qkq92L0OZgqt053S9`o zO8LT}g-#^jLclwhE|`6)Jy`k$0e}O+VvRi8I1vnp$ zgj>rrd-fwgR~NIxI7Zr+&wL_tK!q`oCo|CsBx|2o-yvf(9VD^G-ceOtLJb~vV@sfk z)c+6;N3H~2Tr9ZrY%GzE#5(`9WSqG`G`cGZDxn1JcqB7K6DNzL0Lx=7W5o=I8^Y3Y zu>phrPVU`o7%^EJ-9;S_N9zzWn^cks#k@;EN>W@A<)1w5ovvVd(ocIIWtzNmwvM+P z3xuCok<~)?L-ovo^lV`rE2BM6h=-o0qU7O{;*lhE!e0hXG$7#K376+Q1q`zB3!agq zeq*$n^eGm$_e^HQFz0DMTviNBAlZg%;uq5J&L>;?qJxCW{#yvaouE0IbiA_T7ZRs4 zj@dr{eE;$N`}dHSLE;`wDp68=8eY26pKACOrJu{&DoBzg;7?syzeHm$IfY4E6tw+m z6$rAnOX&EY0C_8ml|aUnqzT~od!7Xts(<*OMT6O2z4M8-(Kl_zV=8x`sSF>g4SZ2( zy&PJcko61GOGGdyQqx{?>21JN;{QN`;5ujuZ1ejZo10ec337`|-mU8mon}X0Kd)cj z4npr7jy-{Pahk<2cudT5JV8M%Z%71Sv*JnSn@4rM>zZ_jJ(~k18 z3c$q~;79=!C`rGH6c-JO`ACz_bxkzS?q8vEWV+F83cqSxz z`xQu}a#Gx5>8{}ytj&YExRt1%)llI{J%Z@G^t!owgQGi^BA;B{$5M#NH4AKiC4zh1 zG{R=dL9XnZ_5<&TJ0dR#^L$>opgw%W+N3W>NU@f83T>A=g1A}Z3Y-LRoFxZ}R1O`l5eArWXhQ4zxK>$rc8{9bc{lVZ3X*=TCG4m1GR?^J zdxQfZ96G4)60B1T>UwIkEe%~5w~V5-7@RP%Tt;@j@{F8FMp-U1T(d#b+(n ze$aK*(oOOvjq}-$E%Ru?KBT{=Oo?PX->b|EyU30M*oRrHgd~wX#l5VANlQx|XRET)Om)&ymCIkp{!Dj=)=XN5tHFs?et4P7C{0|BO~agtXOcJGQn@gnasdd5 z2u?*@?PEU7z-29Dv+EF6mCf*X=qq*)vE71PNM6@43g zSRlYZyodyp@VU=^>2>xgG2rqAkwfD2C>UF?EQcPo0+T5L20^ZxW-_oifHh zUzE3T`80i00B}21_{j=(7e`K7MYhrBzqbF9mP^q?j;~UcYl+vglKKo+50;hXw4;lC zrEW^qmPqAMz)x1^+D&cTAvrUnUOoOvyicdGU9V*a`PS`y5)j-LNPEnWC3RL%r;g1_ zNF)q_2`=AkE~8wmJY9Pk@BR9=YaV?U?U}U94Wo5A2z2YUn!7@f4td{|@2i#~vL5y1 zc`Hc~XM89dg}h>&QN-W2{F}FuAxZwZVxh+@T%bu$9#_QWk&q{5$>Gxy)rhD;bEwM=`~ zvdWK0-0~lybW(F97iG8hBnZ4}yJQ{UV_gQuJ|4HBh6RAEUCLw#W{4J<>(gC`HQCi$@DrF5sT}bt&&IjWpIaR zDR>U;E)5u$bxZ4#s3(x9kE_93&El4@Keb7(4BcQ2(|cg1hsw>>Y%cnSv7+1AM=KYW zf$1EyTHZ^O^^K7v9#Y^M+G7}O>j%qUTjw)bM@{Xmr+wPzukHE)2Slhtz}Tr*7Ax4m z=)91FkjNE`!D%H_##ZMlo)9{POP6&x7c1_OBBO6#7B|)oBTsFf0Po$pQ!akz7p>tw zR3^(CDbX?SA)bLQX3mQs(SR6PLx^1+bSU|s1zEH!oY;t19xDoz<@vuCK+ zjQcYsCK{T|_rtRa#@Uw#X(qCU+1zxak)DfkJxhCxe-45fDPPOQTml=6Gk!?O zAG%ggnm<`yWb*cI;kbS{j~Q?5qsmYXiS>iqo;$J2w+ZKYqK;$MhB6v_DtTjOE3jX} z3Juowqj`vnd!~c7Kw>hK+X0%!$fF31#2c|OgVr-k1%Q#O4yCk-x|u2-*V|r!qKA)D z{MssfMy%{Bm-FC{j@=okE0Xj9FZ6OQN22%Wqo6+&YzP*ju@8N9)3Pz*f2rgy8v?wtlc!DI5I9$G$F!7Pvj2Z^+mYK z%S_C%0oT%*hj7)nh}eU9Y2;{4I)#wfKol}6XU(d<8ghIyyEJrl_X8^h(rnuJDcwk} ztcv%SyghBWYvqxOd95VqMjAJ9v5~1;_hScF))qmQs!mCB-3=5* z?MFzfs%C$o^;i>Of773Mi54D}3o>)Wdkn>GSB>v*cD4oY^?gp;XOo+2MjRTIX&md> zFY7M)DO2EyRY7J8LE6Jn+AeF!{#f2;Zys9;nOWm_KPA9h^JvWs@&k9<`7=r)*Dw{& zmM8L#Ml_6Xd1i;bXVf0#-`I#V(vU_mE{2p&rHT6|%ND@Ws{S|n!%yFS{P^|9H_o9f z{>QiP-$X0He@X2D|EE@h$rspCa>X|yxBu}C%o=WqZ)N@M-#>l-@BVLp`~LvXe>hI@ G@c{tQCwgK4 diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-final-process.log.gz deleted file mode 100644 index 92673693064f598b06df5f88bf579f9b3dc6acee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 353 zcmV-n0iOOJiwFP!00002|E-ZtPs1<_hVT6qKC(m87RC_UX+q-6#}T2DHeQL=CdGC@ zsfwE|K%$=Y8Y=z*QVZ-}}%(8-q~n>#?^1;k7iCDdD6T4$g`#w)fkc+k|w; z8{6(NxWc?e+6y6aLD*Qv{9<{T57vj`7iy8g&zOn!9~s}AIRLJG@cfJpo&;H*V5GS7 zJ;Fl@aTGuJcXW>+mO7LZUVsm9er8suJa@xg3SrWlnz^uS$Ct^ZZdKfhZ&Y3mwL)8D zuJIHN@wYiU&RJACKL!*Io7Cw#{)v?%ww2pmE?4KP@~mmfMV8U0P&F%M(*iD$ZF;OZ z4TGgG#CkmQ{oM#lAvmzS6Svp!#nvZ#(PsY@?E z9Gbdn^D6tn{ZYQWefjb0k6&K?|H{>Oj) zg#YtjWm;8v^M5q?+yB1Le$D@te>wd6^~*oM{*rzB*Vli2$ zYCr~~Z9yZ9!vtRbuc9j4DbHU17cI5@UoW)8Ls8~0|8G~-m$o<;f9Bc$_tK`#F>n9F z{pU+jy?i~teErq&uV4OWS5=r^R~0P=a4)Xe|5O&c9~&2D=rZ?%YiV*n$Ur}3`K~+C z|GmUV`NQGMKX(7z{rvMUKY#say8HI^KmYc(M#um9_2(BJ|8MxeFX?{Yox3t^>n3rR zeE-{T{NZoEWtn@;+x;nN@-%bwf9X5LTb|$tO}hU`((-ajUw-`Y$Cvu`!C1o&lB~{M zQq^s8$(y!ocFCnK3mWr4Gt8SyQ&jD=kxw64lP0fanG`O0%|G%C|CsjMece@UaxSVw zS9(q_E@|sze|E_wZPK%wHV{rmhM-kpF0y5>GRU0C=tSsXT~g5huivZWwWzYB$nvT! z+K+`RdnehHoO0ric#Yw&b5@%-4gD!A9P$1KjVH^#G1J zwz!luK)1Wp7ks6tE1qIn7U{%!ooM4vsY~{B8fAU-Tb3N~P1%RMddtiDk|*FZw{CiY z^=3>m8tWWbmi{sA=Po_t$K8ckE>D_{{yWR}bw&qJ=11I0KFPez8)BYnpHCc?w&QAj znBk0->4Lgy=91_K71;d7F4MnQrX|dFZr+$T)^&eG2sb~{dqI57uSvRZlZNh9mb6WA z(aS@kct-^IidMAm{lSjMh7Lt|xvmby5#v=^{hrXxsk=5&B5$#2ZC*=~_QfQPD1zdN{V&-dAO_v0*8VDPO10@CuVQmJ*vM zS$dp0@%;ORwxehVGo^u(kMztBNpY;|CeNmh81G<#xzqJKLmOy|Jzh9wpnUkG-8nBh z+@x-bBTXr0IdH@NltOe75u#e?+(+Jq@x5-+_oQnN|4c|kG&H$A>811v7iXF!9U+8@ z{ZN@EtFg_!p=qf}y`%Gglgc?QNwlA7u7`$rIj?BCOqqB(Ey`qH)-IpIf%ak;^PLL^$>Fzxv@6mD zY5c;jByFRsyJpX_;XoVJx&?GxULNQlhrGc7(!xk+p3|m(koos~--(!kvN^mq*Z05= z4{=fEt^ASB!qGO)9NJ610<)!+yN_zmTR|HB7PT$Hv@L8*Z*X(EGaV&ZO1lq_QvGHYOEs8=VhvyDLeuC$v>n zcg}IYLD$g~Xs{h-lQzZlXTuB1{-fo(=B>cN?>pDlXW2>+V(CfjQn^drw2zk*y~%W# z$>8nQ?aDe#0)h!7dRzexxOvk@DJlsan)xa0q?6-kMhHJzx{NF@$>eKU_!E=i?g<;z z4GD(?ggVj~cEz!)J2!di@yR!NOE=$%G81bGVb6rw`om+AyAL|e9ogYfT3i^N{)aUC z152cV^7!ay*PqY&IhnnK&Av7XgTbX!Y;`AS;Ey6N$>s{5+%hF64Xe-Nwu8D$%2NJN zl;DfWgOBoqw{ASYxjl&*sKWlEqS}`o)JWMS&nHz!>ej9(hk|F)DpVM3vo&Hv;uTN% zjXCAtMX?@H2<{A2!BkN}ma};)cO&>2Qkv#|(ImY=1=Cq??-s+JJ{(K2zHlwr^Uy*) zqGztX@vz<-cd@QNuCuuOoL10d9Vx|W+u^Ne?HC)72|xV-ml@uiT`o@zEriudR^F%1 zoNU`%pCIEd{?mfZ9o#hkWE6jjoX#44z;2B^z7r*+ISpFoDSiwIPI|+MzGv16>A5&8 z+5uu`HIK}!QIR^j8^pI{D>T9CJT&yK%2v3Z1yI)YYj>G6hA<8|%m%_klwKSgYFgzvqM}Ju`6eJlPBs+Izu=O`D}dX zBXK4vlHpmyo;BqMzG;_q!Z2j<@U-YO$%t;%X*)i`EG9jWBEhaI(A1B@i45!qQGaI~ zhW8ALGDc5&*HpFyqDr&E^;K(q!hqhp=EWFX!|f@TheqkpMM#7XXvgUUPI=QC+hw8^ zw0-Z3H!|g4EX~S#*B3n}dOCu{8{G>Ye&8^O#G>6QGmi@Rv9EoPy-~yu5KD$6a>&^H z^Ggk;fO|>o)uPJNY^~@IFnxpT+2mkpeg}CkA1?S%z;0?kA|HmhSNN`bO^n^r zgZCyLM)X~^v}h{cAR=j0Gt-D0?eT=vb(Q#u6;9_i5bpgSkvN6m7KNZDNeK@F%{}Fy zq1Ohy=5R?r$~wI>;+0A-k9wzE;io4CKNLqtPtZBNIU>n~BZ=VyZ7I9Dm?w`NBA5`_ zYP`xxl5GYr!_BcGH(g@-qKj2oUiG*pJ4f93$GE9o%m1Se7M|rhbgpySywcL^%kA0v zQH0^cJJG^pZ>>C^yzl`d>uU9V2_8RwT73Z3$un_#I@pc=C4_4> z$Ks33-k9Nn)Zy&;ENIo{7XPD!Tsod}xboHQ$9t0}YJ~NdLvMXKa3@u8DR6YEZwpSF zNpuN3ulFS+v#@G3Y7Wzp{NCYN=XYn)e%nrmqxTOK^HB2MunSR~1Jj0#9?YQ?dh_kN zF_!lyk{~HX=RCM@#O!~KIc@C8xiw9^zZjAvb;AMNNPB=Rgg{OcNZD{T&BHZ`LXhjx zX|e;>D30IbTf?1L9*0TL^hDUl+AUc$LDd*+zq_NL$y=TW^*9neoS-4e2K>5rJMYNL zzpFM7mN;oKL&kRl#dOFxytGIz4{iOMn^vV4cdAFCqHHAv$Hu(Tg18?YSEH@4TwMV( z%tlw+Ab0dm{YHQIk&~e=sdIVu@0Twxh$YB>X#g1id8W6s)BhG3Lbr!*1a~m1Cq2$uRnkJ z;JwPWh78cT+_pN zQWKcDbm72F&aae1pfML~0wec^qha>1Wa+p(7J5*8pl-MMk=P*wDD{;J2?Q*EKFG+W z8!j*JqtSg55=#>HL7Ka-$?00{!Sf%SV~1;+yywO7)IJ)O^bj9f6(v>!Y)~{|v>v)) z9CBn7=r=ij-ZvdY`D$|Lh%`HSSZT62w@M^7YxMoK=VYh-_S=~(aXOmddpjck&9mUK zP7c-Ez~4W%zR|ToJ5RGSgqird;W8PY8`%rjAF=FIg_ z)89!2vaivl9cXC}2B1xUT`N9$w=UviR;9q|~WlY)|dXl;f&Qtbb5vS&PsXi^_ zsP#svFEoir>e75T{6cEiP>sL;)da4LW5w?b9JcT2z}2AF&^CYE`KhMA z&@(5AUHd^opU+7Ub%*#dKfuI6d|}AUH(noJ3SA|bOzu5Zp1aQCyzz6Hp)OIQa*)?} z7&vXE7{;2!Bg~%c+C}GOeo|N4{-|FkO;CRT{iioE zq()5{fODyMSx_~8{{EfSWpMb4Wqc!L6fQitj;bMx-x$is3zLcG09jp-3^QgldSZ8% zHkrL(jqsi5r}VS70<(rEJNb5KSbC57g}l`p{*rr=T>c>Gg!Gni;ulohI)rQk_hbIH zs<$G4chkv;FSxGg758>AZkM;=WtZ4{Cb39fTM7eK{@`Y=sY!B9@XqoQF$~lmV081i z5_=VOMlU!A1v&oBO4-~aP|tspA4$t|BBmYt9e4#(hdXqftJ2n&*GQ=$y&FUA?>NIJ zOz~7W4(hzo=}L@>1Z%RRCj^S)!UdBs&Nw9JwJ8K9BRgTAW>-mh4^GXB1#ouev1ei` z=y9=YJX#twnmRToC(!@T;qdLdl0u3?8pXO3Nj0wT=~JpRC9GG)!qd#(@NzGc7{RNR zhiHmqo52+#y4S3I@V6aC+sBpLtjyB`wAeHuv+8(Vq;O~CLXBge)h4>2IN?~(sa!$e ztz62$rkow{IMaYgL8nK1=n0z6#G1u+A?*m6;jC1oM`WqCTojkE>2gp-XVzx0S`xzU zQ3M6|M0`t<{jSL}la~+@G}#6UE_hA{zU#%Eq|U*u7?rSO#4Bw9y)$TclP{iDCfGHG zehC##19m3dgyKk6^qrwkAG?OEUzA-$S|o zRGM+HSZgTY-h(&E7cB9$7|#K?w@2c9XSTGNg1hmy)28o?Pgy$3S&)reUd?;NJ#wz! zc)ov>LFeFM=1$TF=9O@CUut9zC8#0i@UQqIi|IU49}X^W{i%VgU+F!hs+?cli4BUIho^4lDldro+rc--8|!NXtPdc2dMkrS!8Ozgk7-< zqKi+L`Gf46fh!tS-rj3qM5e0-w+7ddBI6(V*oStNh5PNc6Q}=1vkwUc0yWo8mj&GI zTDqh@GyZHcW1i1iLZ?}97ITmSFJi(DoD;mOrZK@COG|Jomi-Sg?8lcqL5sMdP12n{ zO5|MqTXWS+eFAJJyhQ+;*uM*x(=Y^gZ$!KFy;?^r%ftg$FzkXiJw#eW1&%Xx?}%}g zS0iO}sN?&5x5=-CWQM~oi0m?dnZw(mW_x6@Pgl6SIJ>E$ZMG-rNYmA;YMz^q_t3#Rh^}o}f1JgkBbvFJ`#z^xdT9s#1~D__-4AXa&X}<(sIemGJJf#Xpkg?0 zQe_*cvbtD7;TXZ$n4-|C%K1lE<9+m(o&%v;U|sBI9IeL5d?KkBuaS8r=RV85G0=wM ze9+PQmOJbL10>qXRhFs@*0(1kSFHt>%slkMW$O4may&6FDCs)xh=oqYW%dP+qam90 z7jp1z{BZw9%a0#h?~=yA8ZfQFd-+6fG=14p)vT_>vYPbnj3ijx+pO-qW1ZNsU96cK zaPemandL*(c$&i!KZiNV&lPE7Wp{}nqw2b31cve0(LKhoY$tl|;Db5fzs=lT;Oys| z+gnQ1g7Jbb*}m7_8^xt|sz)*aU153oqblT`@yU1+$rUNBoEp$`cxkG$ycVw29Q^4t z!q)m&gk+*O`I${lX0yrnCUT?|0o*C=T9lKusw3UWCJbZu9v;`SIYqB*a9|v`%9j+g zddGLBKn$m*Kj4}y95m+&VN6ms{J<8oHB!dG$huD?A~hi&(DAxU&s*DiAqJ8xoZ19Z&n+A+@$bQxoNe;cqv_%9|*mMlezsAuvaR zTBUYm8r=1nYJGNYT#Wd@2^rJKze{NXlDpo{tP;zUCf-bi5%j7O3H%Gi>}MzMx$!UTP4flfnZ0z2{)Gi%m@ku=AN zOgbE&hd8u!e;+tjay-KKorHRrlX%T5K{hGZocO4y>VD6>KZqNQS^Q1 z!V1dhoMT6vIoZ@UkB?L=3V>dm z2;yP+@c6$M*&xCGp3A|i-t`FjI9J{(9k_00duADtGRQ@CUu0Sjt7W|VnhbZhzx~*7 za0D>MAOx2yO33^5CynJ)S(-MdH4k4Vx?DHSqWeU+e4{@&{`5x&4GNpMcw|3BTE0X3 zyKos38DCcC!bLGd^x~ZSZGuafsuUhzB_^AQ+1=M@C`7O}aYSR}y6B8`@0{yTE(4lu(X4==nD zrw=rFgh(0u&R!BZx?%3{Ax;|gUuN?OF+MWayG78^Anp+@!Ym}=f4ZY&Ct+qW+(q`p z;pe-C#GNUNhbhMG_3m5Fc9ZlHOdXY53$}#CjaD~Qh(POR7gIPhG6LxN?|G)I`lU%3 z%`s%&B*=uhZ~fevFV}YNNZaD5vgZ^(=VZW4%b5k`b3T(UP#Ww}0&t4dN7Bx>gNjp0 z*^DL7um{*7QdX7Vf`tfUE8LPk;flaKD<_A#ZUsEfX9~d8={V_v#gSYBH^5F8v){_=e^37Lcqv27m>PJ?Ibv!O&UO!X5wCoM=Bm2)5P9EcPR9Jjq_8im5Wqrht z7jH)}g7Hl1(GrL4_=&haSqn)|CWsxb>i3adP9#KgX_8;mIPNZ!hn0Zw;EBoiaH%kr z-EEEY>i)db>nZ8|zO*fvMGzY|12(tMDV$9d*N^IKu=6_%32#Ex%y28)7{kl z%taVH;}LR@ca|e@Cw)8m$V63F`T7Az=g$WOiCGwv8J6m)qP4g|^Uq zH9pY(WB)*Lm`({#QDB%r;I_^=ee6i9*mV2VH#odZTF?5iV0^i%RX^S+7ILV7(=fmN z{I)OUA*HL%b0?zRuk4&ZbU3?x&58T$KM5(Z_yo!FGYdGiXis@$Sj@?JxL07%Gv^B- zz+o;DGnQ^O)dw_*fR)(m#xT$KZWx~O`2?=z_`TNpgehJ#@y=9j3v{ z442XCp{OKBj?ipfzy(${4K~xs&0b#VQp(7 zx9@rqQP&cTSZF4d6djD9vo9hNu4W*|-C6Da^U^k@W8OMRF_Pj#Tj7?3_%IlSQjcf- z;A7WVktO0F$x{eLJ;$bo%oY)}$2&lb6?{Z@;*rG5v+Jdbo;9ru^{-$#8c@tth zixjFM8N6H?)3`~9ol9Xt91Hf%J~x;H@1Q6wE~@^TKQJ_kMqF29q8|WSr*r4bxoVRU zb>)KES3E3Wo;0}6K*`4J;x!{%zHYJfDGfszI+X?F$;$I4mTz|WX<#U-YS0$J6*&SeM)VZHw^ob%w})(R6%Poe!7{;^k}s3l=*h$Ar)xhnBIM1 zLt4mm-gr?J70h%G?ZC+2Hx;29gNgj2DLO9|7f&qefg7f%?Lv~hW(u8WPEhMlkw6xp zcibLne7IXzJy@p>XooTH$MvPhXWAqVL4i9q|Lw8Dj_F8?nT!VW>}8D+EgKyl-|(fk}oI3%ELOHRVvk@M(Ooi9zM*R;45RP$Fy3ND7F!(onbwbSrh z*115(DBU`U~wJIBXO#VNKF!l6M6HtEGP`N3m2va@8rq?4_aFxg>n0dEAU= z49;;So(Xr*lvMDV;VWv>1XL=|J_BE@JKW{q{1O4Fc&hPt=I)X%D(aNY*aFhcLHS_n z0tU@J6MeLBs@h~j3V=BNdb;PkeGCf3XfIOJq7Q^fy)jTAZA=T%O(SOY9-uG90lDHO zhs&TMelx(vQ9ESP5dW|t@Py7i2>%Yl1?Zvp0&Lc(^yp9PstIRb1vfJ!q+EtHFP5;k zyZI`@f+o@?gH~nsXM5OoP5qizQD3akooNcPnGGAmJY1Ol!^#YY-rp5PsMvj6)f+wD z`jFO8*S-`k6tO9!$nD+nIFcf~$o6ZBBP`Nn&?ek2ghG6~GL_m=h&{5PFRTps6!Jl! zVf3s;A5jp{@0TV@D!DWhelGkC~_Hj7w0|-2)6JU8@(#8 zxF5hMsLcxA)3ke^SQz&g9>Elxv@n6%D+|Y2qEJH`oKWMH(p>9)Mq05?x}osehr77^ zn%-XkJ+$3kn_doAaC_$WNA9ZVjiv>qjg_mDkbh5`Oqv6v&90zlp2qXQ^VxiH(@#qe zWn$i0nYr;!sI#)(dCLR$rd7YJLsg^u2np9PKF)P{^6Qtc^#6Yj>Cz^Nw8(%{h;P(Q zF1ZsSw=~K`MzOG@9?Le~&-=Rrjcrzt3QSbQaevz?S9B>ZH_>9#$CqL!gCNAjE#}!@ zFaHKVd3Ek_$Y~ViE(TVVdTI>`c&$Zn@9)VX?mE!1d!1~zHn;v{A5m+9Ta9Cby%Doh zeXxA+krtujq!n#pSZi%WakSpzuwUsB1<-V{S|(8QXc8qdrSqs&rZmBeLDtgND9rG+ z^^%rUdc(G_MTPcQjWngY^ycz(1Tj-4LT6!*PpRkvf*r^ERyU-DRY6^Gv(F<+BjH0Z zmksGD47Tmh%LbT!2Bbgd(wCvz;dxhn)3XI?*KC+-+4YRnkX?qR~0wiP`AO3n}d*X#5pNyjD&VBSI}kgt`9>x z{lp1_oHig*KGIPv`SFu9Wpt>4N>Ks5uc)yi*+K_k408)JZe-HOCcT^@H%x%q1i+$u z^m7j_Z+C{9`La1kFRfQr&}Ga^<2l@Geo=hQw=@HLNw{|$lkx7DJY1w`erQ`4F$sK- zWKMag%ti486cXSj(4*-(p2wFurVRKbTo!5d2oV{>nm?@EoY6wZu&LE8*&ERO$LMVJZbGYJCMqi(2MgCEO!mzLZmjS-dbYjE%T3Ao&7z3SRA2OuZV zh2XB$aY;`(aaD_TIOA8`RyFi7pEQ5%s%PNG_F!6{_d;CB9eU=`9=1N0tr=!d+`F4? zdc5*^NR!FyPO!Ho8s?^_rv7yJGqb1-gg#v)W#8mD4+FmI?;)dJ3YyVi`xIQ1V>x#Q zur73hUQXAS74EhXyt=8Y<6~e@LC5VEn+!J#%^0l*UvmYJIW`Z!<@b1D#vS*KXRq6N z16;n}IKJ7BddSA+Id8{r?HX^gUO2sKmlQ~!=_TC{H40l|FQbCd^Q$_s_9Bf0P(tXg za3dM*kLA0`#$afQTROGM8E63$r|0LgBTdTnraE_o3*2*>PFpuuX%tgO@JbN|$+FQ< z`_!=nq}ZyWMv@k$4rGab&7FLm(U=H>vq!NgcHPB5*5=H-MuNDSThug{C0sG*2T{x1 z0FyrYcHSMPAWL1c#a0f~$K~+5%_n zL6JFJAA)~EvxGPv?Bf~yC3OJZpCuZylg#+QF#=}z)V3G)*mAxUmj#(rS59K^Ge$WN z7~yr9Pu&f?D!2DyhMpGnkYZm*^TvqtxrADz4THRB9y4N|r_1*_z|~KqN+4g5wdVLSfYkA22!{}N;@y4#1leWt9*qeUeXU2rZ z;q1>O+H~nEB($$v3*B(+GP)i3d>H2s&C(>3;8Ywq=HHklZFvToTd)I+=IBr}&dsj| z$o79)qCm3jOg&|};1nG_5l3Q#myN8Zxcg($ih&jbB zu7dmj>046V-P5~Rxls~*FGOvPqIzuv)E72$TSbx(wb$6=%hBa4Bz9@2aH?;&cHu(g ztZt**%=WB|uc(Z9D!@3m;alC*;X?(qwTj`dxX|j-Qk?F0B)K(jepA?4`Bway@R{sr z{_-G>)Mm^Jw)=l;NI{a-)Y)tev_iTI2jgKjyG3Ul2X>r-2rJ+ndKhRA26eDNIq#&C z=UtHDagsi&GbU2xw)mG5q0qqR%a&%!3RL8%Ih~*vmZjIlN8X|txsZD6qSOfQ#BlTb z^r+KBr6c1#%b~b#2Y@BXV-vHTp9Cm3fNADP&4R2>6Go(t=G++N&0ZX?{*atv?`ar$ zOy_B7_IGhp5B%TKt+SuKM^|pvdiqwpHENCWu2@TeBgWhF80b3HeL>um-vH?pAdMu5 z6D7msJ^d4G}UB^R`YR7y;4z@5qzeE$qzK?PVC_ zUZAYan&aK>%{uRfL`4a`Xp=qwsRlBm_JVcYPLC%0zOUEK8TA~4oMd1+HvvI?oPDBx zhjFVy4skU#3AqU4X^?7pn9+(mtSR&)w#60ITm725q~HG9IvLDbO{xcPifkyjkREjw zScQ3bv-V=7WwWj)BrT&upY~@{u96p^sDMDo-l`dmKX1p+?s_vHi&;rF8cH(%kZ`Ij zcWLq3cTghGO>J=}P{XqjT%n8va>T=)<8*vWs-@@n`_vA)(5v24k3=d^nRWE%>3R0( zwVHKh*ncT^h|E2U`3uNil-D%tID{>(55v7WIf>{9qRf7ei9XXx_V+7fpGgdlHDk_v zw|wX3O{zRBG(WR%y#|nFeSYmeyO>?Nc?20Bp}ww=!5Xo}dtzS+UKM>>Bt_mU>rB8% zYN(0<)PJKt{P^Bi@z#uUN|!2K8j?sRZJOHd%qqonl|hU zl?2U=byTgekGtB3^8$*KTeE;O&6U$+>bAV-IdJMfSAO59Y`sB4U6q^|;KS#=Sedva zBrjWT#g1ff?$(Yj5xXS%O3sbMd?8BRO0~@SIeX>nySLl0@CUI_`A#4+INfe!$8Jbc zedx`vU|0=M__mrhyHwqChCc7|EHgZ|r;zXkCQlii{Xe8dp#3$eBg3H8R%ELD9qz$3 zLdkcjL|HjINEvfMRINxP4Ph&8EuqLX;u$60o9S_A;x>&8*n*%@rh2KA`wB3Xra)dh zJFnwb3)SvhQ1ZLQK46uKvb24DtoW}ygKCH^Hk%YX z&wO1l!t6**Qp`CnedK_A)-|F!I4&)<|F->{Q&$r}dKa41BD2@#|wOkGV30N%Od zbiL4QnryWFR9<0HT^#AK@H;+LVZewlZQE$JFDke=(H+0O)RoKEuS)CVirO#CQX#4& zax{=#P;SGH{MQif082_!-(FB*{le5cZ8^(Lt-&er01IT7{Cu=F^fAq0IWcsbSILv- z20`-u;CU}Hx12e^1&LI;i`>dCK*xel1S2mWm~?tpQvI7RiU3WTBze}!^c$faX z`g~yqE7pj$@FI{kdu(zk|8r!GqE)6yVR;m2*9GMm7}0#e!Rp-^HDOr)B2LcZ`prelZKEESr0<>g_^a ztmpJip>xHp(ZhgbpMLXgNuH4`09C*^15cggdG#zR>{fo#*wO>M%`ZZ{*eFa0ca4{&zk)`wpf6_)-Eb7j+oBVD{vFe<+a*EFD zHCi!}>-v(!-wKwB{Bh+zxz0cj#2E3n?mcZPj&#<1Wf)lC8MFc+4W?R@q@{(IWDb^_ z(7xga=DU09(+XCr-Q2P7-o_BYi$Y4iMwm-E3>5(RE@NT#A!Yj#uDaIdvgTLp`l`3b zdGKcj07i=#O8_&Ba88@oK}PuGTgAnOHO7ll_^ii;Vbie|2^p^Le@?t!p@;R_@rr|| zyBwPo4V9p?;#UD=_wJ7J3E|&zblOQRYQGSWE)eC*_`pBn&(&ssPj9(|=TfqkP!emi zi)LBRjtdsJ>8P#Sjd>b6%W`+})l0)iqB?&ib>O;gS>W`G7ZRB_KmwnJd(0q{|2~PwFSE`TKZO)#97AJE` z5w5*YZa|OUQ(@&8h-@o7Uoh6zKlrGX>MkDTO3GTJUGQP_tGcK~5Q11Ok6?hT*|uO0U~vno!qQpV&SmJ#gOMUC*F*wLaW}4{oW1+H;^#q=Mi{De@&p zg^(;#5xp)3Q})7HCoHbsiqz)8QJ7tweQ@R2RmJ=^^aQrei_6WHJPigWUN(Ce_Cr4e| zlFJ!OA1N3P_Mp}eynXGNhLRe0KX>ge$@mG7hK};`sHMnxgM-_>fQlXR@Sd7KdzWUD zoIzX2fDM>T{9@Vw%nE+TV5C;u53(5aeoF^>e!9^JX9Uw``6?nY0{h)@Uej_p!frmy z^nj?k_kkz>`0{uE;Ew`tOuQc~xozipPzYGLZ^6G$x`^1~liYk0uDbSpcbA+kig4lA zvli3wj7kKLr@+a1^_G|Qu!~QqZAsd1uK5Ngn|e(rv1I%MO^!B@%ay>6A5zJ1eb)S53Be8TvVun-eC7- zI8-8D6h&mKL0@TyU*-BDLQnwmy1N z!7!!F7RqiKS2@?&cCQ<&Wu~}_W|Kq#|OL>>%Jip=gWOBO~J|Xj4q5q;KIGK6F1?0UaQ0l();AqFWvw~k64*pFoVg~ z*0Q5G*$bqx>XcggMJJw?u8y`)YZ**rUv>4Kqa`!+wm>SPx!^!MSFIdMmRUIMbxW&x zOB~Ezps@L=IdPq~cIK`R6Vh6z=p)=>V^<$LXTJ7Zdh|qZpA=HvHtmBCbAwl7;8m3V z&{=0C^lt^qe0RqQOlsJ<0fkGsb^sy|iu(?KIQ&e6FnDEow}+#p0RCE20ZQsI0MWP@Y^2rDCQCQ$MY$Gf6;jI1d2`HH zQEciz^U!cmYw{{Fq<+2`h)hxTa&GNgw36{58k;1H zU!hXXTb}P5q>y+${3}OV28(`XV$IDML7A9IGyY8br_xp(E7$AQ8A4$rn_?ErK!%p&N>Ks3;p?-#p&i=Ox@aB=59TB9+9cP|3p z)*>arH0*0Yp*11I4^mF|oR|XFweYXP?{8)6dhq@4g-^Xy2!e$n*-;OOY1I5+8%;G* zp0kl$WyHy3J!B3OeaE!frH`MlQL8Sx3EIWUH$-xwlY0cd#!Y)F-VM@u-amK2;|L@Z z?tYp2q(Mnbw>Huu{hqRl>u<*GI?3+LKZh)UG_I@aoD|;;!OS7oP{x&(E3ciG1RQBPYz#Qa&#EXU~Kl&$RvJ+h>y{UGUH}8Se9Dzz*xbhAihnD&@iZ@w{V@ zSe8e6i88F7$$h>NT~~qpE}*u zc7y+dg0LIP6#(q65+1hSk9)t_(kx#$l`ZU$bu(CPy>CE%V~c)^8NuOKu1{Uz8(0>H zoO`W%Z!CWC*R621O+SJBGxgR9+hXCogc31MSJwSGy1YdwA0R0x#O8 z9<(PD^d;wvZ1pF13O>8foWTaptw}y6ZHyRFjYV8Xe{r8g*!KJy7~xr_+Obw2I?k5` zH~&h^HB%z0?8&y$SS~j(ocJXtLI|h7In^fd>GtRGKvP_(u1;_- ze~JOdkpJw~=XPGv&t}X4B^tgt9jYY9sHfd`4ld|h&fmj}#yZWiIKqD;BY5r>C>*n_ z165%7O|>;asiYi!bAk9=K@E&Olc!fun=2OAl3J2>w1TG9l6@AndE$1n&0LsIHHkhq zROQTVE-#aQdT~qDPIG~6uiYnZw#c&A_p~{F64}|=UDIhj+i9`MOyaVIN&xy=K`D2C z)l^8OEGJ-3Jn>BCIrfZ z&Wq^z3wlYb(9EPm#*FlRnI}lIiTSaGTbq70fz+nKgN5{-bdXPydD&m?wO==d^$DA6 z>GyN$xI^scQGXJ%ZIq)qfs)6SFvO$hoZv7m(90xwFG(~a{~~)X(XiIgYVkAZ0xtzV zd!Uy|0&3D#>04Tq9I0S4W!o8SnfEw>oKJYtqKZEWnwQmyD&Go)jRF_EDDFpZ26ecY_gv%=3Xc9$nH+r!kuu;#ayQxw( znc(KC($b|qi-Df4m#v%wFhhgfUl?>>IfwPi;pz#fa4uw7rv4QF#P@ePuAwKxKpB z>9LR&SGD3~CMnF~;!ONB_ohss5#=U$I$Kci`eeZ0VaUhcEu5od!8I_Ai|C7Ux#-5& zfr~g!w2X~zj6KRmMrMQYNBJoIzKo9^)9CY&(Ak2*_IzY<@5l1)?pK$*!<*4wkm#`) z?YUaPW`t*R?N)^Glp|A#cKoj(KVRf*w(x~2(~Jo$)hUd^bOm6=2|iE-=8t@LU(D2S znf7gf9?9`~bYns_1XF}@wB=&FFh2ndkz2*&u^*1X;{_~wspy%fkMt&z70ons*ipLE z-}fhQ$rInO2g=2o#XNfQpU5S%w2ke!1Y&+ly7GB5HQ+DMdX_D6jlQvr;$kgh8M(!} z#WHfMAJ%1LmunJt=`DQ2Z;g4Ky3MUL=9zSl2M4yWT1G&I?ao1rZ9vCbspE46a$Xc; zxi!A{wlTPdk5S+5P*KO)YWh*uX|_EbpO(cKzuS?~-afih_!jCF>?N67vhaWkKK4M~ zKys`4=*U}xOK9OA zVq2bbbdkN(WkL5EAk{ul!%;nHD9azr*eC0eHNIEP83_@=GPBrWvda&3V@*zQ=O?r9 z*b*iF2GI2H{0W+FkjxE2Q5T(hnEEAT-qEE2tyB_4#V6$f;61r-6Y*$N?RT;3(8grg zS1#)=?BTSo8*2t5{;Y`SN&R0jibO}{tS70^la7YFxUylwe_Sy?t%`;j<&S2?9F<3k zxWlOQZfI-%PoLs9DWm$%#9{o(-|5?Q5B7XDR}6@(h-v7mcLs6N=Ybm#OdLkq^j@|3 zv~m>(M9OKn_u~U<7a{(u_1O_s3%{>zB>y*w?DMJoILez zm!-$V?N4-t;|?9BVMy{OHwG!V+l-CCK*-vIOOpRG-wI-aE8-?$ZAoG+B+vu@m^%aLuj!UbquKwmM~0;ZN8LFnN}6cgW?F+ixeXf+mIfM zex7Uz@Nz(_czkp>=wOauhpTfdYp)Si+EAiQFBg25mkw55N>e1dW{RiI(b^z_(M1$j zz7shUovk)t!l3iRgVpYo^B5flbs7;SmI?MLE zrz>j78_NV_4OtFTXh2Pd!Wr52`%RXU`JJZ;9g4=MX{{NY`I2ux_if~6Z7xU${rg%9 zjt92`o??H`8c{Fgren6ds}^V|jl>7$1}ne@1E;o*8lx;eIO$HT$73*&VYM|5?~wb73?a7+b7 z=qPqFT8zOm_PxjEk|LF(#OE2QSkUV5O4Iv-0v&1Vnl<%Bt&W*vo82yD4TZBz%XhpX zs@j${lmu>FioS^|o6I=7Q^PLAL3YOvcRpGl+iD~aWvL}c^wyUlG z9{Y0IW-d#`daUBZMqMdY4~uoMX}aaZl?1C!T|(&|E*rWd$NASA!xz?t0k=1DDhER7 z>&dPSxju`|1R2h z;FbDA+3^l#HB&jKuXGcO@4a5Ia(Lwg56FOuy15X0p`Y*aQeo?MbgQ9S_Q!G9vDFE;?W_~+3KrpK8fYA>)hro-_!LgoC}!h}RWrE^Zut^D z4@g_b!l43Xg3ssHfZ$h|6P#Q?D`H99b{zj#K$1nf0A#YwPK8+2?{8o?2 zsaa8EOZAx1HJDl{@A6t}PAqqMe1*r`CQmusEghYbkMvyj1{HU;j0bxS?H~7K7{8us z^ks@arcd|#yp+2ZEaI|Y&$ov6@)mAfgoCPH1N!}m`-RahIO8^uh4ZxS^O4V)wNCe( zq#D>k(56OM!$K>!Qz%w%ECXFfzW{BqIZ&qS8`psK3)1X-@0usiLi(r&^$b=YC%i^| zM$c@yc<{D?YjE|PfW4h^UL3m6({>=!A^#3Vxy;i88~tJqvKL7P%+llXS@e#;-^@%p1PHA|Bc&?^D&40@-sY@-oB2kI0e1>rKAYfTk`?AAH!d46Mjcula~9%5hmAlT*=#yaEm*3=R*0dKQi*3qCIVc&)qe(>AQ4 zI}Kk2J(r!j`7srL-9*7=5cTFp{?-#_B|E_OozFr=M-ND%D$RZ;)2b2=o)?+o2HN5* z$ELU^<5l({%#m%_bKa(G1qDun+s8?IknCm7xwKf*#{3WbqO{>~5Br@4mw;@UuB?gv z8!#p~e&s7anDw{i_jCQ$GkY3!F+W){-PnY)z{$hZ;b%R|_oZh0Fen-yjYK4TR?X*3 zOtq~u?NEUyD{zh-T)>ctFhPn>;vQ@q{f+h`npsFZ5=mw1)2GY`33uQuy+~DBaS+Ex zLS!!`dz?Ol5!E6iICYX)uSIJaF!#zOVR^&2IAt!=6`RSve7TF_dM(-BOJf`|(shso+!KxT3ehWL^?G zF(1GVMt9C^@|=0Fo#CyECP%75Bw@HF&?_Nhft{@;(p8$D4jn^84=FfwE-ZRQ#7GX* za`Mv5HnPuwK`Kc&{CL44ShVM#^X+T>t4E)C15hE3V zvzs~9e%_YPQje?}4X-&Ejuwd2QL+Kx7c!~2u}WrHj(E_UkXT`9lqo&2WKXVAeIBcV zyP{%?oqoqjouUxVWN1&$v73U#I=yyuQx6}eChV{x?u8UGA$f15Ak+NXm!FegHm9w} zrxkg?x>NI5LsM2Mft$}+MkBGR=gh+-P7JyVF;B+RbxP!xQz#5uA}$bvUKHgDVH_-`HDftv}vnxzX34z#ss^nM^a2ZGc{?!&!aBUaLueI-g-y6S`bptAjv*<0~PPp zPI8JL{o;^v@U=Z{P4%i;b2ExJxw)&Xz`-^ducqaOBZMcA8LT>l^pq#awi|a@lG8YV zSk+Q^(Cb7a!01=)XP-DDP7zew)S{@QX%}qa^9*}4$$*33@Nh)l^cFy-w(Vt*`($Ax z)|;$oQv`{K`EOsoEnFr7Rs|=0dY4ULJ^cg^Yte9*Epdo9=6w1-~ zvc0mM!Gwx8nN8-cw%G_7(;_*&PcDj+nswA%_M6ccbH_l3%{DaOEmsUn(*wx12>+R_ zC>BGoVX zL!L>z)KrbP5D+?)=`pDf2YOy|c8>TGktBS3JsH6rXJT>h6i(Wh5Z|7wKZFN^p9(uO zd)K{IE0xbKq-M`mu1Jh=DQuP=X{~q}jfYS9{*`@D3M8EsZxVUXf_sT)dr3c5lDmd{ zkbDJQ{MP#uu%i7qs7WD_mZRu=b5;q-H`jTzho2 zUhO&^c$rA$woQVDVe&^c4>^DF*Rf4cr?J(7yK0}4c5x_18J<+`3bFGjTI`b!%8#w) z5FRmq*A*|}DvA%Y67RMg2y)oFi`!QHk|a1@Z^U+7kZA14{O4^QWyGaw1KW;X{{kC~ zD_)Nm<{=hjhkLn~0^?y%2k;SwFa!(7Pu)kg_lb?s^|%7_IOpejB3u?8wKlJiBVrGCv4r@TSoql0#L?Qe?K$+VC!7{L3?!!n_eeF( zt2SL=u|ArF$b}@$GH7q?6uZLdb8C1I1NT1H-L2}1QEDOH0|OfWi9lzyaYSUSS$52V zfMIjYh>YMOG563wPZF~ZlANL99>MsHk3haH?-)q1K@yjH>(5q6P4N<7U(tGA5}Ggg zPZOtPmNfj-7Mb}EX0Fqu4dmt!FUaeF;t%nnRT?B2oX{3-kPxS4){+a2MQ7r85`~EF zQfGM{TP!n5_>+wB<5Ba}b6KIPKqw*O?>>X50IVG9QkPj&T4=^e&>X$nf~+Yg{Q76o z%!@-qIMaHQ?Ua?&2Mab=%SO z&x~ql!Y=F+f!(mw7tTttYYT23*vEO2&xWotu9R6A+Mcky2>2}8%gvlR>Rbrb8D2Y# zl3+aWK=rY}2RXh-pPyh)gLuhs*Kzq%>%E$7K@E^bfeyRq1jmYY<+>B_QX;t_=Xw!F zPkq-2zI_C{UXq$nZgLZR?lxnfrVeKn1z!)nzz7tSGb#Hpz_uU`#wl}MC%hH#ln7-0 zb_UzPO)<2KqK19mW3auKS1w?~j3eM=P-8R@MaoY8-KuN1*w;8YO)U>!MIj+r%YO9} zEbIDp_M3QHukvUYd`qRKMcayg=UNw4elmrvjc^K&>17RPcX2L}o~f_1VWpSLj_Cg0 z1R7nl@ZcWs0p7TdM500w#)EhX0>a2vD@T3-uO62R8f9T|Fn$TPB6G~Ej-iA)P8af3 zfub~c38}hQSCz$U{)E*cSt(9Z8q?QT4Bx|_6s}=3SmkACN5%29tDBIxwiyd63}b09 zq8dhLbIH_-m1g=)+EpE;W*84e&%tKQ8Z;DD^rE=!u#DyJsDd+x&f#b)7VJQYgxiFQ+bXFw6o^W!LiWWwkh(SCUzS(Ou( z*9hw{eCc^)yP#}ELKddQ*(Au~yc(85N|Q$m^XYjH8m z3zt7p9P&Zd% zv%c3?!rck@qBuFl?d0F`P3eNv6;7OT4SNn%B0^_`fE3RU5-fL{&m&s0#K}tY{DS)L z=vTtU!5bXrJkXgL3(#Zlci+LggqGG!rTkx84~|N+s8k~z{$&$- za<`>ljun=2>8Z5hY3s!2gKkWZgI-!^V!9yb3$z;&LM}0^2k8aqu^_fSDk8F_z8y-(PUyWG_$t#5o42sEfYle^b7X| z?vj&BcMjU!q@!Dsaw1BaJEfPL%Q*D`ftdhzZhc60xirw&qYNN>=Js-LsJ*?UP+6aI zr2epkZKZ}bYQ83)yRFGu9=ei`QZk>;#P1Dk^hS1@ZmDEb!5H{JG`h;SS*-OmWvS-1 zv&|#{BhqO<+hCsaFiCw;REqw#B9CBu+I`W-3v}C4vE9@aK8uWHEBGpt?dU{uY^1g` zb>pRGo2-goA!*&x)B+8`Jq6mXc?Pu*GjH;9%{}%cMWG(~Yyuv}vT0a)3j$%F^~RYC zX;r)4v#>p36KP^{xo{g}6rM=TZOvUAl|s%pn2P_w58tWYKYy*4f=hmY_Kpf0HpDlQ z;F5tb5NQIkCc?)zuaF0PLQ3!oE22@W=*jzQ&!j9VW}c^*ecZP`mPuP#N~+E9n;Z1S zg#WnpBG~p%<$t%Cyc$3T1+}De`yy)3YcR9OvCJ$QL^_ny%W07%M2IK}aTn zm=h2o_7H9?u13@`zzB5>yKe>G8Zmb0;Y7Qs9;e~VTW_sB$ZXwxu@a1j?I;R1qwS}( zX60?x|9LOAJkP)65gMiZ#3zC2z5SAWO*BrY4}r!(iGE0HlU8#2h)9Y#Wa^}O#KqQ{ z?-G>^5vxewyo^0BdZ5@_ZaOPL(an4^yk^Nw1PrTWia};KMds1QyoD^g13&HRg+`y* z+-$DtvB|I7XRkZeUo|yQkJK*=eo5TWhGzE`fiTUnZ|*JgT0<3Xf-jq2;F<|x~RFF zkft+lcr&6k+(Yv68B|Q=(yL`ekr=ui5e;Gc3?EGO;T;=cicw5~ygS!Ph z8~^>VsL!&iNCjzhL;CyXlqEWTbIFgw?n`LJ$7yho3=HkK{Psz!{$i)<4Qpm_-6{a)BE_nX-P7EoDZe_@il)m7~)uNBqt8N$OC2UpJsa`m2c~ zMvG*fOZ;iw=pknT;IZ}=1it!<~B4c6Ow!S23c$%l62~ zx~?Qa5xXaq+yG{-a$>5@Ru&(WrGfmJCtn9JT=OVId>U~Umq3HJm}6HkSe9K6o|czW zI$qbdZ0z2~$mm~^FJG6BoL6soSzqE>B~z0Pl?)2u81(-vI>%b88KgnD6?wFEhaE#+ z;Z|e6lj%?e;el{&+N{}plo;7)W^EnH^!K{V+qL%$WB1N9H1YWQZN;!3S(6H?XpQFV z5-VXX7=K(j>DQ4BdF5=iq)gGewb~dcra_n&k#XJGo9CT*EBVIIcnhGMqy9pg z6|EW*e)htx9z{@T4(S>7_TcYuJ*D)k5%fMl)~tACtsA2BMZ?wKr*wMFRrlUmquLC) z4UM^kdMiQ`n;Fu?0Bi>>-{4A)#Ji?^hu$0tbgZ2__yNvM4`Vf&rt^xlw6Z%_n@+}B z@<{265Rl%;ES=yqt)$VDa0&}929n!!8gM~{b{M^Ra22C%1a}|@i)GTSXWL0wQ?ULd zxXOSh*O+{IPd~W9flp4vL$T>xB+L>)-e?mOt`0Q*ke3M)IS{eyMEbRp=qf4mgI9St zAOA3FB_L_i3KTWdBA7N2xfyaK@x`%%?UX&usXLp8A`SOh`dn`-KT)jc0qo#nkY?B(;5^hXz+tvnnDuTI z+}bYf1a*faE0(S!kJuB53UUh4zG`bSh%sE5y3;cktN9vMmb59M1DY60MBkw!MHbf> zZAz)hSqatpIaGg*C+pgA!g;mv1dtl75~e^8);5C55?8dPhw@*aynKVBjClFU6xl*y zG#IYD<)Y^isNQLWx$A;NPzQtstr)SBGG!pZQ+G~9%>yHEN=uO&Re!~;fuR$%a^miq zMO)Hr^;23NfaO?ic~eF@ zY}mnazei^RQBkg*Misb|$v?h-NB3Bc{B;-UV=$6;zv=$!6NW<-##ZkA_!N>6T-4LH z_DqaZH>m&R{8Zau=@0wt?KOoVZpZC0e|tK`2wv!xyC4lqq#~CcM*ac%oQ)9XGO#$` zF$vzzwjRMk+LU+3hqq!%yE zbT@>!%JlL)0UeAutqB>!wOG4Qm*-ioA!iQ~!^PQ8TUZ>dXDv$ci$D7z^2Zvgf{W@& zyd-dy*+(T!(%WL0j(BCTirZ^-D#8_oJ0WX{5x@{uI1K^G|WXD;$>Z5R?Oef#$+L-&Rp|)us2Cov$L7Y$BJFo z=37uRPf?q`%GlhI+Nwi%U7>})%wZYwf|oy|AWc7^%bkWM$;D&g{1+hlrsyt81F>!oF2kFl14g z4_rUj@C>skqHSKD-35;{ua;nwpvSlaCObPqme z#uC|oS*5jHQ8LWbODZqSN(I8-y?t(Xj>S6q2+8XrXvUn)&4;u&q;GmF#~=$1qMt7{ zUwZ=%O-q;&AL03IyvScmu%(Sc>moGFQhl7>Dxd8y*Qh-`}d`zh@lr}Qq3 zDtkma9g+)R%~3$=WW#Dxw9pkaiv4TQRQT=9I`+M+*2AL*ws$D{Hm+La z>UH7pViHjYzx;bE%4Fv2v2*671(bYbfTJKcLhFWQsR*ng`DHtT!>k%~(_!CY2)-@$ z`AKLy2)a&KLapa>!M3s*waY{8BWET`W|LcuSCwAescyGhLjRcP&ZE}=XOa6WwarRg z*SrGY4(8wqlIs`qR9Ce zOQN=3qe9)F#o&K>&C;E#o1II_{Lo5$%E{tytiVZ+;APIU8{~Zy(iHcUy;H2xZ%0L) zfX^kNEw~1%!ZLg90g?EvDmGk%d7DIkXXU-M1TbeR-bfkqJ)%hE>en7viH7!TFM9Q} zsq+nOCX+BU4%cs51n@h~(iWU~w~0!2>iX(jb|gM{)=cL*`l#Dy56o<+u1G){{r)U`Vd`kl8da zXBWei`L@_|)fA8Z`L! z#nMm)Y#=xV@c&osv#C~ftuQo#a=9T*K|hVm+>ORPcalXYACujjH{x)(h9}yJh`F)f z1Tv6}_4Azd+;}%&T_~)yy3M8PEa`vW-j36ZG{zegfwjS|YG__QfqnQ$Nw+sxIZ}M2 z*sxpOfRJRK6%Lzh9M6J)@vk>K#-80%2{EPWqHK5Ey|sHALQ&XF5#rIisjX1bJx~eQ zNRd9D8!go3K(dM>dEsWqn_yf1G_q*U&FyaJ6Ry;ul}gc8G>t8(+snZh34d|^{^;e! zW14MD7774=YR#f^fJ@BuM$RB(5(v*I7cw+ymX)`<=WRpOnSm{@VcEIU^NzC6$OM6_bs&0rH%%jbLW;9cmR3e7t7h5kvQ(RC4pr>n?An1#( z=%c91f)_zbA4!;^DQ&yiv;`-x@VM?izf(%OCC`2{Y66s>y9@E7fxL52UQ8|Ry}ESQ zF2r@;F+7`Q$?n6W)ww9lDOe(kXf1KPg#^i9Lgb%==CR|50NiO!84V4N2lb$DI6t7<3Vp@F}+@d4x} ze-pq3oSid!b&tJ1e89J|&|n)vcX&vid9VnB(HKfZUl?_KYlRGiq@WPlj9utp%H{5( z{*2WZj|Y&}h=7o*_Qj&B4a2T(iqv4!_Vi3g%}}If-Mn54cTb)0w{*nx#u|BAHl73B-tDyDZ)uVFDMmh35cK z%I&M^t7?uRcw|eFXfM(}x1;)+e?TkLjuBUtav(`xSt@P^X`YdpJKmXU&d7WtTkb{0 ziUH|^Vno0x(ev~yBmpQZc&*T!RKZF~FFaNncelK0P5u*mx~r^J-(uZIcV3u)@xr|h zZ>J}>Q(*G?0s_rS`VZoWIk8L`+UYj#Uy)kCtZ%DcYnjuQb&R1@YUDLITCf1ca6 zT5VQeKpfGQc%{`VN}FmNrlsFYi4qs~To8rZjxzrtpwMQy+mqhLO_A%%g81x+Z+qZL z|NGCsM&9jReoEhp8F^6yIxtVpmyX*OeGGgmQ7w`TxR8i;O!cH#IKlJc%SnOM!l)dQ z2Py{cOmvQkGzI*ouU2o?6j}g54M9#5XD=xP3cFcJ(O(>K=_Sp|d<^WAK?d0z<;Pe3 zZ!>c&Zj;HsD{eay+M~>0S=j~zs+yOAS5q`k6&@u+Zn5CR!Zx7fIgZupeY-G=Vq|2p z+#2r}j$HWIBUyrW;?#w1K$0^hEO=lwYh&@t=$0D`1KW^$m+ZNq2jVz5RO^$&_ z>&|+6F#g1eVonp`kyYYkbo>Sm0hg;XNmqLjD%~w3TyrP$i0`-xwm@>n%>3t@*+71a zc3mcgX`1I~-}L&EDdb4eSYiHNX`^aieE4=r>XI_;cw8M`Bty6Ww)hjkt_W4At?V7n zQqQPLDRHLU!Q-gGv53BXD8M(u`^OG>ZH^TnV4^#*NP&E`aEHa)E;gqaPJrilL&UMw zR`C+3%$#5IoQ&-OHCZ=?g&rjWVu0!(1&%HUQim!23(5IDghgicvnyRY@reDhOjPU! z=N@1c>cyT>S-$QLA41wl)-@wcMsj2_0Y5c$@5a|U+%k{j@y?LfN3i=`>~hx&5eCO98WsnE z%E(Bb1?RyMJs_ici^cy8nu05^HYB7u0h*_vrzHJtEJ=7P#s`G8+Ti{}JVsr$Kf$!g zY#M2^-;g5OheiZhCIv!?(B9}8==XyB<%o{mbS`rZH)1R}JYufx>zY+{Ngwyl_1F=` zCJBDY037tsqZNO8-}!zuoj8jfSdDQoCM{(Y%c)h9vHUCeWu*Kp8 z^%u!A>Cu;G!=&z9>_eIG~OqW?$y&bzIlXn^qX)rT5ydpWtzMY65Pgs(q z%iEO^0$)QNd7pi)*a3-r!TBD{Z9`gh%%H{vnn$cz3wY<1v)jpndx>R0%OL&8Fn zvr}UqIWyE9Uw)WjIhnXhMTvPr*52$3YVA@jCLevI&P(5?PdcS-GKX=B96lFlLHtHx zu7O&Qd&qK1ep41Z_WNuxmn@UVj-FV9wZ?l8LDR@QHrafDCq&eIFCqpgmWag@$UJ#b zub)co@=n891%vB$mJ^9?+ZywcpYfFDJ+TYOf*^A7O3ti;;-a6-!+@Sjxji+xubY3` zLcu@VQbLA9$vVX??`q1Krgb^1(c`Vr^|j)6@uSGm)J<^!^_#0H^z#H*J_g6)0aycM zq|}OAa-3Duev4!nU&0kxE_iL&$^8bSGc=6%I2$d+0(0?F&>-$|QJlM3b-A~ww_lSZ z!_;W^u=eNPxvd}ny$);Q___oBg#fJy*cpE%pV?h;K#v>RK>h_~Bg+cjQGVvSO8rTh z-^#-IuseUn02$|KSPy3rrl*$#>n5rgU9?{srKo*iYERNOW;xK^3H>-^iXILt3fGvbmE`4T=+Z zn|{HkBFFZ=vElGlD!vVTGG02r;rJpx_#ZP{b<8A$q%Uno;MEpw4fY8n_4WzDXVGJy z3%zfD=Fg(w3hNf<$|4K!AWx%L0vSz{63{>nWGn;!UxPg|DkCOkc$ctNRS91#dIx_VIPB7?G zlCTlL{*W;n< z%E|?$Y8i?Sqk~%ag=rC$hlDg^m-A~73HwF>bfmK1JS3 zmxTNoEn1hz2Pev{q!&;h@^;^g8_F6cc5*$J*z%i z^f?8$dltQzi1)e5s&ttf5rckhXOd@@zUG6lQoeAwKqr!KA^MyO@MOEy94zC52*810 zu|{rXyb%rV;5gc;BRbkmkMTCe8QeL9w2D%8{HYmg-A8&ZdwtkAM%osSd?Is5`4*NZ zGtn|6ZJ$`*A!BqqN@9QU1w89C{_wl78B{DAVLSCkZ*ru|W8V6pq5*;YJMHB;-GYXoZU(O{+1@BJKTuoYD(eMq4Gbls0&)>g){qp6D&�a9ypmO@;!C0P2o-z z{EC7S^S+9bWC^%aQ&g`}%TP{XQV@nqds;<;tnC6i{v9IkMYfViNJ*MNj=$&l0Yl{o z|1+!6N3G4>M$>5POTrB~ci^c6AF2)9E3{rtEl$Y##{3d7%!$;r*Hmy2m`mIrNDy2{ zO^H4IJOkQkc{$RdvX zuiyO1AIkLiy3E_;Yx3n=bo9%&>vh^vURM!#{W;>^SFVXB^PXWGcX>mv!q3Ui|N3Py zi6$@2EQ6*1EP;bZI#8;inN3l(-L+jlq$k~*&wrEwHG24!eJ!ycy%bXF9kOQ`pIB2! z7m~uCbJFv95~Zs1+hQ4zb1eRGbB>wQetH3uKpLS&4arbjnvJ;Z@8a) zJj9By{&u$ll@OwBlU!laIdPYi7p3aBmJ(C0mSD?oqq+h0;Um^2 zxEwykTJ9;dT}A}SXQ7|=h;kQ)DC`KYuw{wzM+5L{1x10A0FJZGi6WIlyLR}VAsm{} z`aZ66)tQZPGdu6P7R5mVT~xq+3QS~1rrRSNfXAVM`YypbwV;L_ux)7Q!r(5H&c)~@jS=m(!Q*yV$cs}kJGt2E9>Keo)H3G)!JQt9o<`#e^e z=Vy@|JBSanSZR~w<;ljf5+)5Tb);$bCZJ?F8ptQj5TA9@P-ULJ1$?H_4oj29;c9ad zl^;%eTndQGv94J@aDVsOSt=KLeJ&s&5n=5q)UY#h%!r;i7$y-Ka2V8<8x0`O8B%8! zHE+Y8`EkgO3rji?`N5ftbrQeUk*G&Xjlj{-8XK{?&9*0l;;K<_9C#SsXcO71>7je{KII z2v5;Pj*n86Yl+vglKN<`ZVW5KDWZ#grK$_nmPqAM!VgC0LQMtkker#ISBF1|_vsXX zfh{}8UGlx#gal*tC`^5*5iNA*Yn>UhelKgYUK)1JO zMOE*GTQIV!HiQQrQ$!+LMtf?58RMx(e;#dEH+i7w8*<^exPh#OE1b3|L+V_#4mtW< znm%wQzp*3^jW=Ffn65@$Ktxo5wYAlbp%4Usjq4b|0z!gZ#qc#!(9aJI&H)wILi_(bvnY@Hozg0T*lU&*&en%%xGURw{j?L*%i5h|I4 zy5~1*nDnSCK0SB@q_uS#dp?V71q4goU$ZVIJdm6vIwDEt0&E$EYfCkkU76x*PTPg8~dmdR6}BX@3v=bbon;z z`3g{&n-dySt`!uBc|o=aa}D7?k)|6@#M%WKaFt3&;~8x(Y0Zukeb|Q6p51@=2J;#c zXw(f^QtQlWW7mzuJpBnr2RJfKRBLp^Tu7!0=j1 ztvRPHGdiGx!b|IzBpA5-lDdt>m3aE~nG7ttr=8(uFAY0LJ~lpwAWpRR8c*=X*~MSdOVPv=Ci)vAfLFb6m6Nub0f1|1r$V?l^5}|rhk)^cL50c*vT-TrSFC8> zeBJ)k^$r1>Hc?KuGgT?c^V=NS*WY^;&${zkNzep&vC1gDuOTVi9*wG2K~JkN zfLxc%X?;HJ;>G%(si79igwl3s>Z%Q(sjQzuD5{PqP{?a;#g!Hwl?$>|f^7E1Z3Eyt z%!IuXW4O=TqM9<4X(Sp^nZ~i6{W9RoyF!6`a(Q(@7}73|@^(>4*2!|5y?J8EXJ&=Q zQpyh3=FyraUI#|o`7=r)*Dw{gw(QWh(Pr$*9l(X=Die;l+X<`6j znF2&w+5Sd<`0?fI*I&NAa1Lem@0Twxq6^`_r1pUSQ!Byb3v7(#*$c7Tzh5A%;a7Pn Xs+V7X{QEcex4-@W_>Z_M8m|HX4_n-< diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-live-before-fix.log.gz deleted file mode 100644 index 71b437d69c0b7995b59b0ccd9f9190248dbff64a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 589 zcmV-T0yn#_xKHy|tOa{~%2pbvUG@onhbrZ89= z6I(J`IUzhfE2q#54^SUu*_L*H-u*rS0Nix0-WuD$UYQW@8|PtTI)wNEoP}yMod#_@ z*1`FM;7zx#A50flYZX*Ebjl*^^0Y{EIM4b+e4DKvS6{x4=nqMy>MdHGrF-YM!6Q=U zu~LM|<$Rg+%Gm5B&MYQ?dt(i?s9_zD!WsteCaCYDbwe<{`HlJq1mznH=QIOjq39vc zY1$1BqqPxlv?jy=1mx?iGga#8g^)BQ6c{c!2Ff&6W5eS>CqT)3@$TZPDi&|6%SE-& z#Y_nQ9KV1VUBm&1tD_n$(Kf~+Nl9A_bzAC6N@F$tl5#?k(&D~s(4tqtI9q!BIdXNW z-0;wrOcb!>Hk6~STV)&6rN#{xB(8=uP2qm^@$=mWAvVf%BX5#IBz(OV0tM~H05PQy zq4lUVoV`x&GdRCmF6a4t27|JOb)`Xt6B%Z)(f_ez|4_1tB zqNn(C$DGiQ+)~xC2^a?yOmHx?2QszlfY2KxVO>B8tOnSd(8BuR+v8Jtv%3BC^su_U zdta;p?ap{-d$a*|${SU&ii2Z65%V56a&$-Vsz!0z(5T|qDXBQhmsgHmkk8smA?E@U zp}uC3N31x@!8F$S{|Q*Qso-frUU>0RQVp% diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz b/docs/evaluation/pr749-sds-foundation-2026-09-28/pr749-recovery-e2e.log.gz deleted file mode 100644 index 4365115a7d3e3547a52a76df5770cc24debb688a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 626 zcmV-&0*(D2iwFP!00002|E*L_Z`&{oz3W#HzO(^qJMKr?xY!}>T42}?ThpC}!I3Q{ z(PB%UL?_6G{r6E$){k2U#0Ocn$j8S=QUCxqwXIg#EMOxwh4q+>q-Z6Isl;?(!v@X#3(=mlq=N-x`=fk{y@aY?#l^0X}&v_kI-JjKNMXXak{ z*b|eEcL7p`Ww+q9eG^uUSY>KHJ|A6-hG%6t90=j|gtDHJ2n(pr$+E`6cTFSR6GWc3 z=)CrX>gmX_y>#S@ji#mTsHpWC`LDwT8#<=J{g5OH+)qAyx_d9gT0zXILHwkUyU!bv+)30X*Bnc0!c4gF8!bK=BHPq{2~|1alb*(2UQNg zqu^5`VU*8h?eQc+=d6qJXu3kGmQx-r1cyI($T)rmcFxN`Ck4#C?Z`(bkv6<3s@i%~ zQBJO7EKZU-&tN;yKf+kT9Jm`GBo4EOuhYlkW^()S@nLd%_b!_OnzeS;G-wFc(&?Zm zkmWh|ob(nXgmGetDikm49NPZC1rWPFWz(*veBWCM47nVhle}gjb~;}3KfN%lRKIOd z`mWz+m*L7HM(vfl=CA!+hk;E-=U#|jn!KNjJ={sim#MHyArj2u_T@zr$GzoCD~T&1EK>#0TlW4-Xj3Oqp>D*sG%ebRKn<8V|s*dlXWAR!Tdt~9Z2~W zX%^;SEG(1tDhP;YH(6_J3zIbz-vR%n+xI3{WFN&MGO$gdDUWpwlx-mlwprlcZIH`G*;tJ zDUVR3wD?@KXwfS&&K4ejOvYm<8V+5-hzN_u`C_tlr)-P5(AY2^u^N&jfyeE~&krAj zXq4$Eek6rh@VOL(K-x_NaikE`c~ly%E;IWIX1AO5^=dPRQCU;(*&&MHpIT`Y`adk` zUnuDjNjf;qMvymImdlM43#Bpmx0$))7izz2VtV2j%YTm#53}Ls$DD~;0`DIoUxCHHgXYhI^5$hpmIoowd*|hGCC&}s eL?S#!`r<4o7vn3f*g?FKrT7iro@e&(0{{RIap*|^ From be2b593f6fd918c2b10b4bdc42b41d383e75e29e Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:51:45 +0000 Subject: [PATCH 21/25] Normalize validation summary formatting --- docs/evaluation/pr749-sds-foundation-2026-09-28/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md index 35b776f96..f00cea979 100644 --- a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md +++ b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md @@ -42,4 +42,3 @@ transitional. Canonical semantic definitions, independent deployed-output identity and explicit logical dataset identity land in #774 using Planner #462; subsequent PRs complete shared execution and storage integration. The developer installation guide and PR description make that boundary explicit. - From 09cf67de17e7c01c83abed051dc64ec58a2892db Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 17:54:28 +0000 Subject: [PATCH 22/25] Remove PR process reports and defer execution design to shared-runtime PR --- docs/design_docs/precompute-dag-execution.md | 24 -------- .../pr749-final-sds-2026-09-28/README.md | 58 ------------------- .../pr749-final-sds-2026-09-28/cleanup.md | 35 ----------- .../pr749-sds-foundation-2026-09-28/README.md | 44 -------------- 4 files changed, 161 deletions(-) delete mode 100644 docs/design_docs/precompute-dag-execution.md delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/README.md delete mode 100644 docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md delete mode 100644 docs/evaluation/pr749-sds-foundation-2026-09-28/README.md diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md deleted file mode 100644 index 7504c948b..000000000 --- a/docs/design_docs/precompute-dag-execution.md +++ /dev/null @@ -1,24 +0,0 @@ -# Precompute execution from post-ASAP IR - -Audience: backend developers and reviewers of issue #762. - -The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. - -```mermaid -flowchart LR - P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] - I --> R[Raw source → SummaryAgg streaming kernel] - I --> M[Maintenance dependency scheduler] - R --> S[Stored summary frontier] - S --> M - S --> Q[Query projection and readout] - M --> S -``` - -For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. - -`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. - -Catalog schema version 3 carries Planner family in SDS. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. - -The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats. diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/README.md b/docs/evaluation/pr749-final-sds-2026-09-28/README.md deleted file mode 100644 index 3a91b999f..000000000 --- a/docs/evaluation/pr749-final-sds-2026-09-28/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# #749 final SDS contract validation - -Audience: implementation reviewers. This supersedes the earlier -[foundation report](../pr749-sds-foundation-2026-09-28/README.md). - -## Contract established in this PR - -#749 consumes Planner's canonical semantic export and binds it to an explicit -logical dataset identity. Catalog schema 6 separates semantic definitions from -deployed outputs; planning snapshot schema 3 requires dataset identity. -Hot and rebuild outputs can share meaning without sharing read authorization. -The typed Count/Rate support previously staged in #771 is included here because -collapsing those families produces conflicting semantic identities. - -An installed query resolves only its selected output within its plan version, -then checks definition, revision, format and coverage. Recovery accepts the same -installed generation; a new version must populate fresh state before serving it. -Ad-hoc semantic discovery and cross-version state adoption are not implemented. -Restricted native configuration helpers remain for explicit imported-state -fixtures; they do not establish a multi-dataset Planner binding. - -## Problems found before the fixes - -- Policy fingerprints coupled semantic identity to deployment routing and could - not express separate hot/rebuild outputs sharing one definition. -- Metric/table names alone could not distinguish equal expressions over - different logical datasets. -- Importing semantic definitions without typed Count/Rate support failed the - workload tests: one output claimed different definitions. -- A Planner API adapter could discard a join pruning contract. Unsupported - pruned relational joins now take the explicit fallback path; the vector - adapter validates the complete label-equality predicate. -- Old tests assumed Planner always selected a heap. Collector fixtures now - advertise the intended capabilities; exact backend candidates remain legal. -- A persistence test waited for any new disk part, which could belong to the - old series. It now waits for the newly allocated series' actual record. - -## Validation - -Passed locally on the final implementation: - -- Type, control-plane and data-plane libraries: **117 + 431 + 1,164 tests**. -- Installed-plan serving/transport integration: **13 tests**. -- Offline evidence integration: **6 tests**. -- Production-process restart/new-version warm-up: **1 test**. -- All-target strict Clippy for all three packages and workspace formatting. - -Identity collisions were exposed while extracting the contract ahead of typed -Count/Rate support. Generated logs are kept outside version control. -The production-process test covers same-version restart without re-ingestion, -new-version cold state, and fresh input becoming queryable in that version. -No production workload, production-cost, or independent human approval claim is -made. #728/#742/#775 retain their structural, synthetic-selection, and deployment -execution acceptance roles. - -## Obsolete-format cleanup - -See [cleanup and validation](cleanup.md) for removed wire adapters, strict metadata recovery, and downstream verification. diff --git a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md b/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md deleted file mode 100644 index a17d120b9..000000000 --- a/docs/evaluation/pr749-final-sds-2026-09-28/cleanup.md +++ /dev/null @@ -1,35 +0,0 @@ -# SDS obsolete-format cleanup - -This follow-up removes obsolete installed-plan and persistence compatibility paths. - -- Removed old identity and projection field aliases from installed plan contracts, along with superseded deployment API aliases. -- Removed `reused_from_generation`; unknown-field decoding rejects adoption metadata, and payload generation must still match the installed generation. -- Removed untyped projection decoders and the unused materialization JSON/byte adapter and serializer. Canonical typed Serde is the installed materialization format. -- Removed flat-map metadata migration and old-version readers. Only the branch's current sidecar schema is accepted (4 in #749/#774; 5 from #763 onward). -- Removed the recovery caller's log-and-return-zero fallback. Invalid persisted metadata propagates as an error; a missing sidecar is a fresh store. -- Made the producer partition roster explicit on the wire. An empty roster cannot authorize completion. -- Corrected stale recovery documentation and changed old-format acceptance tests into rejection tests. - -Current semantic checks for dataset, definition, state format, revision and coverage remain required. Low-level raw storage formats are distinct from installed SDS authorization; this change does not delete current storage operators or codecs. - -## Verification - -Foundation checks passed locally: - -| Check | Result | -| --- | --- | -| Type library | 117 passed | -| Control-plane library and HTTP API tests | 431 + 8 passed | -| Data-plane library | 1,165 passed | -| Serving integration | 13 passed | -| Restart and new-version warm-up process | 1 passed | -| Strict Clippy, all targets for the three crates | passed | - -Generated logs are kept outside version control. The restart process test covers same-version recovery without re-ingestion and new-version cold start followed by fresh input. Downstream verification checks the restacked Level 1/2 plans and current storage schema. - -No manual deployment verification or human approval is claimed. - -Downstream checks on the restacked #775 branch also passed: Level 1 (3 tests), -Level 2 (1 exhaustive cost-selection test), storage (332 tests), and serving -integration (13 tests). Workspace formatting passed. Concurrent remote fixture -fixes were merged and retained before push. diff --git a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md b/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md deleted file mode 100644 index f00cea979..000000000 --- a/docs/evaluation/pr749-sds-foundation-2026-09-28/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# PR #749 foundation rebase and SDS review - -The implementation is based on main `b7c0f08a`, including the merged #737 design. -Commit `79eb631a` preserves the prior implementation tree while removing the -already-squashed documentation history. Fixes are in `070fad6e` and `3c840ff9`. - -## Corrections - -- An unchanged definition previously authorized reading an older generation. - The compatibility map and read path are removed; adoption metadata is rejected. -- After restart, the persisted resolver could send fresh input to an excluded, - completed old series. The old physical address is reserved and fresh writes - allocate a new series. -- A live version change now authorizes the same fresh allocation without first - requiring the old series to be removed. - -Metadata, store-visibility and live-reactivation regressions fail before their -fixes. The process test also exposed the completed-series rejection before the -fresh-allocation fix. Generated failure logs are kept outside version control. - -## Validation - -On the corrected #749 code: - -- 112 type-library tests, 427 control-plane tests and 1,161 data-plane tests pass. -- The production-process test recovers the same version without re-ingestion, - verifies a new version is cold, then ingests fresh data and verifies its result. -- Strict all-target Clippy passes for those three packages. - -The data-plane suite and process test were rerun after the final allocation fix. -No production-cost or independent human approval claim is made. - -## Historical scope - -This report records the earlier foundation validation. The final SDS identity -contract and its new test results supersede the scope below; see -[final identity validation](../pr749-final-sds-2026-09-28/README.md). - -This is the plan/schema foundation described in #737's staged migration, not the -completed SDS implementation. Its policy-fingerprint-based schema remains -transitional. Canonical semantic definitions, independent deployed-output -identity and explicit logical dataset identity land in #774 using Planner #462; -subsequent PRs complete shared execution and storage integration. The developer -installation guide and PR description make that boundary explicit. From 2a0f8ed5dc1f56a4c26279048fd34540bbb43b51 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 18:10:11 +0000 Subject: [PATCH 23/25] Keep discovery and calibration on dataset-bound snapshot version 3 --- .../planning/repeated-dashboard-panes.md | 2 +- docs/evaluation/e2e-physical-dag.md | 2 +- docs/examples/workload-cost-evidence.md | 4 +-- tools/o11y-execution/CALIBRATION.md | 2 +- tools/o11y-execution/README.md | 2 +- tools/o11y-execution/calibrate.py | 3 +- tools/o11y-execution/discover_snapshot.py | 5 +-- tools/o11y-execution/test_calibrate.py | 35 +++++++++++++++++++ .../o11y-execution/test_discover_snapshot.py | 3 ++ tools/shared-workload/ACCURACY_E2E.md | 2 +- tools/shared-workload/planned_run.py | 2 +- tools/shared-workload/test_planned.py | 6 ++-- 12 files changed, 55 insertions(+), 13 deletions(-) diff --git a/docs/developer_docs/planning/repeated-dashboard-panes.md b/docs/developer_docs/planning/repeated-dashboard-panes.md index 119c2014d..36827f25b 100644 --- a/docs/developer_docs/planning/repeated-dashboard-panes.md +++ b/docs/developer_docs/planning/repeated-dashboard-panes.md @@ -23,7 +23,7 @@ pane width, without bypassing capability checks. Serialization does not confer compiler provenance: incoming quotes are always measured/provider evidence. Unquoted layouts use supplied lifecycle unit costs multiplied by structural counts. Layout changes never inherit another layout's measured scalar cost. -Workload-versus-exact deployment evidence is still required by snapshot version 2. +Workload-versus-exact deployment evidence is still required by snapshot version 3. Temporal requirements are derived from PromQL. A range selector supplies its own readout window; each rangeless source uses required diff --git a/docs/evaluation/e2e-physical-dag.md b/docs/evaluation/e2e-physical-dag.md index a9cbfe108..0bcc13a67 100644 --- a/docs/evaluation/e2e-physical-dag.md +++ b/docs/evaluation/e2e-physical-dag.md @@ -228,7 +228,7 @@ cargo build --locked -p control_plane --example compile_workload_artifact cargo build --locked -p data_plane --bin data_plane python3 tools/o11y-execution/replay.py \ --metrics /path/metrics.txt --queries /path/queries.json \ - --snapshot /path/costed-version2-snapshot.json \ + --snapshot /path/costed-version3-snapshot.json \ --compiler target/debug/examples/compile_workload_artifact \ --data-plane target/debug/data_plane \ --exact-url http://127.0.0.1:9090 --output /path/new-run diff --git a/docs/examples/workload-cost-evidence.md b/docs/examples/workload-cost-evidence.md index 64e60498c..fc5430d55 100644 --- a/docs/examples/workload-cost-evidence.md +++ b/docs/examples/workload-cost-evidence.md @@ -1,6 +1,6 @@ # Complete workload cost evidence -Planning snapshots use one schema, `snapshot_version: 2`. Version 1 is rejected. +Planning snapshots use one schema, `snapshot_version: 3`. Versions 1 and 2 are rejected; the environment must declare its logical dataset identity. Candidate discovery may omit `workload_cost_evidence`; compiling a deployable snapshot requires complete, valid quotes and selects by complete workload cost. There is no unquoted snapshot deployment path. The checked-in JSON examples are @@ -9,7 +9,7 @@ discovery templates, not ready-to-deploy plans. ## Workflow 1. Prepare the canonical workload, deployment capabilities and implementation - evidence as in `asapquery-planning-snapshot.json`. Set version 2. + evidence as in `asapquery-planning-snapshot.json`. Set version 3. 2. Obtain requirements without deploying: ```sh diff --git a/tools/o11y-execution/CALIBRATION.md b/tools/o11y-execution/CALIBRATION.md index b95ec262b..cefa91fec 100644 --- a/tools/o11y-execution/CALIBRATION.md +++ b/tools/o11y-execution/CALIBRATION.md @@ -13,7 +13,7 @@ compares its complete quotes and installs the selected artifact. Duplicate corpus occurrences increase their registered query frequency, so the manifest accounts for all 28 occurrences, not merely 24 unique strings. Provenance preserves input hashes and source counts. This discovery snapshot contains an **uncalibrated unit enumeration seed**, - not cost evidence; version 2 with no quotes cannot deploy it. + not cost evidence; version 3 with no quotes cannot deploy it. 2. Run `calibration_candidates DISCOVERY`. It calls the control plane and Planner, and exports every bindable candidate with its manifest and install request. Errors remain in the output. These artifacts are solely for calibration. diff --git a/tools/o11y-execution/README.md b/tools/o11y-execution/README.md index b95e487b5..46a10e662 100644 --- a/tools/o11y-execution/README.md +++ b/tools/o11y-execution/README.md @@ -15,7 +15,7 @@ No family override, candidate index, or benchmark-selected winner is accepted. Export the pinned upstream task queries, not the historical 27-query fixture. Provide generator revision, command and seed separately; a file hash cannot establish that provenance. -- A canonical **version 2** planning snapshot registering exactly the corpus's +- A canonical **version 3** planning snapshot registering exactly the corpus's unique queries and containing valid complete-workload provider cost evidence. Collect evidence using `workload_cost_manifest`; keep calibration inputs and evaluation inputs distinct. Do not reuse the demo snapshot's declared costs. diff --git a/tools/o11y-execution/calibrate.py b/tools/o11y-execution/calibrate.py index dc5e7d1a7..17ec9eada 100644 --- a/tools/o11y-execution/calibrate.py +++ b/tools/o11y-execution/calibrate.py @@ -101,10 +101,11 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() snapshot = json.loads(args.snapshot.read_text()) + if snapshot.get("snapshot_version") != 3: + raise ValueError("snapshot_version 3 with explicit dataset identity is required") evidence, audit = calibrate(json.loads(args.candidates.read_text()), json.loads(args.measurements.read_text()), "sha256:" + digest(args.metrics), snapshot["environment"]["observed_at_unix_ms"], snapshot["environment"]["max_evidence_age_ms"]) - snapshot["snapshot_version"] = 2 snapshot["workload_cost_evidence"] = evidence args.output.write_text(json.dumps(snapshot, indent=2, allow_nan=False) + "\n") args.output.with_suffix(".calibration.json").write_text(json.dumps(audit, indent=2, allow_nan=False) + "\n") diff --git a/tools/o11y-execution/discover_snapshot.py b/tools/o11y-execution/discover_snapshot.py index 9b2ec9a65..8cf15e7ad 100644 --- a/tools/o11y-execution/discover_snapshot.py +++ b/tools/o11y-execution/discover_snapshot.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Register the complete real corpus for candidate discovery, never cost selection. -Unit discovery costs are an explicit uncalibrated enumeration seed. Version 2 +Unit discovery costs are an explicit uncalibrated enumeration seed. Version 3 without quotes cannot select/deploy. Replace implementation evidence with measured calibration and re-export candidates before producing final deployment quotes. """ @@ -47,6 +47,8 @@ def main(): first_timestamp_ms = timestamp if first_timestamp_ms is None else first_timestamp_ms last_timestamp_ms = timestamp snapshot = json.loads(args.template.read_text()) + if snapshot.get("snapshot_version") != 3: + raise ValueError("snapshot_version 3 with explicit dataset identity is required") now = int(time.time() * 1000) input_span = (last_timestamp_ms - first_timestamp_ms) / 1000 horizon = args.repetitions * args.interval_ms / 1000 @@ -77,7 +79,6 @@ def evidence(value): "declared_interval_ms": interval, "evaluation_phase_ms": phase, "lookback_method": "derived by the backend compiler from PromQL and the declared scrape cadence"}) snapshot["query_workload"].update(repeating_queries=registrations, query_batch=None) - snapshot["snapshot_version"] = 2 snapshot.pop("workload_cost_evidence", None) implementation = snapshot["implementation"] implementation.pop("source_sample_interval_ms", None) diff --git a/tools/o11y-execution/test_calibrate.py b/tools/o11y-execution/test_calibrate.py index b98a77fbd..8b9eefc06 100644 --- a/tools/o11y-execution/test_calibrate.py +++ b/tools/o11y-execution/test_calibrate.py @@ -1,4 +1,9 @@ import copy +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile import unittest from calibrate import calibrate @@ -21,6 +26,36 @@ def setUp(self): "queries": {"q": {"cpu_ns": 100, "evaluations": 10, "classification": "warm", "correct": True, "raw_measurement_file": "q.json"}}}]} + def test_cli_preserves_current_snapshot_and_dataset_identity(self): + # Calibration adds quotes without downgrading or inventing dataset semantics. + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + metrics = root / "metrics.prom" + metrics.write_text("m 1\n") + measurements = copy.deepcopy(self.measurements) + measurements["data_snapshot_id"] = "sha256:" + hashlib.sha256(metrics.read_bytes()).hexdigest() + dataset = {"namespace": "test-tenant", "dataset": "metrics"} + snapshot = {"snapshot_version": 3, "environment": { + "observed_at_unix_ms": 1000, "max_evidence_age_ms": 1000, + "dataset_identity": dataset}} + for name, value in [("candidates", self.candidates), ("measurements", measurements), ("snapshot", snapshot)]: + (root / (name + ".json")).write_text(json.dumps(value)) + command = ["python3", str(Path(__file__).with_name("calibrate.py")), + "--candidates", str(root / "candidates.json"), + "--measurements", str(root / "measurements.json"), + "--metrics", str(metrics), "--snapshot", str(root / "snapshot.json"), + "--output", str(root / "output.json")] + subprocess.run(command, check=True, capture_output=True) + output = json.loads((root / "output.json").read_text()) + self.assertEqual(output["snapshot_version"], 3) + self.assertEqual(output["environment"]["dataset_identity"], dataset) + self.assertTrue(output["workload_cost_evidence"]["quotes"]) + snapshot["snapshot_version"] = 2 + (root / "snapshot.json").write_text(json.dumps(snapshot)) + result = subprocess.run(command, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("snapshot_version 3", result.stderr) + def run_provider(self): return calibrate(self.candidates, self.measurements, "d", 1000, 1000) diff --git a/tools/o11y-execution/test_discover_snapshot.py b/tools/o11y-execution/test_discover_snapshot.py index f195eb77b..e062430d0 100644 --- a/tools/o11y-execution/test_discover_snapshot.py +++ b/tools/o11y-execution/test_discover_snapshot.py @@ -24,6 +24,9 @@ def test_historical_input_does_not_backdate_plan_activation(self): "--template", str(template), "--output", str(output), "--repetitions", "1", ], check=True) snapshot = json.loads(output.read_text()) + self.assertEqual(snapshot["snapshot_version"], 3) + self.assertEqual(snapshot["environment"]["dataset_identity"], + json.loads(template.read_text())["environment"]["dataset_identity"]) self.assertEqual(snapshot["query_workload"]["repeating_queries"][0]["demand"], {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}) # Discovery output must obey the same schema as the compiler input. self.assertIsNone(snapshot["query_workload"]["repeating_queries"][0]["time_selection"].get("lookback")) diff --git a/tools/shared-workload/ACCURACY_E2E.md b/tools/shared-workload/ACCURACY_E2E.md index e7cd3619e..7867a534c 100644 --- a/tools/shared-workload/ACCURACY_E2E.md +++ b/tools/shared-workload/ACCURACY_E2E.md @@ -328,7 +328,7 @@ a scheduled live-load or throughput benchmark. `planned_run.py` connects one selected corpus query to the existing production planning/replay workflow. It owns fresh, separate Prometheus baseline/fallback -stores, invokes the normal compiler on measured version-2 cost evidence, installs +stores, invokes the normal compiler on measured version-3 cost evidence, installs its selected artifact, loads data, drains finite-input materialization and probes **every evaluation window** before timed replay. A readiness probe requires a nonempty warm response, positive summary-readout count and zero exact-subquery RPCs; diff --git a/tools/shared-workload/planned_run.py b/tools/shared-workload/planned_run.py index 2f7c57cdd..5e0c8b57c 100644 --- a/tools/shared-workload/planned_run.py +++ b/tools/shared-workload/planned_run.py @@ -48,7 +48,7 @@ def prepare(data, manifest, query_id, snapshot, end_ms, repetitions): registered = snapshot["query_workload"] if registered.get("query_batch") or {q["query"] for q in registered["repeating_queries"]} != {query["promql"]}: raise ValueError("costed snapshot must register exactly the selected query") - if snapshot.get("snapshot_version") != 2 or not snapshot.get("workload_cost_evidence", {}).get("quotes"): + if snapshot.get("snapshot_version") != 3 or not snapshot.get("workload_cost_evidence", {}).get("quotes"): raise ValueError("measured complete-workload cost evidence is required; discovery/demo costs are not deployment quotes") return query, {"upstream_revision": "shared-workload:" + hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest(), "queries": [{"id": query_id, "query": query["promql"], "eval_timestamp_ms": end}]} diff --git a/tools/shared-workload/test_planned.py b/tools/shared-workload/test_planned.py index 6b5bb327c..565caab6c 100644 --- a/tools/shared-workload/test_planned.py +++ b/tools/shared-workload/test_planned.py @@ -27,7 +27,7 @@ def test_series_ordered_trace_reaches_production_replay(self): manifest = accuracy_suite.corpus("google") (root / "queries.json").write_text(json.dumps(manifest)) (root / "snapshot.json").write_text(json.dumps({ - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": {"repeating_queries": [{"query": "sum_over_time(google_cluster_cpu_rate[1m])"}]}, "workload_cost_evidence": {"quotes": [{"test_placeholder": True}]}})) argv = ["planned_run.py", "--data", str(root / "data"), "--manifest", str(root / "queries.json"), @@ -73,11 +73,13 @@ def test_complete_window_and_costed_registration_are_required(self): root = Path(tmp) dataset.write(root / "data", dataset.synthetic(1, 1, 0, 120000), 2402, {"dataset": "synthetic"}) manifest = accuracy_suite.corpus("synthetic") - snapshot = {"snapshot_version": 2, "query_workload": {"repeating_queries": [{"query": "sum_over_time(fake_metric[1m])"}]}} + snapshot = {"snapshot_version": 3, "query_workload": {"repeating_queries": [{"query": "sum_over_time(fake_metric[1m])"}]}} qid = "synthetic/1m/False/temporal_sum" with self.assertRaisesRegex(ValueError, "cost evidence"): prepare(root / "data", manifest, qid, snapshot, None, 2) snapshot["workload_cost_evidence"] = {"quotes": [{"test_placeholder": True}]} + with self.assertRaisesRegex(ValueError, "cost evidence"): + prepare(root / "data", manifest, qid, {**snapshot, "snapshot_version": 2}, None, 2) _, corpus = prepare(root / "data", manifest, qid, snapshot, None, 2) self.assertEqual(corpus["queries"][0]["eval_timestamp_ms"], 120000) with self.assertRaisesRegex(ValueError, "full temporal history"): From 84e94688fb1d58d2b865d1e7009fc7673b2265f3 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 18:22:10 +0000 Subject: [PATCH 24/25] Use typed deployment configuration in process E2E fixtures --- .../tests/all_sketches_process_oracle_e2e.rs | 38 +++++++++++----- .../asapquery_compatibility_process_e2e.rs | 2 +- data_plane/tests/backend_process_e2e.rs | 12 ++--- .../tests/clickhouse_differential_e2e.rs | 10 +++-- data_plane/tests/component_process_e2e.rs | 44 +++++++++---------- .../disable_query_forwarding_process_e2e.rs | 6 ++- data_plane/tests/monitor_process_e2e.rs | 14 +++--- data_plane/tests/support/streaming_config.rs | 16 +++++++ 8 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 data_plane/tests/support/streaming_config.rs diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index cc2d34b06..9cee54846 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -109,22 +109,23 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(config_yaml: &str) -> Backend { +async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - config - .write_all(config_yaml.as_bytes()) - .expect("write streaming config"); - config.flush().expect("flush streaming config"); - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_str(config_yaml).unwrap(), + let mut install = + physical_fixture::artifact_from_materializations(vec![materialization.clone()]); + serde_yaml::to_writer( + &mut config, + &serde_json::json!({ + "precompute_plan": install.precompute_plan, + }), ) .unwrap(); + config.flush().unwrap(); let mut physical = tempfile::NamedTempFile::new().unwrap(); - let mut install = physical_fixture::artifact(&runtime); for rule in &mut install.transmission_plan.rules { if matches!( install @@ -294,9 +295,24 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { .collect() } -fn config(metric: &str, kind: &str, parameters: &str) -> String { - format!( - "aggregations:\n - aggregationType: {kind}\n aggregationSubType: ''\n labels:\n grouping: [service]\n rollup: []\n aggregated: []\n metric: {metric}\n parameters:\n{parameters}\n windowSize: 1\n windowType: tumbling\n spatialFilter: ''\n" +fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { + use asap_types::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + PrecomputeMaterialization::new( + kind.parse().unwrap(), + String::new(), + serde_yaml::from_str(parameters).unwrap(), + KeyByLabelNames::new(vec!["service".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 1, + 1, + WindowKind::Tumbling, + String::new(), + metric.into(), + None, + None, + None, ) } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 2afb897c1..b9205e03e 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -402,7 +402,7 @@ async fn certified_kll_state_to_query_oracle() { let mut bootstrap_config = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer( &mut bootstrap_config, - &serde_json::json!({"aggregations": []}), + &serde_json::json!({"precompute_plan": install.precompute_plan}), ) .unwrap(); let mut child = ChildGuard( diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 9c1082c54..40c24dd83 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -5,7 +5,9 @@ //! data plane, sends a modified-OTLP DDSketch, and verifies the resulting //! PromQL value. No server or planner is constructed in the test process. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -374,7 +376,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut bootstrap = tempfile::NamedTempFile::new().expect("create bootstrap config"); - writeln!(bootstrap, "aggregations: []").expect("write bootstrap config"); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let data_child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--streaming-config") @@ -547,9 +549,9 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { active["aggregation_count"], 1, "physical plan was not installed: {active}" ); - let installed_aggregation = active["streaming_config"]["aggregation_configs"] - .as_object() - .and_then(|configs| configs.values().next()) + let installed_aggregation = active["streaming_config"]["precompute_plan"]["materializations"] + .as_array() + .and_then(|configs| configs.first()) .expect("installed aggregation details"); let planned_alpha = installed_aggregation["parameters"]["alpha"] .as_f64() diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 64ad1e5e5..471b71530 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -2,9 +2,11 @@ //! //! Set `CLICKHOUSE_URL` (for example `http://127.0.0.1:8123`) to run it. +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::{ collections::HashMap, - io::Write, net::TcpListener, process::{Child, Command, Stdio}, sync::Arc, @@ -345,7 +347,7 @@ async fn run_mixed_aggregate(aggregate: &str) { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -540,7 +542,7 @@ async fn moving_windows_match_clickhouse_after_automatic_publication() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -724,7 +726,7 @@ async fn collection_sql_executes_local_elements_after_typed_exact_leaf() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 220cdef85..8f0d89dc6 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -7,7 +7,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -125,30 +124,29 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: component_process_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), + let materialization = asap_types::PrecomputeMaterialization::new( + asap_types::AggregationType::DDSketch, + String::new(), + std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(0.01))]), + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::WindowKind::Tumbling, + String::new(), + "component_process_e2e_latency_ms".into(), + None, + None, + None, + ); + let install = physical_fixture::artifact_from_materializations(vec![materialization]); + serde_yaml::to_writer( + &mut config, + &serde_json::json!({"precompute_plan": install.precompute_plan}), ) .unwrap(); - let install = physical_fixture::artifact(&runtime); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); diff --git a/data_plane/tests/disable_query_forwarding_process_e2e.rs b/data_plane/tests/disable_query_forwarding_process_e2e.rs index 7c8c65855..529134fc9 100644 --- a/data_plane/tests/disable_query_forwarding_process_e2e.rs +++ b/data_plane/tests/disable_query_forwarding_process_e2e.rs @@ -1,6 +1,8 @@ //! The production CLI's no-forwarding mode keeps query traffic inside the backend. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::sync::{ @@ -61,7 +63,7 @@ async fn cli_mode_blocks_instant_and_range_forwarding() { }); let mut config = tempfile::NamedTempFile::new().unwrap(); - writeln!(config, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut config, &streaming_config::empty()).unwrap(); let output = tempfile::tempdir().unwrap(); let port = unused_port(); let mut child = ChildGuard( diff --git a/data_plane/tests/monitor_process_e2e.rs b/data_plane/tests/monitor_process_e2e.rs index bf7bedbfd..2a515ad0b 100644 --- a/data_plane/tests/monitor_process_e2e.rs +++ b/data_plane/tests/monitor_process_e2e.rs @@ -4,7 +4,9 @@ //! data-plane executable, register, report different rates, and receive //! differentiated sampling grants over bidirectional gRPC streams. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -110,11 +112,11 @@ async fn production_coordinator_differentiates_edge_sampling_grants() { let monitor_port = unused_port(); let output_dir = tempfile::tempdir().expect("create output directory"); let mut config = tempfile::NamedTempFile::new().expect("create monitor config"); - write!( - config, - "aggregations: []\nmonitors:\n - agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n" - ) - .expect("write monitor config"); + let mut startup = streaming_config::empty(); + startup["monitors"] = serde_json::json!([{ + "agg_id": 1, "key": "", "tau": 5000.0, "epsilon": 0.05, "window_ms": 60000 + }]); + serde_yaml::to_writer(&mut config, &startup).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--streaming-config") diff --git a/data_plane/tests/support/streaming_config.rs b/data_plane/tests/support/streaming_config.rs new file mode 100644 index 000000000..ccd35c12e --- /dev/null +++ b/data_plane/tests/support/streaming_config.rs @@ -0,0 +1,16 @@ +//! Current typed startup configuration for processes awaiting plan installation. +pub fn empty() -> serde_json::Value { + use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "process-fixture".into(), + }; + let plan = asap_types::precompute_plan::PrecomputePlan::build(envelope, vec![], &[]).unwrap(); + serde_json::json!({"precompute_plan": plan}) +} From c11a540d3f77cd48e6474b1e6280a5f2ba9a26db Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 18:31:08 +0000 Subject: [PATCH 25/25] Align process assertions with dataset-bound SDS and cold successor activation --- .../asapquery_compatibility_process_e2e.rs | 54 +++++++++++++++---- data_plane/tests/backend_process_e2e.rs | 38 ++++++++++--- .../tests/support/current_series_process.rs | 1 - 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index b9205e03e..99fae4b0a 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -651,10 +651,47 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg query.accuracy_target.clone(), ) .unwrap(); - let model = control_plane::physical::post_asap::cost_model::ForcedFamilyCostModel::new( - query.accuracy_target.clone(), - algorithm.clone(), - ); + // Family ordering alone does not select a whole DAG. Price the desired + // legal heap candidate explicitly so this process test exercises its runtime. + struct HeapFixtureCost(planner_types::post_asap::SketchAlgorithm); + impl asap_aware_mapping::CostModel for HeapFixtureCost { + fn rank_candidates( + &self, + _: &planner_types::pre_asap::AggIntent, + candidates: &[planner_types::post_asap::SketchAlgorithm], + ) -> Vec { + let mut ranked = candidates.to_vec(); + ranked.sort_by_key(|kind| kind != &self.0); + ranked + } + fn candidate_cost( + &self, + candidate: &asap_aware_mapping::ReplacementSubDAG, + _: &asap_aware_mapping::TargetSubDAG<'_>, + ) -> Option { + let heap = + if let asap_aware_mapping::Replacement::Summary(root) = &candidate.replacement { + planner_types::post_asap::compile_executable_dag(root) + .unwrap() + .nodes + .iter() + .any(|node| { + matches!(&node.payload, + planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), .. + } if kind.algorithm() == &self.0) + }) + } else { + false + }; + Some(asap_aware_mapping::cost_model::Cost(if heap { + 1.0 + } else { + 1e12 + })) + } + } + let model = HeapFixtureCost(algorithm.clone()); query.selected_plan_root = control_plane::planner_selection::select_query_with_models( &expr, &model, @@ -942,7 +979,6 @@ async fn run_shared_dashboard(multi_pane: bool) { } }) .collect(); - typed.schema_version = 2; typed.workload_cost_evidence = Some( control_plane::physical::workload_cost::WorkloadCostEvidence { backend_revision: control_plane::physical::compiler::BACKEND_REVISION.into(), @@ -1547,12 +1583,12 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() Some(1) ); assert_eq!( - topk_sum["data"]["result"][0]["metric"]["item"], - "asap_demo_gauge{job=\"worker\"}" + topk_sum["data"]["result"][0]["metric"]["job"], "worker", + "TopK must preserve the selected series labels: {topk_sum}" ); assert_eq!( - topk_count["data"]["result"][0]["metric"]["item"], - "asap_demo_gauge{job=\"api\"}" + topk_count["data"]["result"][0]["metric"]["job"], "api", + "TopK must preserve the selected series labels: {topk_count}" ); assert!((first_value(&sum, "value").expect("sum value") - 240.0).abs() < 1e-9); diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 40c24dd83..910025999 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -478,6 +478,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { } } }], + "dataset_identity": {"namespace": "process-e2e", "dataset": "metrics"}, "collector_ids": ["whole-e2e-collector"], "capability_snapshot_id": "whole-e2e-capabilities", "evidence": {}, @@ -725,8 +726,9 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!(first_scalar(&still_warm), Some(value)); - // Retry the staged successor while queries are in flight. Each - // request must retain a complete active snapshot through cutover. + // Retry while queries are in flight. Old-generation reads may finish, + // but the successor must stay cold until its own output is published. + // Reusing the old payload would violate StoredOutputReference identity. request["activation_unix_ms"] = (std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() @@ -749,11 +751,12 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .json() .await .unwrap(); - assert_eq!( - first_scalar(&response), - Some(value), - "torn serving snapshot: {response}" - ); + if let Some(actual) = first_scalar(&response) { + assert_eq!(actual, value, "incorrect old-generation result: {response}"); + } else { + assert_eq!(response["status"], "error", "{response}"); + assert_eq!(response["error"], "No result for query", "{response}"); + } tokio::time::sleep(Duration::from_millis(10)).await; } }); @@ -776,6 +779,27 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { assert_eq!(activated["plan_version"], 2); let (successor, _collector_socket) = collector.await.unwrap(); readers.await.unwrap(); + let cold_successor: serde_json::Value = client + .get(format!("{data_base}/api/v1/query")) + .query(&[ + ("query", query.to_string()), + ("time", (window_end_ms as f64 / 1000.0).to_string()), + ]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(cold_successor["status"], "error", "{cold_successor}"); + assert_eq!( + cold_successor["error"], "No result for query", + "{cold_successor}" + ); + assert!( + first_scalar(&cold_successor).is_none(), + "successor reused old state" + ); let old_frame = client .post(format!("http://{otlp_http}/v1/metrics")) .header("content-type", "application/x-protobuf") diff --git a/data_plane/tests/support/current_series_process.rs b/data_plane/tests/support/current_series_process.rs index bc7f1140d..4f0dc6992 100644 --- a/data_plane/tests/support/current_series_process.rs +++ b/data_plane/tests/support/current_series_process.rs @@ -27,7 +27,6 @@ async fn current_series_quantiles_topk_share_and_replace_values() { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - snapshot.schema_version = 2; // Exercise a short declared horizon; native differential mode keeps its five-minute contract. let horizon_ms: i64 = if native.is_some() { 300_000 } else { 5_000 }; let scrape_ms = horizon_ms / 5;