diff --git a/.gitignore b/.gitignore index a90758c77..0461e0072 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ uuid /store/ .claude/ + +# Generated evaluation archives stay outside version control. +/docs/evaluation/**/*.gz diff --git a/Cargo.lock b/Cargo.lock index 4ba3a108d..acc50d509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -393,30 +393,19 @@ dependencies = [ "serde_json", ] -[[package]] -name = "asap-precompute-rs" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPCollector?branch=main#1d8efd07e40fc151cbd4678a5c6aa9774b1aed34" -dependencies = [ - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", - "prost", - "serde", - "serde_json", - "thiserror 1.0.69", -] - [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cd7e9e0f710816d49190dabd6c789359067a208f#cd7e9e0f710816d49190dabd6c789359067a208f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bccc837f5caf21c888b2cce73c64a1be283b679a#bccc837f5caf21c888b2cce73c64a1be283b679a" dependencies = [ "serde", "serde_json", + "sha2", "thiserror 2.0.20", ] @@ -431,6 +420,14 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "asap_sketch_codec" +version = "0.1.0" +dependencies = [ + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", + "prost", +] + [[package]] name = "asap_sketchlib" version = "0.3.0" @@ -1159,9 +1156,9 @@ dependencies = [ "arrow", "asap-aware-mapping", "asap-frontend-promql", - "asap-precompute-rs", "asap-types", "asap_otel_proto", + "asap_sketch_codec", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", "asap_types", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index eaca35231..b5592550e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "crates/asap_otel_proto", "crates/asap_types", + "crates/asap_sketch_codec", "data_plane", "control_plane", ] @@ -11,19 +12,13 @@ members = [ edition = "2021" version = "0.1.0" -# ASAPCollector's `asap-precompute-rs` currently declares Sketchlib as a -# relative path. When Collector is consumed from Git, resolve that dependency -# to the same Git-sourced Sketchlib package as the backend. -[patch."https://github.com/ProjectASAP/ASAPCollector"] -asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch = "main" } - [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cd7e9e0f710816d49190dabd6c789359067a208f" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "bccc837f5caf21c888b2cce73c64a1be283b679a" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } diff --git a/README.md b/README.md index 3293c7f8a..f9391cbc1 100644 --- a/README.md +++ b/README.md @@ -241,8 +241,8 @@ git -C ../ASAPCollector checkout main [MVP CI](.github/workflows/mvp-ci.yml) is the source for compatible dependency checkouts; currently Collector uses its default branch. For reproducible runs, -record all three exact revisions. ASAPPlanner is fetched at the revision pinned -in Cargo manifests; do not substitute an unrelated local planner checkout. +record the exact revisions. ASAPPlanner is fetched at the revision pinned in +the Cargo manifests; do not substitute an unrelated local planner checkout. ### 2. Check prerequisites and build diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index ceaa441f2..d7488a927 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -69,7 +69,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { ) .unwrap(); let reference = sds.reference().unwrap(); - precompute_plan.summary_catalog = Some(reference.clone()); + precompute_plan.bind_catalog(&sds).unwrap(); transmission_plan.summary_catalog = Some(reference); ClickHouseSqlWorkload { summary_catalog: sds, diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index f2894c4fb..5c95608bc 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -1,6 +1,6 @@ //! Export every bindable candidate for isolated measurement, without selecting a winner. use control_plane::physical::{ - compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}, + compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}, workload_cost, }; use planner_types::post_asap::{SummaryExpr, SummaryNode}; @@ -37,17 +37,6 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation vec![lhs, rhs], json!({"operator_debug":format!("{operator:?}"),"timing_debug":format!("{timing:?}")}), ), - SummaryExpr::CandidateTopK { - candidates, - values, - k, - grouping, - completeness, - } => ( - "CandidateTopK", - vec![candidates, values], - json!({"k":k,"grouping_debug":format!("{grouping:?}"),"completeness_debug":format!("{completeness:?}")}), - ), SummaryExpr::ValueOperation { child, operation, @@ -84,10 +73,11 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation right, kind, pred, + pruning, } => ( "RelationalJoin", vec![left, right], - json!({"kind_debug":format!("{kind:?}"),"predicate_debug":format!("{pred:?}")}), + json!({"kind_debug":format!("{kind:?}"),"predicate_debug":format!("{pred:?}"), "pruning_debug":format!("{pruning:?}")}), ), SummaryExpr::SummarySubtract { left, right } => { ("SummarySubtract", vec![left, right], json!({})) @@ -105,7 +95,7 @@ fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilation vec![summary_input], json!({"query_debug":format!("{query:?}")}), ), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { ("SummaryMerge", children.iter().collect(), json!({})) } }; @@ -138,9 +128,9 @@ fn main() -> Result<(), Box> { let enabled_materialization_keys = candidate.enabled_materialization_keys.clone(); let planner_selected_queries = planner_forest(&queries); let compiled = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate, environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate, environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) }; let plan = match compiled { Ok(plan) => plan, diff --git a/control_plane/examples/offline_planner_replay.rs b/control_plane/examples/offline_planner_replay.rs index 7a703ce18..78b6e1dd0 100644 --- a/control_plane/examples/offline_planner_replay.rs +++ b/control_plane/examples/offline_planner_replay.rs @@ -59,7 +59,7 @@ fn inspect( inspect(summary_input, model, seen, states, raw) } SummaryExpr::ValueOperation { child, .. } => inspect(child, model, seen, states, raw), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { inspect(child, model, seen, states, raw); } @@ -82,12 +82,6 @@ fn inspect( inspect(lhs, model, seen, states, raw); inspect(rhs, model, seen, states, raw); } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - inspect(candidates, model, seen, states, raw); - inspect(values, model, seen, states, raw); - } } } diff --git a/control_plane/examples/workload_cost_manifest.rs b/control_plane/examples/workload_cost_manifest.rs index 142f4a07c..f01fd0574 100644 --- a/control_plane/examples/workload_cost_manifest.rs +++ b/control_plane/examples/workload_cost_manifest.rs @@ -1,6 +1,6 @@ //! Emit pricing requirements; never fabricate quotes or publish a plan. use control_plane::physical::{ - compiler::BackendLocalPlanningInput, compiler::PhysicalPlanCompiler, workload_cost, + compiler::BackendLocalPlanningInput, compiler::DeploymentPlanCompiler, workload_cost, }; fn main() -> Result<(), Box> { @@ -14,7 +14,7 @@ fn main() -> Result<(), Box> { .into_iter() .filter_map(|candidate| { let queries = candidate.queries.clone(); - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(candidate, environment.clone()) .and_then(|plan| workload_cost::manifest(&plan, &queries)) .ok() diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f1e1b1df0..5a3c92eb1 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -204,8 +204,9 @@ fn moving_window(canonical: &QueryExpr) -> Option<(String, (u64, u64))> { pub use asap_frontend_sql::SqlCatalog as ClickHouseSqlCatalog; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ClickHouseSqlWorkload { - #[serde(rename = "sds", alias = "summary_catalog")] + #[serde(rename = "sds")] pub summary_catalog: SummaryCatalog, pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, @@ -247,6 +248,7 @@ pub async fn compile_automatic_clickhouse_workload( let mut entries = std::collections::BTreeMap::new(); let mut window_templates = std::collections::BTreeMap::>::new(); let mut installed_dags = std::collections::BTreeMap::new(); + let mut selected_dags = std::collections::BTreeMap::new(); let mut materializations = std::collections::BTreeMap::new(); let mut selection_traces = std::collections::BTreeMap::new(); for query in &request.queries { @@ -267,6 +269,9 @@ pub async fn compile_automatic_clickhouse_workload( ) .then_some(config.slide_interval.saturating_mul(1_000)), materialization: config.policy_fingerprint().into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::Reduce(config.grouping_labels.names()), window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, @@ -286,7 +291,13 @@ pub async fn compile_automatic_clickhouse_workload( "duplicate canonical SQL query identity".into(), )); } - installed_dags.insert(query.sql.clone(), installed); + selected_dags.insert(query.sql.clone(), installed.document.clone()); + installed_dags.insert( + query.sql.clone(), + installed + .maintenance_projection() + .map_err(ClickHousePlanningError::Lower)?, + ); } let configs: Vec<_> = materializations.into_values().collect(); let sds = SummaryCatalog::from_materializations( @@ -297,11 +308,10 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut precompute = PrecomputePlan::build_backend_local(request.envelope.clone(), configs) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - precompute.summary_catalog = Some( - sds.reference() - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, - ); precompute.executable_dags = installed_dags; + precompute + .bind_catalog(&sds) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut transmission = crate::physical::compiler::build_transmission_plan( request.envelope.clone(), &precompute, @@ -309,7 +319,7 @@ pub async fn compile_automatic_clickhouse_workload( ) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; transmission.summary_catalog = precompute.summary_catalog.clone(); - let publication = crate::physical::publication::PhysicalPlanPublication { + let mut publication = crate::physical::publication::PhysicalPlanPublication { summary_catalog: sds, precompute_plan: precompute, collector_plans: Vec::new(), @@ -322,9 +332,14 @@ pub async fn compile_automatic_clickhouse_workload( tables: request.tables.clone(), accuracy: request.accuracy.clone(), }), + selected_dags, entries, }, }; + publication + .query_plan + .bind_catalog(&publication.summary_catalog) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; publication .validate() .map_err(ClickHousePlanningError::Lower)?; @@ -378,7 +393,7 @@ fn materialize_selected_sql( let aggregation = BackendAggregation { aggregation_id: String::new(), metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: crate::physical::compiler::physical_materialization_family(family), + family: family.clone(), window_secs, spatial_filter: String::new(), grouping: grouping.names(), @@ -423,6 +438,7 @@ pub async fn compile_clickhouse_workload( let mut entries = std::collections::BTreeMap::new(); let mut window_templates = std::collections::BTreeMap::>::new(); let mut installed_dags = std::collections::BTreeMap::new(); + let mut selected_dags = std::collections::BTreeMap::new(); for query in &request.queries { let planned = plan_clickhouse_sql(&query.sql, &catalog, request.accuracy.clone()).await?; let template = planned.canonical_sql.clone(); @@ -430,7 +446,13 @@ pub async fn compile_clickhouse_workload( bind_selected_node(node, family, query, request) })?; index_sql_template(&mut window_templates, template, &executable); - installed_dags.insert(query.sql.clone(), installed); + selected_dags.insert(query.sql.clone(), installed.document.clone()); + installed_dags.insert( + query.sql.clone(), + installed + .maintenance_projection() + .map_err(ClickHousePlanningError::Lower)?, + ); let identity = QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &executable.canonical_query); if entries.insert(identity.clone(), executable).is_some() { @@ -441,7 +463,7 @@ pub async fn compile_clickhouse_workload( } let mut precompute_plan = request.precompute_plan.clone(); precompute_plan.executable_dags = installed_dags; - let publication = crate::physical::publication::PhysicalPlanPublication { + let mut publication = crate::physical::publication::PhysicalPlanPublication { summary_catalog: request.summary_catalog.clone(), precompute_plan, collector_plans: Vec::new(), @@ -454,9 +476,14 @@ pub async fn compile_clickhouse_workload( tables: request.tables.clone(), accuracy: request.accuracy.clone(), }), + selected_dags, entries, }, }; + publication + .query_plan + .bind_catalog(&publication.summary_catalog) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; publication .validate() .map_err(ClickHousePlanningError::Lower)?; @@ -591,7 +618,7 @@ fn bind_selected_node( .. } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = crate::physical::compiler::physical_materialization_family(family); + let expected = family.clone(); let selected = select_materialization( &request.precompute_plan.materializations, &table_ref, @@ -612,6 +639,9 @@ fn bind_selected_node( ) .then_some(selected.slide_interval.saturating_mul(1_000)), materialization: selected.policy_fingerprint().into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + selected.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::Reduce(selected.grouping_labels.names()), window_ms: selected.stored_window_ms(), pane_origin_ms: selected.pane_origin_ms, @@ -1405,7 +1435,7 @@ mod tests { }; let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - precompute.summary_catalog = Some(sds.reference().unwrap()); + precompute.bind_catalog(&sds).unwrap(); let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -1641,10 +1671,16 @@ mod tests { installed.binding.nodes.len(), installed.document.nodes.len() ); - assert!(installed.binding.nodes.values().any(|binding| matches!( + assert!(!installed.binding.nodes.values().any(|binding| matches!( binding, crate::physical::executable_binding::BackendNodeBinding::Query { .. } ))); + assert!( + publication.query_plan.selected_dags[&request.queries[0].sql] + .nodes + .len() + > installed.document.nodes.len() + ); let entry = publication.query_plan.entries.values().next().unwrap(); // External SQL retains its literal time range until it can be bound. assert!(!entry.canonical_query.starts_with("moving-window-v1:")); @@ -1717,8 +1753,10 @@ mod tests { let envelope = request.precompute_plan.envelope.clone(); request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - request.precompute_plan.summary_catalog = - Some(request.summary_catalog.reference().unwrap()); + request + .precompute_plan + .bind_catalog(&request.summary_catalog) + .unwrap(); request.transmission_plan = crate::physical::compiler::build_transmission_plan( envelope, &request.precompute_plan, diff --git a/control_plane/src/emit/backend_wire.rs b/control_plane/src/emit/backend_wire.rs index 829d1be41..ff294e8ce 100644 --- a/control_plane/src/emit/backend_wire.rs +++ b/control_plane/src/emit/backend_wire.rs @@ -5,7 +5,7 @@ //! * the storage-routing table, which maps each metric's materialized summary //! families to the query shapes the ASAP tier serves natively versus the //! ones that belong to the archive; -//! * the aggregation and readout JSON the backend's `AggregationConfig` +//! * the aggregation and readout JSON the backend's `PrecomputeMaterialization` //! parser consumes. //! //! `backend_plan::from_stage_config` reuses [`build_backend_aggregation_json`] diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c2ede7ee3..bf51c16e5 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1,7 +1,7 @@ //! Backend-facing emission for a compiled physical plan. //! //! * [`backend_wire`] builds the storage-routing table and the aggregation / -//! readout JSON the backend's `AggregationConfig` parser consumes. +//! readout JSON the backend's `PrecomputeMaterialization` parser consumes. //! * [`monitor`] carries the CDM monitor declarations. pub mod backend_wire; @@ -54,11 +54,8 @@ fn extract_from_node(node: &Rc) -> Option { // `ExactAgg` case. SummaryExpr::SummaryAgg { .. } => None, SummaryExpr::SummaryEstimate { summary_input, .. } => extract_from_node(summary_input), - SummaryExpr::SummaryMerge { children } => children.iter().find_map(extract_from_node), + SummaryExpr::SummaryMerge { children, .. } => children.iter().find_map(extract_from_node), SummaryExpr::ValueOperation { child, .. } => extract_from_node(child), - SummaryExpr::CandidateTopK { - candidates, values, .. - } => extract_from_node(candidates).or_else(|| extract_from_node(values)), // Not surfaced by any `Bind*` path yet (gated on rules that // haven't landed — see `deployment_expr.rs`'s module docs). SummaryExpr::BinaryOp { .. } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index b0b3bf860..edd4afe08 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -199,7 +199,8 @@ struct CompileAndPublishPhysicalPlanRequest { workload_cost_evidence: Option, queries: Vec, data_workload: planner_types::workload::DataWorkload, - #[serde(rename = "collector_ids", alias = "target_collector_ids")] + dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, + #[serde(rename = "collector_ids")] target_collector_ids: Vec, capability_snapshot_id: String, #[serde(default)] @@ -234,13 +235,13 @@ use physical::compiler::QueryFrontend; #[derive(Debug, Serialize)] struct CompileAndPublishPhysicalPlanResponse { cost_comparison: Option, - #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + #[serde(rename = "logical_selection")] planner_selection_trace: Vec, plan_id: u64, plan_version: u64, status: &'static str, generated_at_unix_ms: u64, - #[serde(rename = "collector_ids", alias = "target_collector_ids")] + #[serde(rename = "collector_ids")] target_collector_ids: Vec, lifecycle_estimates: Vec, } @@ -630,6 +631,7 @@ fn compile_physical_plan_request( retained_summary_memory_budget_bytes: None, }; let environment = physical::compiler::PhysicalDeploymentContext { + dataset_identity: request.dataset_identity, target: request.target, target_collector_ids: request.target_collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, @@ -852,6 +854,7 @@ mod api_tests { "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": snapshot.physical_inputs.window_cost_model }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, @@ -929,6 +932,7 @@ mod api_tests { "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": [], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs index 073ea1d6f..bc60cf45a 100644 --- a/control_plane/src/physical/backend_stage.rs +++ b/control_plane/src/physical/backend_stage.rs @@ -1,7 +1,7 @@ //! Backend-facing projection of one planning cycle. //! //! These types are the input to [`crate::backend_plan::from_stage_config`] and -//! to `emit::backend_wire`'s backend JSON builders. `PhysicalPlanCompiler` builds +//! to `emit::backend_wire`'s backend JSON builders. `DeploymentPlanCompiler` builds //! them directly from the summaries ASAPPlanner selected. //! //! They are deliberately not `Serialize`/`Deserialize`: `SummaryFamilyType` @@ -35,7 +35,7 @@ pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, /// Source metric the aggregation runs over. Required by the backend's - /// `AggregationConfig` parser. + /// `PrecomputeMaterialization` parser. pub metric_name: String, /// Planner-owned committed summary identity. Sketch entries carry a /// validated `SketchKind` (category + algorithm + params); exact entries diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 69254a730..cf9d6ea56 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -199,6 +199,8 @@ pub struct TopKMembershipEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PhysicalDeploymentContext { + /// Semantic dataset served by this deployment's input channel; never an endpoint. + pub dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, pub target: PhysicalDeploymentTarget, #[serde(rename = "collector_ids")] pub target_collector_ids: Vec, @@ -219,7 +221,7 @@ pub enum PhysicalDeploymentTarget { } /// Startup and candidate-discovery input for backend-local planning. -/// Version 2 is the sole supported schema; deployment always requires quotes. +/// Version 3 requires an explicit logical dataset identity; it is the sole supported schema; deployment always requires quotes. /// Query/data semantics use ASAPPlanner's canonical workload types directly; /// this wrapper adds only backend-owned implementation evidence and lifecycle /// identity required to choose a concrete physical realization. @@ -472,7 +474,7 @@ pub struct CompiledPhysicalPlan { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationLifecycleEstimate { - pub materialization: asap_types::sds::SummaryDefinitionId, + pub materialization: asap_types::sds::StoredOutputId, pub consumer_query_ids: Vec, #[serde(rename = "window_implementation_id")] pub window_realization_id: String, @@ -527,7 +529,7 @@ impl AccuracyEvidenceProvider for QueryEvidence<'_> { } #[derive(Debug, Default)] -pub struct PhysicalPlanCompiler; +pub struct DeploymentPlanCompiler; impl BackendLocalPlanningInput { /// Invoke the pinned Planner from canonical startup workloads and compile @@ -569,9 +571,9 @@ impl BackendLocalPlanningInput { pub fn into_physical_compilation_request( self, ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { - if self.schema_version != 2 { + if self.schema_version != 3 { return Err(CompileError::Snapshot(format!( - "unsupported workload snapshot version {}; only version 2 is supported", + "unsupported workload snapshot version {}; only version 3 is supported", self.schema_version ))); } @@ -807,7 +809,7 @@ fn has_unsafe_raw_entity_leaf( SummaryExpr::SummaryEstimate { summary_input, .. } => { has_unsafe_raw_entity_leaf(summary_input, selected, false) } - SummaryExpr::SummaryMerge { children } => children + SummaryExpr::SummaryMerge { children, .. } => children .iter() .any(|child| has_unsafe_raw_entity_leaf(child, selected, false)), _ => false, @@ -940,7 +942,7 @@ fn preserve_metricsql_counter_only_roots( Ok(()) } -impl PhysicalPlanCompiler { +impl DeploymentPlanCompiler { pub fn compile_promql( &self, request: PhysicalCompilationRequest, @@ -963,6 +965,10 @@ impl PhysicalPlanCompiler { environment: PhysicalDeploymentContext, frontend: QueryFrontend, ) -> Result { + environment + .dataset_identity + .validate() + .map_err(CompileError::Snapshot)?; if let Some(data) = &request.data_workload { data.validate() .map_err(|error| CompileError::Snapshot(error.to_string()))?; @@ -1256,10 +1262,7 @@ impl PhysicalPlanCompiler { .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - // Rate is a readout over the same reset-aware counter state - // as Increase. Keep that semantic distinction in QueryPlan, - // while the physical store binds both to Increase state. - let physical_family = physical_materialization_family(&selected.family); + let physical_family = selected.family.clone(); let physical_algorithm = match &physical_family { SummaryFamilyType::ExactAggregate(kind, _) => { format!("{kind:?}").to_ascii_lowercase() @@ -1470,6 +1473,26 @@ impl PhysicalPlanCompiler { })?, ); } + let compiled_dag = executable_dags[query_index].as_ref().expect("selected DAG"); + let semantic_root = + compiled_dag + .node_ids + .node_id(&selected.node) + .ok_or_else(|| CompileError::Query { + query_id: query.query_id.clone(), + reason: "persisted semantic root is absent".into(), + })?; + runtime_materialization.semantic_fragment = Some( + asap_types::semantic_fragment::SemanticFragment::from_stored_output_in_dataset( + &compiled_dag.dag, + semantic_root, + environment.dataset_identity.clone(), + ) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, + })?, + ); let materialization = runtime_materialization.policy_fingerprint(); let consumer_query_ids = state_consumers .iter() @@ -1623,6 +1646,17 @@ impl PhysicalPlanCompiler { // summary. PrecomputePlan is keyed by physical identity, not query ID. let mut materializations_by_fingerprint = BTreeMap::new(); for materialization in compiled_materializations { + if materializations_by_fingerprint + .get(&materialization.policy_fingerprint()) + .is_some_and(|old: &asap_types::PrecomputeMaterialization| { + old.semantic_fragment != materialization.semantic_fragment + }) + { + return Err(CompileError::Query { + query_id: "shared-output".into(), + reason: "one deployed output cannot have different semantic definitions".into(), + }); + } materializations_by_fingerprint .entry(materialization.policy_fingerprint()) .or_insert(materialization); @@ -1698,7 +1732,7 @@ impl PhysicalPlanCompiler { .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? .family; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != physical_materialization_family(node_family) + if materialization_family != *node_family || window_ms == 0 || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 @@ -1713,6 +1747,7 @@ impl PhysicalPlanCompiler { .then_some(materialization.slide_interval.saturating_mul(1_000)), readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint.into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output(fingerprint.into()), output_grouping: PhysicalGrouping::Reduce( materialization.grouping_labels.names(), ), @@ -1850,10 +1885,11 @@ impl PhysicalPlanCompiler { }); } } - let query_plan = QueryPlan { + let mut query_plan = QueryPlan { plan_id, plan_version: envelope.plan_version, clickhouse_context: None, + selected_dags: BTreeMap::new(), entries: query_entries, }; let mut installed_dags = BTreeMap::new(); @@ -1882,6 +1918,9 @@ impl PhysicalPlanCompiler { query_id: query_id.clone(), reason, })?; + query_plan + .selected_dags + .insert(query_id.clone(), installed.document.clone()); installed_dags.insert(query_id, installed); } for materialization in &mut materializations { @@ -1933,6 +1972,18 @@ impl PhysicalPlanCompiler { } })?; } + // QueryPlan already owns executable readout nodes. Persist only + // maintenance ancestors under PrecomputePlan. + installed_dags = installed_dags + .into_iter() + .filter(|(_, installed)| !installed.binding.precompute_sinks.is_empty()) + .map(|(query_id, installed)| { + installed + .maintenance_projection() + .map(|projected| (query_id.clone(), projected)) + .map_err(|reason| CompileError::Query { query_id, reason }) + }) + .collect::>()?; let mut precompute_plan = match environment.target { PhysicalDeploymentTarget::DistributedCollectors => { PrecomputePlan::build(envelope.clone(), materializations, &producer_ids).and_then( @@ -1955,6 +2006,7 @@ impl PhysicalPlanCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; + precompute_plan.ingest.dataset_identity = Some(environment.dataset_identity.clone()); let mut transmission_plan = crate::physical::compiler::build_transmission_plan( envelope.clone(), &precompute_plan, @@ -2012,7 +2064,7 @@ impl PhysicalPlanCompiler { reason: error.to_string(), })?; } - query_plan.validate_against_catalog(&summary_catalog)?; + query_plan.bind_catalog(&summary_catalog)?; let storage_routing = crate::emit::backend_wire::storage_routing_document( crate::emit::backend_wire::DEFAULT_TENANT, &routed_algorithms.into_iter().collect::>(), @@ -2042,15 +2094,9 @@ fn summary_agg_metric(node: &SummaryNode) -> Option { } } SummaryExpr::SummaryAgg { child, .. } => walk(child, metrics), - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - walk(candidates, metrics); - walk(values, metrics); - } SummaryExpr::ValueOperation { child, .. } => walk(child, metrics), SummaryExpr::SummaryEstimate { summary_input, .. } => walk(summary_input, metrics), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { walk(child, metrics); } @@ -2118,10 +2164,9 @@ fn observed_population_matches_root( || !measures.iter().all(|intent| { matches!( intent, - AggIntent::Cardinality { col: None, .. } - | AggIntent::FrequencyL2 { col: None, .. } + AggIntent::FrequencyL2 { col: None, .. } | AggIntent::FrequencyEntropy { col: None, .. } - ) + ) || matches!(intent, AggIntent::Cardinality { cols, .. } if cols.is_empty()) }) { return false; @@ -2307,7 +2352,7 @@ fn requires_exact_erp_fallback( child: summary_input, .. } => walk(summary_input, out), - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { children.iter().for_each(|child| walk(child, out)) } SummaryExpr::SummaryJoin { outer, inner, .. } => { @@ -2324,12 +2369,6 @@ fn requires_exact_erp_fallback( walk(left, out); walk(right, out); } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - walk(candidates, out); - walk(values, out); - } SummaryExpr::KeepPreAsap(_) => {} } } @@ -2356,7 +2395,7 @@ fn requires_exact_erp_fallback( #[cfg(test)] /// Planner-adapter selection step used before physical compilation. Keeping /// this separate makes the ownership boundary explicit: callers supply the -/// selected post-ASAP DAG to [`PhysicalPlanCompiler::compile`]. +/// selected post-ASAP DAG to [`DeploymentPlanCompiler::compile`]. pub fn select_post_asap( expr: &QueryExpr, accuracy: AccuracyTarget, @@ -2377,9 +2416,49 @@ pub fn select_post_asap( delete: false, }, ); + // These fixtures exercise collector heap transport. Collector fixtures do + // not offer backend-only temporal value readouts as a physical capability. + struct CollectorFixtureModel(ControlPlaneCostModel); + impl asap_aware_mapping::CostModel for CollectorFixtureModel { + fn rank_candidates( + &self, + intent: &planner_types::pre_asap::AggIntent, + candidates: &[planner_types::post_asap::SketchAlgorithm], + ) -> Vec { + self.0.rank_candidates(intent, candidates) + } + fn size_params( + &self, + kind: planner_types::post_asap::SketchAlgorithm, + intent: &planner_types::pre_asap::AggIntent, + eps: f64, + delta: f64, + ) -> planner_types::post_asap::SketchParams { + self.0.size_params(kind, intent, eps, delta) + } + fn candidate_cost( + &self, + candidate: &asap_aware_mapping::ReplacementSubDAG, + target: &asap_aware_mapping::TargetSubDAG<'_>, + ) -> Option { + self.0.candidate_cost(candidate, target) + } + fn summary_support_evidence(&self, summary: &SummaryNode) -> Option { + if matches!( + summary.expr, + SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::Limit { .. }, + .. + } + ) { + return Some(false); + } + self.0.summary_support_evidence(summary) + } + } crate::planner_selection::select_query_with_models( expr, - &model, + &CollectorFixtureModel(model), &DefaultAccuracyModel, &QueryEvidence(evidence), ) @@ -2826,13 +2905,11 @@ pub(super) fn estimated_state_bytes( A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), A::DDSketch => 64 * 1024, A::Sum + | A::Count | A::Increase + | A::Rate | A::Min | A::Max - | A::MultipleSum - | A::MultipleIncrease - | A::MultipleMin - | A::MultipleMax | A::SingleSubpopulation | A::MultipleSubpopulation => 256, } @@ -2850,7 +2927,7 @@ fn retained_partition_count( if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( materialization.aggregation_type, - A::Increase | A::MultipleIncrease | A::Min | A::Max | A::MultipleMin | A::MultipleMax + A::Increase | A::Rate | A::Min | A::Max ) || !materialization.grouping_labels.names().is_empty() { @@ -3097,7 +3174,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -fn raw_time_series_input_contract( +pub fn raw_time_series_input_contract( expr: &QueryExpr, exact: bool, ) -> Result<(String, Option, String), String> { @@ -3194,7 +3271,7 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option Option @@ -3293,7 +3370,7 @@ fn physical_aggregation( BackendAggregation { aggregation_id, metric_name: selected.metric.clone(), - family: physical_materialization_family(&selected.family), + family: selected.family.clone(), window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected @@ -3523,8 +3600,12 @@ fn collect_selected_materializations( } } match &node.expr { - SummaryExpr::CandidateTopK { - candidates, values, .. + SummaryExpr::RelationalJoin { + right: candidates, + left: values, + kind: planner_types::pre_asap::JoinKind::Semi, + pruning: Some(_), + .. } => { walk(candidates, readout, composable, grouping.clone(), selected)?; // In a hybrid TopK, the sketch is only a candidate-membership @@ -3577,7 +3658,7 @@ fn collect_selected_materializations( grouping.clone(), selected, )?, - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { walk(child, readout, composable, grouping.clone(), selected)?; } @@ -3618,25 +3699,9 @@ fn collect_selected_materializations( SummaryInputExpr::Column( planner_types::pre_asap::ColumnRef::SampleValue, ) => "value", - SummaryInputExpr::ResetAwareCounterDelta { .. } - if matches!( - input.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { - proof: planner_types::post_asap::NonNegativeWeightProof::ResetAwareCounterDerivative - } - ) => "counter_delta", - SummaryInputExpr::ResetAwareCounterDelta { .. } => { - return Err("counter-delta TopK input lacks a non-negative reset-aware proof".into()) - } _ => return Err("unsupported TopK SummaryUpdate weight".into()), }; parameters["weight_mode"] = mode.into(); - if mode == "counter_delta" { - // CMS/CountSketch heap implementations quantize - // weights to integer counters. Preserve sub-unit - // counter increments used by CPU metrics. - parameters["weight_scale"] = 1_000_000.into(); - } } let (metric, window_secs, spatial_filter) = match selected_input_contract(node) { @@ -3703,26 +3768,6 @@ fn collect_selected_materializations( Ok(selected) } -pub(crate) fn physical_materialization_family(family: &SummaryFamilyType) -> SummaryFamilyType { - match family { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) => { - // The SummaryStore Sum accumulator retains the observation count - // alongside its sum. Both logical states can share this producer. - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ) - } - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Rate, _) => { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, - ) - } - _ => family.clone(), - } -} - pub(super) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { @@ -3798,7 +3843,7 @@ impl QueryFrontend { request: PhysicalCompilationRequest, environment: PhysicalDeploymentContext, ) -> Result { - PhysicalPlanCompiler.compile_for_frontend(request, environment, self) + DeploymentPlanCompiler.compile_for_frontend(request, environment, self) } } #[cfg(test)] @@ -3812,7 +3857,7 @@ pub(crate) mod tests { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - snapshot.schema_version = 2; + snapshot.schema_version = 3; snapshot.data_workload.data_ingestion_interval.value = Some(DurationMs(1_000)); let template = snapshot.query_workload.repeating_queries.as_ref().unwrap()[0].clone(); let queries = [ @@ -3839,7 +3884,7 @@ pub(crate) mod tests { .unwrap() .into_iter() .filter_map(|r| { - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(r, environment.clone()) .ok() }) @@ -3872,8 +3917,8 @@ pub(crate) mod tests { assert_eq!(populations.len(), 1); let installed = serde_json::to_string(&plan.precompute_plan.executable_dags).unwrap(); assert!( - installed.contains("MaintainPopulation"), - "shared state must originate in the installed Planner DAG" + !installed.contains("MaintainPopulation"), + "query-only population readout must not be executable maintenance" ); } @@ -3955,7 +4000,7 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); let request = request("average", "avg_over_time(a[1m])"); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!( @@ -3985,7 +4030,7 @@ pub(crate) mod tests { env.target_collector_ids.clear(); let mut input = request("minimum", "min_over_time(data[1m])"); input.allow_mixed_summary_and_exact_execution = true; - let plan = PhysicalPlanCompiler.compile_promql(input, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(input, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].aggregation_type, @@ -4089,9 +4134,9 @@ pub(crate) mod tests { .enumerate() .filter_map(|(index, candidate)| { let plan = if frontend == QueryFrontend::MetricsQl { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -4139,7 +4184,7 @@ pub(crate) mod tests { let mut reasons = vec![]; assert!( candidates.into_iter().any(|candidate| { - match PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) { + match DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { Ok(plan) => { !plan.precompute_plan.materializations.is_empty() && plan @@ -4170,7 +4215,7 @@ pub(crate) mod tests { snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query(text.into()); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{text}"); @@ -4210,7 +4255,7 @@ pub(crate) mod tests { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - for version in [0, 1, 3] { + for version in [0, 1, 2, 4] { let mut old = snapshot.clone(); old.schema_version = version; assert!(old @@ -4218,7 +4263,7 @@ pub(crate) mod tests { .into_physical_compilation_request() .unwrap_err() .to_string() - .contains("only version 2")); + .contains("only version 3")); assert!(old.compile_promql().is_err()); } assert!(snapshot.into_physical_compilation_request().is_ok()); @@ -4229,7 +4274,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let mut plan = PhysicalPlanCompiler + let mut plan = DeploymentPlanCompiler .compile_promql(request("scope", "sum_over_time(m[1m])"), env) .unwrap(); let installed = plan @@ -4260,6 +4305,8 @@ pub(crate) mod tests { &dag, ) .unwrap(); + installed.document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; assert!(plan .precompute_plan .validate() @@ -4279,7 +4326,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("per-entity", query), environment) .unwrap(); assert!( @@ -4298,7 +4345,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("reduced", query), environment) .unwrap(); assert!( @@ -4314,7 +4361,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -4338,7 +4385,7 @@ pub(crate) mod tests { #[test] fn raw_counter_artifact_is_valid_for_backend_precompute() { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request("counter", "rate(m[1m])"), environment(10_000)) .unwrap(); plan.precompute_plan.validate().unwrap(); @@ -4348,7 +4395,7 @@ pub(crate) mod tests { raw.ingest.endpoint_path = "/api/v1/write".into(); raw.ingest.timestamp_unit = TimestampUnit::UnixMilliseconds; raw.ingest.require_plan_identity = false; - raw.ingest.require_summary_definition_identity = false; + raw.ingest.require_stored_output_identity = false; raw.ingest.require_registered_producer = false; raw.producers.clear(); raw.bind_catalog(&catalog).unwrap(); @@ -4365,7 +4412,7 @@ pub(crate) mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); @@ -4421,7 +4468,7 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk", query, Some(evidence)).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); @@ -4433,7 +4480,7 @@ pub(crate) mod tests { } #[test] - fn weighted_counter_topk_keeps_heap_membership_separate_from_exact_values() { + fn unsupported_rate_heap_uses_explicit_exact_route() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -4443,86 +4490,24 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .unwrap(); - let entry = plan.query_plan.entries.values().next().unwrap(); - let crate::query_plan::QueryPlanNode::CandidateTopK { inputs, .. } = - &entry.nodes[&entry.root] - else { - panic!("Planner weighted TopK must lower to CandidateTopK: {entry:#?}"); - }; - assert!(matches!( - entry.nodes[&inputs[0]], - crate::query_plan::QueryPlanNode::SummaryEstimate { - query: crate::query_plan::QueryReadout::TopK { .. }, - .. - } - )); - let candidate_read = match &entry.nodes[&inputs[0]] { - crate::query_plan::QueryPlanNode::SummaryEstimate { input, .. } => *input, - _ => unreachable!(), - }; - assert!(matches!( - entry.nodes[&candidate_read], - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - )); - assert!(!entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. }))); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactReadout { - readout: crate::query_plan::ExactReadout::Rate, - .. - } - ))); - let heaps = plan - .precompute_plan - .materializations - .iter() - .filter(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap - && materialization.parameters["weight_mode"] == "counter_delta" - }) - .collect::>(); - assert_eq!(heaps.len(), 1, "unpartitioned TopK owns one global CMS"); - assert!(heaps[0].grouping_labels.names().is_empty()); - assert_eq!(heaps[0].aggregated_labels.labels, vec!["job"]); - assert_eq!(heaps[0].parameters["weight_scale"], 1_000_000); - assert_eq!(retained_partition_count(heaps[0], Some(5)), 1); - let crate::query_plan::QueryPlanNode::ReadMaterialization { binding } = - &entry.nodes[&candidate_read] - else { - unreachable!() - }; + let entry = plan.query_plan.lookup(query).unwrap(); assert!(matches!( - binding.output_grouping, - crate::query_plan::PhysicalGrouping::Reduce(ref labels) if labels.is_empty() + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } )); - assert_eq!(binding.item_labels, vec!["job"]); - let counter = plan - .precompute_plan - .materializations - .iter() - .find(|materialization| { - matches!( - materialization.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease - ) - }) - .expect("reset-aware exact counter"); - assert_eq!(retained_partition_count(counter, Some(5)), 5); + assert_eq!( + entry.fallback, + crate::query_plan::FallbackPolicy::ExactBackend + ); + assert_eq!(entry.canonical_query, query); + assert!(plan.precompute_plan.materializations.is_empty()); } #[test] - fn hybrid_weighted_topk_installs_only_candidates_and_delegates_filtered_exact_values() { - use crate::query_plan::{ - residual::ResidualQueryOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, - }; + fn hybrid_rate_topk_preserves_the_original_exact_subquery() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -4537,60 +4522,34 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); - - assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - asap_types::AggregationType::CountMinSketchWithHeap - ); let entry = plan.query_plan.lookup(query).unwrap(); - let QueryPlanNode::CandidateTopK { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("expected candidate TopK: {entry:#?}"); - }; + use crate::query_plan::{residual::ResidualQueryOperator, QueryPlanNode}; assert!(matches!( - &entry.nodes[&inputs[1]], - QueryPlanNode::ExternalExact { - request, - inputs: exact_inputs, - } if request.language == crate::query_plan::QueryLanguage::PromQl - && request.expression == "sum by (job) (rate(m[1m]))" - && request.output == ExternalExactOutput::InstantVector - && request.input_contracts == vec![ExternalExactInput::CandidateMembership { - item_label: "job".into(), - }] - && exact_inputs == &vec![inputs[0]] + &entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: 2, .. }, + .. + } )); - assert!(entry.nodes.values().all(|node| !matches!( - node, - QueryPlanNode::ExactReadout { .. } - | QueryPlanNode::Logical { - operator: ResidualQueryOperator::Scan { .. }, - .. - } - ))); - let installed = plan - .precompute_plan - .executable_dags - .get(&entry.query_id) - .expect("compiled query retains its Planner DAG and backend placement"); - installed.validate().expect("typed DAG document"); - crate::physical::executable_binding::validate_query_plan(installed, entry) - .expect("query node bindings"); - assert_eq!(installed.binding.query_plan_sink, entry.root); - assert!(installed.binding.nodes.values().any(|placement| matches!( - placement, - crate::physical::executable_binding::BackendNodeBinding::Materialization { .. } - ))); - let encoded = serde_json::to_value(installed).unwrap(); - let decoded: crate::physical::executable_binding::InstalledPostAsapDag = - serde_json::from_value(encoded).unwrap(); - assert_eq!(&decoded, installed); - decoded - .validate() - .expect("round-tripped typed DAG document"); + assert_eq!( + entry + .nodes + .values() + .filter(|node| matches!(node, QueryPlanNode::Logical { + operator: ResidualQueryOperator::ExactSubquery { query }, .. + } if query == "sum by (job) (rate(m[1m]))")) + .count(), + 1 + ); + assert!(entry.materialization_bindings().is_empty()); + assert!(plan.precompute_plan.materializations.is_empty()); + let artifact = plan.to_publication_artifact().unwrap(); + artifact.validate().unwrap(); + let encoded = serde_json::to_value(&artifact).unwrap(); + assert!(!encoded.to_string().contains("counter_delta")); } #[test] @@ -4602,11 +4561,11 @@ pub(crate) mod tests { observed_at_unix_ms: 9_500, source: "unit-fixture".into(), }; - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql( request_with_evidence( "topk-rate", - "topk(2, sum by (job) (rate(m[1m])))", + "topk(2, count_over_time(m[1m]))", Some(evidence), ) .unwrap(), @@ -4652,7 +4611,7 @@ pub(crate) mod tests { let mut request = request("bounded", "sum(sum_over_time(m[1m]))"); request.retained_summary_memory_budget_bytes = Some(1); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .unwrap_err(); assert!(error.to_string().contains("retained summary footprint")); @@ -4680,8 +4639,51 @@ pub(crate) mod tests { ); } + /// Dataset changes alter persisted meaning; relocating the same input does not. + #[test] + fn dataset_identity_survives_binding_and_rejects_wrong_input() { + let compile = |env| { + DeploymentPlanCompiler + .compile_promql(request("q", "sum_over_time(m[1m])"), env) + .unwrap() + }; + let first = compile(environment(10_000)); + assert!(!first.summary_catalog.definitions.is_empty()); + let mut other = environment(10_000); + other.dataset_identity.namespace = "other-tenant".into(); + let second = compile(other); + assert_ne!( + first.summary_catalog.definitions.keys().collect::>(), + second + .summary_catalog + .definitions + .keys() + .collect::>() + ); + let mut relocated = environment(10_000); + relocated.target_collector_ids = vec!["relocated-source".into()]; + let relocated = compile(relocated); + assert_eq!( + first.summary_catalog.definitions, + relocated.summary_catalog.definitions + ); + let mut forged = first.precompute_plan.clone(); + forged.ingest.dataset_identity.as_mut().unwrap().namespace = "other-tenant".into(); + assert!(forged + .validate() + .unwrap_err() + .to_string() + .contains("dataset")); + forged.ingest.dataset_identity = None; + assert!(forged.validate().is_err()); + } + fn environment(now: u64) -> PhysicalDeploymentContext { PhysicalDeploymentContext { + dataset_identity: planner_types::post_asap::LogicalDatasetIdentity { + namespace: "test".into(), + dataset: "metrics".into(), + }, target: PhysicalDeploymentTarget::DistributedCollectors, target_collector_ids: vec!["edge-a".into(), "edge-b".into()], capability_snapshot_id: "caps-7".into(), @@ -4810,7 +4812,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); @@ -4856,7 +4858,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 3); @@ -4931,7 +4933,7 @@ pub(crate) mod tests { } }) .expect("unknown HLL candidate stays inspectable"); - let result = PhysicalPlanCompiler.compile_metricsql(workload, environment(10_000)); + let result = DeploymentPlanCompiler.compile_metricsql(workload, environment(10_000)); assert!( matches!(result, Err(CompileError::Query { reason, .. }) if reason.contains("no certified accuracy guarantee")) ); @@ -4959,7 +4961,7 @@ pub(crate) mod tests { None, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -4985,7 +4987,7 @@ pub(crate) mod tests { deployment.target_collector_ids.clear(); let mut workload = request("confidence", "distinct_over_time(m[1m])"); workload.allow_mixed_summary_and_exact_execution = true; - let result = PhysicalPlanCompiler.compile_metricsql(workload, deployment); + let result = DeploymentPlanCompiler.compile_metricsql(workload, deployment); assert!(result.unwrap().precompute_plan.materializations.is_empty()); } @@ -5016,7 +5018,7 @@ pub(crate) mod tests { second.query_string = "max_over_time(b[1m])".into(); workload.queries.push(second); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap_err(); assert!(matches!( @@ -5037,7 +5039,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -5054,7 +5056,7 @@ pub(crate) mod tests { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); @@ -5064,8 +5066,7 @@ pub(crate) mod tests { .iter() .all(|m| !matches!( m.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Increase | asap_types::AggregationType::Rate ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); @@ -5092,7 +5093,7 @@ pub(crate) mod tests { workload.queries[0].query_string = query.into(); workload.queries[0].selected_plan_root = crate::planner_selection::keep_pre_asap(&canonical).unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_metricsql(workload, environment(10_000)) .unwrap(); let identity = canonical_promql(query).unwrap(); @@ -5225,7 +5226,7 @@ pub(crate) mod tests { workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -5323,11 +5324,11 @@ pub(crate) mod tests { let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; backend.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(with_evidence, backend) .unwrap(); assert!( - !plan.summary_catalog.materializations.is_empty(), + !plan.summary_catalog.outputs.is_empty(), "measured exact-composition evidence must expose the rate child as a SummaryStore binding" ); } @@ -5354,7 +5355,7 @@ pub(crate) mod tests { let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; backend.target_collector_ids.clear(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(unavailable, backend) .unwrap(); // Planner 54f can realize this particular shape directly as an exact @@ -5362,7 +5363,7 @@ pub(crate) mod tests { // ExactComposition candidate. The absence of evidence must therefore // leave that direct legal path intact rather than inventing a composed // cost or forcing an exact fallback. - assert!(!plan.summary_catalog.materializations.is_empty()); + assert!(!plan.summary_catalog.outputs.is_empty()); let entry = plan .query_plan .entries @@ -5469,7 +5470,7 @@ pub(crate) mod tests { &workload.exact_composition_costs, ) .unwrap(); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -5501,7 +5502,7 @@ pub(crate) mod tests { // Adding another readout adds recurring reads, not another update stream. #[test] fn joint_lifecycle_charges_shared_updates_once() { - let baseline = PhysicalPlanCompiler + let baseline = DeploymentPlanCompiler .compile_promql( request("q90", "quantile_over_time(0.9, m[1m])"), environment(10000), @@ -5513,7 +5514,7 @@ pub(crate) mod tests { .remove(0); second.summary_lifecycle_inputs.evaluation_interval_ms = 20000; workload.queries.push(second); - let shared = PhysicalPlanCompiler + let shared = DeploymentPlanCompiler .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(shared.lifecycle_estimates.len(), 1); @@ -5542,7 +5543,7 @@ pub(crate) mod tests { second.summary_lifecycle_inputs.ingestion_rate_per_second = 200.0; workload.queries.push(second); assert!(matches!( - PhysicalPlanCompiler.compile_promql(workload, environment(10000)), + DeploymentPlanCompiler.compile_promql(workload, environment(10000)), Err(CompileError::Lifecycle { .. }) )); } @@ -5568,11 +5569,11 @@ pub(crate) mod tests { if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { env.target_collector_ids.clear(); } - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); let bindings = bundle @@ -5592,14 +5593,14 @@ pub(crate) mod tests { #[test] fn adding_shared_consumer_changes_plan_identity() { let workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let one = PhysicalPlanCompiler + let one = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); workload .queries .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let two = PhysicalPlanCompiler + let two = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); assert_ne!(one.envelope.plan_id, two.envelope.plan_id); @@ -5612,10 +5613,10 @@ pub(crate) mod tests { let mut other = request("qn", "quantile_over_time(0.90, n[1m])"); other.queries[0].legacy_query_source = Source::TimeSeries { metric: "n".into() }; workload.queries.extend(other.queries); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); - assert_eq!(bundle.summary_catalog.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.outputs.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { assert_eq!(collector.materializations.len(), 2); @@ -5623,19 +5624,23 @@ pub(crate) mod tests { } #[test] - fn rate_and_increase_share_physical_counter_state() { + fn rate_and_increase_keep_planner_families_distinct() { let mut workload = request("rate", "rate(m[1m])"); workload .queries .extend(request("increase", "increase(m[1m])").queries); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert_eq!(collector.materializations[0].algorithm, "increase"); + let algorithms: std::collections::BTreeSet<_> = collector + .materializations + .iter() + .map(|materialization| materialization.algorithm.as_str()) + .collect(); + assert_eq!(algorithms, ["increase", "rate"].into()); } } @@ -5646,7 +5651,7 @@ pub(crate) mod tests { other.queries[0].window_realization_candidates[0].realization_id = "another-implementation".into(); workload.queries.extend(other.queries); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(workload, environment(10_000)) .expect_err("conflicting shared state must fail before publication"); assert!(error @@ -5655,8 +5660,7 @@ pub(crate) mod tests { } #[test] - fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { - // Both dashboard roots use one packed raw accumulator, with explicit readouts. + fn exact_dashboard_preserves_distinct_sum_and_count_producers() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5671,8 +5675,8 @@ pub(crate) mod tests { ); entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { assert!( @@ -5682,7 +5686,7 @@ pub(crate) mod tests { )), "{entry:?}" ); - assert_eq!(entry.materialization_bindings().len(), 1); + assert!(!entry.materialization_bindings().is_empty()); } assert!(bundle .query_plan @@ -5753,7 +5757,7 @@ pub(crate) mod tests { }; request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: selected.clone(), rhs: selected.clone(), operator: planner_types::post_asap::BinaryOperator { @@ -5773,7 +5777,7 @@ pub(crate) mod tests { environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.target_collector_ids.clear(); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap_err(); @@ -5800,7 +5804,7 @@ pub(crate) mod tests { entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); request.allow_mixed_summary_and_exact_execution = false; - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); @@ -5826,13 +5830,13 @@ pub(crate) mod tests { let candidates = super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap(); - match PhysicalPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { + match DeploymentPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), Err(error) => assert!(error .to_string() .contains("semantically identical original subtree witness")), } - let native = PhysicalPlanCompiler + let native = DeploymentPlanCompiler .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert!(native.precompute_plan.materializations.is_empty()); @@ -5849,7 +5853,7 @@ pub(crate) mod tests { entry.query = Query("sum_over_time(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].partitioning, @@ -5876,7 +5880,7 @@ pub(crate) mod tests { // The derivation now covers both range selectors, so this no longer // needs a hand-supplied 5m candidate to keep `b` from falling back. let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let bindings = plan .query_plan .entries @@ -5887,7 +5891,7 @@ pub(crate) mod tests { let actual = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.materializations[&binding.materialization]; + let identity = &plan.summary_catalog.outputs[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -5920,7 +5924,7 @@ pub(crate) mod tests { value["data_workload"]["ingestion_rate"]["value"] = json!(rate); let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -5954,7 +5958,7 @@ pub(crate) mod tests { let right = right.queries[0].selected_plan_root.clone(); let right = Rc::new(SummaryNode { expr: SummaryExpr::ValueOperation { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, child: right.clone(), }, @@ -5963,7 +5967,7 @@ pub(crate) mod tests { }); request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: left.clone(), rhs: right, operator: planner_types::post_asap::BinaryOperator { @@ -5997,7 +6001,7 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::Pane { .. } ) }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -6051,7 +6055,7 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(0), }; let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), expected_states); let entry = plan.query_plan.entries.values().next().unwrap(); let bindings = entry.materialization_bindings(); @@ -6089,7 +6093,7 @@ pub(crate) mod tests { }; entries.push(second); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if phase == 0 { 1 } else { 2 } @@ -6143,7 +6147,7 @@ pub(crate) mod tests { .window_realization_candidates .iter() .any(|c| !c.derived)); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); } @@ -6505,7 +6509,7 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::FullWindow ) == full }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; assert_eq!(config.slide_interval, u64::from(evaluation)); assert_eq!(config.window_size, 60); @@ -6561,7 +6565,7 @@ pub(crate) mod tests { ) }); } - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if read_cost == 0.0 { 1 } else { 2 } @@ -6614,7 +6618,7 @@ pub(crate) mod tests { ) }); } - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); assert!(plan .lifecycle_estimates @@ -6635,7 +6639,7 @@ pub(crate) mod tests { }; let (request, env) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.queries[0].window_realization_candidates.is_empty()); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); } @@ -6712,7 +6716,7 @@ pub(crate) mod tests { fn retained_state_count_follows_the_derived_pane_width() { let snapshot = planning_snapshot(); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!( @@ -6737,7 +6741,7 @@ pub(crate) mod tests { }; } let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); let materialization = &plan.precompute_plan.materializations[0]; @@ -6774,7 +6778,7 @@ pub(crate) mod tests { Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let query = plan.query_plan.entries.values().next().unwrap(); let bindings = query.materialization_bindings(); // Both operands now hold a summary. The filtered denominator is no @@ -6786,7 +6790,7 @@ pub(crate) mod tests { let bound = bindings .iter() .map(|binding| { - let identity = &plan.summary_catalog.materializations[&binding.materialization]; + let identity = &plan.summary_catalog.outputs[&binding.materialization]; let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( data.time_series_metric().unwrap(), @@ -6859,7 +6863,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); @@ -6962,11 +6966,37 @@ pub(crate) mod tests { .collect::>(); assert_eq!(bindings.len(), 1); query_plan.validate(&bindings).unwrap(); + let stored_outputs = query_plan + .entries + .values() + .flat_map(|entry| entry.materialization_bindings()) + .map(|binding| binding.stored_output_reference.clone()) + .collect::>(); + assert_eq!(stored_outputs.len(), 2); + assert_eq!(stored_outputs[0], stored_outputs[1]); + assert_eq!( + stored_outputs[0], + bundle.precompute_plan.schemas[0].stored_output_reference + ); + asap_types::plan_publication::validate_stored_output_references( + &bundle.precompute_plan, + &query_plan, + ) + .unwrap(); + // V1 has one stored output per definition; arbitrary output IDs are + // rejected before writer/reader agreement is considered. + let mut rebound_writer = bundle.precompute_plan.clone(); + rebound_writer.schemas[0] + .stored_output_reference + .stored_output_id = asap_types::sds::StoredOutputId(123); + assert!(rebound_writer + .validate_against_catalog(&bundle.summary_catalog) + .is_err()); } #[test] fn precompute_catalog_validates_without_backend_projection() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("catalog", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -6982,8 +7012,7 @@ pub(crate) mod tests { schema.as_object_mut().unwrap().remove("value_projection"); schema["value_column"] = serde_json::json!("SampleValue"); } - let decoded: PrecomputePlan = serde_json::from_value(legacy).unwrap(); - decoded.validate_against_catalog(catalog).unwrap(); + assert!(serde_json::from_value::(legacy).is_err()); let reject = |mutated: PrecomputePlan| assert!(mutated.validate_against_catalog(catalog).is_err()); let mut bad = original.clone(); @@ -7023,7 +7052,7 @@ pub(crate) mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7048,7 +7077,7 @@ pub(crate) mod tests { #[test] fn compiles_one_decision_into_matching_collector_and_backend_views() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7064,7 +7093,7 @@ pub(crate) mod tests { assert_eq!( bundle .summary_catalog - .materializations + .outputs .keys() .cloned() .collect::>(), @@ -7115,7 +7144,7 @@ pub(crate) mod tests { bundle.transmission_plan.validate_frame(&wrong_version), Err(TransmissionPlanError::InvalidFrame(_)) )); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!( bundle .query_plan @@ -7187,7 +7216,7 @@ pub(crate) mod tests { #[test] fn backend_local_hll_and_envelope_ingest_are_supported() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7203,11 +7232,12 @@ pub(crate) mod tests { ) .unwrap(); envelope_plan.ingest = IngestContract { + dataset_identity: envelope_plan.ingest.dataset_identity.clone(), protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }; envelope_plan.producers.clear(); @@ -7217,7 +7247,7 @@ pub(crate) mod tests { #[test] fn backend_local_precompute_contract_has_no_collector_producers() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7290,7 +7320,7 @@ pub(crate) mod tests { .queries .remove(0); let snapshot = BackendLocalPlanningInput { - schema_version: 2, + schema_version: 3, workload_cost_evidence: None, query_workload, data_workload, @@ -7366,9 +7396,9 @@ pub(crate) mod tests { bundle.precompute_plan.schemas[0].window.pane_origin_ms, Some(7_000) ); - let definition = &bundle.summary_catalog.materializations - [&asap_types::sds::SummaryDefinitionId::from(config.policy_fingerprint())]; - assert_eq!(definition.pane_origin_ms, Some(7_000)); + assert!(bundle.summary_catalog.outputs.contains_key( + &asap_types::sds::StoredOutputId::from(config.policy_fingerprint()) + )); assert_eq!( bundle .query_plan @@ -7405,9 +7435,9 @@ pub(crate) mod tests { deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); let compiled = - PhysicalPlanCompiler.compile_promql(request(query_id, promql), deployment); + DeploymentPlanCompiler.compile_promql(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); - assert_eq!(plan.summary_catalog.materializations.len(), 1, "{promql}"); + assert_eq!(plan.summary_catalog.outputs.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); assert!(plan.collector_plans.is_empty(), "{promql}"); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -7421,8 +7451,8 @@ pub(crate) mod tests { assert_eq!( materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, + planner_types::post_asap::ExactKind::Rate, + planner_types::post_asap::ExactParams::Rate, ) ); } @@ -7446,7 +7476,7 @@ pub(crate) mod tests { .clone() .into_physical_compilation_request() .unwrap(); - let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); + let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan @@ -7459,7 +7489,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(native, environment) .expect("native fixture compiles"); assert!(plan.precompute_plan.materializations.is_empty()); @@ -7486,7 +7516,7 @@ pub(crate) mod tests { .clone() .into_physical_compilation_request() .unwrap(); - let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); + let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan @@ -7499,7 +7529,7 @@ pub(crate) mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(native, environment) .expect("native demo compiles"); @@ -7529,12 +7559,12 @@ pub(crate) mod tests { .remove(0), ); } - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(compilation_request, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 4); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); assert_eq!(bundle.precompute_plan.producers.len(), 2); @@ -7559,6 +7589,7 @@ pub(crate) mod tests { }; let merge = Rc::new(SummaryNode { expr: SummaryExpr::SummaryMerge { + timing: planner_types::post_asap::ExecutionTiming::QueryTime, children: vec![left.clone(), right.clone()], }, schema: left.schema.clone(), @@ -7573,10 +7604,10 @@ pub(crate) mod tests { guarantee: left_root.guarantee.clone(), }); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(compilation_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); - assert_eq!(bundle.summary_catalog.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.outputs.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!( bundle @@ -7608,9 +7639,8 @@ pub(crate) mod tests { .values() .filter_map(|node| match node { crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => Some( - bundle.summary_catalog.data_descriptors[&bundle - .summary_catalog - .materializations[&binding.materialization] + bundle.summary_catalog.data_descriptors[&bundle.summary_catalog.outputs + [&binding.materialization] .data_descriptor_id] .time_series_metric() .unwrap(), @@ -7623,7 +7653,7 @@ pub(crate) mod tests { #[test] fn precompute_schema_must_match_materialization_semantics() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7655,7 +7685,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let materialization = bundle.precompute_plan.materializations.first().unwrap(); assert_eq!(materialization.window_size, 60); assert_eq!( @@ -7709,7 +7739,7 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( bundle.lifecycle_estimates[0].window_realization_id, expected_id @@ -7760,7 +7790,9 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.target_collector_ids.clear(); - let bundle = PhysicalPlanCompiler.compile_promql(workload, env).unwrap(); + let bundle = DeploymentPlanCompiler + .compile_promql(workload, env) + .unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 2); } @@ -7772,14 +7804,14 @@ pub(crate) mod tests { asap_types::WindowMaterializationLayout::Pane { pane_secs: 7 }; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - assert!(PhysicalPlanCompiler.compile_promql(request, env).is_err()); + assert!(DeploymentPlanCompiler.compile_promql(request, env).is_err()); } #[test] fn missing_window_implementation_evidence_fails_closed() { let mut request = request("q-window", "quantile_over_time(0.99, m[1m])"); request.queries[0].window_realization_candidates.clear(); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect_err("Planner must not receive a zero-cost invented window"); assert!(matches!(error, CompileError::Lifecycle { .. })); @@ -7787,7 +7819,7 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_schema_or_producer_drift() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7811,7 +7843,7 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_empty_or_duplicate_schema_ids() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7858,7 +7890,7 @@ pub(crate) mod tests { }), ) .expect("selection accepts evidence before freshness validation"); - let error = PhysicalPlanCompiler + let error = DeploymentPlanCompiler .compile_promql(request, environment(100_000)) .expect_err("stale certificate must fail"); assert!(matches!(error, CompileError::InvalidEvidence { .. })); @@ -7879,7 +7911,7 @@ pub(crate) mod tests { ) .expect("selection occurs before deployment-time freshness validation"); assert!(matches!( - PhysicalPlanCompiler.compile_promql(topk, environment(10_000)), + DeploymentPlanCompiler.compile_promql(topk, environment(10_000)), Err(CompileError::InvalidEvidence { .. }) )); @@ -7888,7 +7920,7 @@ pub(crate) mod tests { .cost .observed_at_unix_ms = 10_001; assert!(matches!( - PhysicalPlanCompiler.compile_promql(window, environment(10_000)), + DeploymentPlanCompiler.compile_promql(window, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7907,10 +7939,10 @@ pub(crate) mod tests { }), ) .expect("selection accepts valid evidence"); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect("certified TopK compiles"); - assert_eq!(bundle.summary_catalog.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.outputs.len(), 1); assert_eq!( bundle.collector_plans[0].materializations[0] .evidence_source @@ -7924,7 +7956,7 @@ pub(crate) mod tests { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); request.planner_revision = "different".into(); assert!(matches!( - PhysicalPlanCompiler.compile_promql(request, environment(10_000)), + DeploymentPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::PlannerRevision { .. }) )); } @@ -7939,7 +7971,7 @@ pub(crate) mod tests { .summary_lifecycle_inputs .evidence_valid_for_ms = 10; assert!(matches!( - PhysicalPlanCompiler.compile_promql(request, environment(10_000)), + DeploymentPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7965,7 +7997,7 @@ pub(crate) mod tests { #[test] fn runtime_policy_encoding_is_checked() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), @@ -7997,7 +8029,7 @@ pub(crate) mod tests { absolute_threshold: 0.0, gos: None, }); - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql(request, environment(10_000)) .expect("delta-capable physical plan"); let rule = &bundle.transmission_plan.rules[0]; @@ -8032,7 +8064,7 @@ pub(crate) mod tests { #[test] fn runtime_adaptation_requires_fresh_exact_evidence_and_successor_version() { - let bundle = PhysicalPlanCompiler + let bundle = DeploymentPlanCompiler .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 1856672cd..72dfb3e2a 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -363,7 +363,7 @@ pub struct ErpObservedShapeSource { #[serde(deny_unknown_fields)] pub struct ErpPopulationObservationScope { pub catalog_generation: asap_types::sds::CatalogGeneration, - pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub stored_output_id: asap_types::sds::StoredOutputId, pub input_semantics: asap_types::erp_observation::ErpObservationInputSemantics, pub freshness: asap_types::erp_observation::ErpObservationFreshness, } @@ -389,8 +389,8 @@ impl ErpPlanningInput { return Err("ERP evidence catalog differs from the active catalog".into()); } let materialization = catalog - .materializations - .get(&populations.summary_definition_id) + .outputs + .get(&populations.stored_output_id) .ok_or("ERP evidence summary is absent from the active catalog")?; let summary = catalog .summary_descriptors @@ -477,7 +477,7 @@ impl ErpPlanningInput { .map_err(|error| format!("invalid ERP population observations: {error}"))?; observed.validate_identity_and_freshness( &scope.catalog_generation, - scope.summary_definition_id, + scope.stored_output_id, now_ms, scope.freshness, )?; @@ -495,7 +495,7 @@ impl ErpPlanningInput { asap_types::erp_observation::ErpPopulationObservations { schema_version: 1, catalog_generation: scope.catalog_generation.clone(), - summary_definition_id: scope.summary_definition_id, + stored_output_id: scope.stored_output_id, observed_at_unix_ms: now_ms, window_start_ms: 0, window_end_ms: 0, @@ -1352,7 +1352,7 @@ mod tests { let observed = ErpPopulationObservations { schema_version: 1, catalog_generation: generation.clone(), - summary_definition_id: definition, + stored_output_id: definition, observed_at_unix_ms: 1_000, window_start_ms: 0, window_end_ms: 1_000, @@ -1370,7 +1370,7 @@ mod tests { implementation: "asap_sketchlib".into(), population_scope: Some(ErpPopulationObservationScope { catalog_generation: generation, - summary_definition_id: definition, + stored_output_id: definition, input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, freshness: ErpObservationFreshness { max_age_ms: 100, @@ -1480,13 +1480,12 @@ mod tests { .unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = catalog.reference().unwrap(); - observed.summary_definition_id = *catalog.materializations.keys().next().unwrap(); + observed.stored_output_id = *catalog.outputs.keys().next().unwrap(); observed.input_semantics = asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; policy.observed_populations = Some(observed.clone()); policy.resolve_population_data_descriptor(Some(&catalog)); - let expected = - &catalog.materializations[&observed.summary_definition_id].data_descriptor_id; + let expected = &catalog.outputs[&observed.stored_output_id].data_descriptor_id; assert_eq!( &policy.resolved_data_descriptor.as_ref().unwrap().id, expected diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 28f82a57c..378967d87 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -9,17 +9,17 @@ pub fn install_selected_dag( query_plan_sink: QueryNodeId, materialization: impl Fn( planner_types::post_asap::PostAsapNodeId, - ) -> Option, + ) -> Option, query_node: impl Fn(planner_types::post_asap::PostAsapNodeId) -> Option, ) -> Result { let mut nodes = std::collections::BTreeMap::new(); let mut precompute_sinks = Vec::new(); for node in &dag.nodes { - let binding = if let Some(summary_definition) = materialization(node.id) { + let binding = if let Some(stored_output) = materialization(node.id) { precompute_sinks.push(node.id); - BackendNodeBinding::Materialization { summary_definition } + BackendNodeBinding::Materialization { stored_output } } else if node.output_state.timing - == planner_types::post_asap::ExecutionTiming::MaintenanceTime + == planner_types::post_asap::ExecutionTiming::IngestionTime { BackendNodeBinding::MaintenanceInput } else { @@ -39,7 +39,7 @@ pub fn install_selected_dag( precompute_sinks, }, }; - installed.validate()?; + installed.binding.validate(&installed.document.decode()?)?; Ok(installed) } diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index b76cdbf00..e1e29a4dc 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -34,10 +34,7 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!( - m.aggregation_type, - AggregationType::Sum | AggregationType::MultipleSum - ) + || !matches!(m.aggregation_type, AggregationType::Sum) { continue; } diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index 67bad1b79..19d602413 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -154,7 +154,7 @@ fn query_node_label(node: &QueryPlanNode) -> String { QueryPlanNode::SummaryEstimate { query, .. } => format!("SummaryEstimate\n{query:?}"), QueryPlanNode::ExactReadout { readout, .. } => format!("ExactReadout\n{readout:?}"), QueryPlanNode::SummaryMerge { .. } => "SummaryMerge".into(), - QueryPlanNode::CandidateTopK { k, .. } => format!("CandidateTopK\nk={k}"), + QueryPlanNode::MembershipFilter { .. } => "MembershipFilter".into(), QueryPlanNode::ExternalExact { .. } => "ExternalExact".into(), QueryPlanNode::ExactFallback { reason } => format!("ExactFallback\n{reason}"), } diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 571c054c3..5fa6c75d9 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -521,8 +521,8 @@ impl CostModel for ControlPlaneCostModel { fn value_operation_capabilities(&self) -> ValueOperationCapabilities { ValueOperationCapabilities { - read_time: true, - maintenance_time: false, + query_time: true, + ingestion_time: false, } } diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index cdff8f355..aa572cdf8 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -1,5 +1,4 @@ //! Query binding delegates selection to Planner's costed workload search. -//! Backend-specific rate normalization remains part of the physical binding. #![allow(dead_code)] @@ -118,41 +117,8 @@ fn bind_recursive( )) } - _ => { - let rewritten = rewrite_rate_to_increase(expr); - let node = crate::planner_selection::select_query(&rewritten, cost_model)?; - Ok(PostAsapPlan::Summary(node)) - } - } -} - -/// Rewrite Rate to Increase along the aggregate spine traversed by Planner. -/// This deployment computes rate by dividing the Increase readout by window -/// seconds, rather than storing a separate Rate accumulator. -fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs - .iter() - .map(|intent| { - if matches!(intent, AggIntent::Rate) { - AggIntent::Increase - } else { - intent.clone() - } - }) - .collect(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(rewrite_rate_to_increase(child)), - }, - other => other.clone(), + _ => Ok(PostAsapPlan::Summary( + crate::planner_selection::select_query(expr, cost_model)?, + )), } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index e2eec8b92..344b60534 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -112,13 +112,10 @@ fn node_is_archive(node: &Rc) -> bool { SummaryExpr::SummaryAgg { child, .. } => node_is_archive(child), SummaryExpr::ValueOperation { child, .. } => node_is_archive(child), SummaryExpr::SummaryEstimate { summary_input, .. } => node_is_archive(summary_input), - SummaryExpr::SummaryMerge { children } => children.iter().any(node_is_archive), + SummaryExpr::SummaryMerge { children, .. } => children.iter().any(node_is_archive), SummaryExpr::SummaryJoin { outer, inner, .. } => { node_is_archive(outer) || node_is_archive(inner) } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => node_is_archive(candidates) || node_is_archive(values), SummaryExpr::SummarySubtract { left, right } | SummaryExpr::RelationalJoin { left, right, .. } | SummaryExpr::BinaryOp { @@ -332,7 +329,7 @@ fn uncertified_hll_keeps_exact_execution() { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, measures: vec![AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Epsilon(0.01), }], output_names: Vec::new(), @@ -474,13 +471,9 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { /// `ONLY_SPATIAL` — `sum by (host) (m)`. /// Control plane path: `Aggregate{Sum, by=[host]}` over a bare `Scan`. /// -/// The old locally-defined `AggregationType::MultipleSum` (keyed vs -/// unkeyed sum) identity no longer exists at the L4 IR level — -/// `SummaryKind::Sum` covers both; the keyed/unkeyed distinction now -/// lives on `SummaryAgg::by` (non-empty ⇒ the old "MultipleSum" shape), -/// per `emit::mod.rs`'s exact-accumulator classification notes. +/// Family remains Sum; the reduction carries the grouping columns. #[test] -fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { +fn phase_b_pattern_only_spatial_aggregate_binds_to_grouped_sum() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), // service column measures: vec![AggIntent::Sum { col: None }], @@ -501,7 +494,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { assert_eq!( reduction.group_keys().map(|k| k.keys()), Some(&[1][..]), - "keyed sum must carry the group-by column (the MultipleSum-equivalent signal)" + "Sum reduction must retain the group-by column" ); } other => panic!("expected SummaryAgg(Sum, by=[1]), got {other:?}"), @@ -511,14 +504,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { } /// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// `bind_query_expr` (not `implement_tree` directly) rewrites -/// `AggIntent::Rate` to `AggIntent::Increase` before binding (see -/// `lower.rs`'s `rewrite_rate_to_increase` — this deployment's data -/// plane has no Rate accumulator). The old -/// `AggregationType::MultipleIncrease` identity is now -/// `SummaryKind::Increase` with a non-empty `by`. +/// Planner preserves the Rate family and the `by` reduction independently. #[test] -fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { +fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), measures: vec![AggIntent::Rate], @@ -534,11 +522,11 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { } => { assert_eq!( family, - &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } - other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), + other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), }, other => panic!("expected Committed(Summary(_)), got {other:?}"), } @@ -671,12 +659,9 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { ); } -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Control plane path: `Aggregate{Rate}` over -/// `Window` → `bind_query_expr` rewrites `Rate` to `Increase` and binds an -/// exact accumulator (`SummaryAgg{Increase}`) — no approximate summary -/// family. Both paths produce a single non-summary streaming row; the L5 -/// emitter is the one that picks the actual MultipleIncrease processor. +/// A Rate query keeps Planner's exact Rate family through binding. The +/// physical emitter chooses the runtime processor without changing that +/// family identity. #[test] fn phase_b_e2e_rate_falls_through_to_logical() { let bound = pipeline_l1_to_l4( diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs index 4a620b94d..0f16ebf30 100644 --- a/control_plane/src/physical/realization.rs +++ b/control_plane/src/physical/realization.rs @@ -3,8 +3,8 @@ //! Providers may validate and price physical implementations, never rewrite //! selected logical roots or infer a pane width from a query's slide. use super::compiler::{ - CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, - PhysicalPlanCompiler, QueryCompilationInput, + CompileError, CompiledPhysicalPlan, DeploymentPlanCompiler, PhysicalCompilationRequest, + PhysicalDeploymentContext, QueryCompilationInput, }; use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; use asap_aware_mapping::cost_model::Cost; @@ -51,9 +51,9 @@ impl RealizationProvider for ExistingRealizations { frontend: super::compiler::QueryFrontend, ) -> Result { if frontend == super::compiler::QueryFrontend::MetricsQl { - PhysicalPlanCompiler.compile_metricsql(request, environment) + DeploymentPlanCompiler.compile_metricsql(request, environment) } else { - PhysicalPlanCompiler.compile_promql(request, environment) + DeploymentPlanCompiler.compile_promql(request, environment) } } diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 5b9f407c5..8dc48e18d 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -101,7 +101,7 @@ pub enum Capability { /// * Sum-over-time requires archive execution because cumulative samples /// cannot be reconstructed from delta state alone. /// -/// Rate and Increase require the Increase capability; plain sum requires Sum. +/// Rate and Increase have distinct exact-family capabilities. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -275,8 +275,7 @@ impl Capability { /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` /// has no wildcard concept of its own; family-matching subsumes it. - /// `ExactAgg` is intentionally NOT routed through this path — see - /// [`multi_pop_satisfies_single`]'s doc for why. + /// Exact families require identity; keyed layout is checked separately. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { @@ -313,22 +312,9 @@ impl Capability { SketchAlgorithm::CmsWithHeap, ) } - // Exact-aggregation family: the agg_type must match - // exactly OR be the single-pop ⇆ multi-pop equivalent. A - // `MultipleSum` policy can serve a `Sum` query by - // re-aggregating across keys; the `find_matching_policies` - // group_by ⊆ policy_grouping_labels check is what - // ultimately decides whether the re-aggregation is - // semantically valid. The reverse direction (single-pop - // serving multi-pop) is NOT allowed — the single-pop - // policy has lost the key dimension and can't recover it. - // - // Exact counter summaries are a distinct state contract. A sum of - // cumulative sample values cannot reconstruct reset correction or - // Prometheus boundary extrapolation. - (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have || multi_pop_satisfies_single(*req, *have) - } + // Exact family identity must match. Grouping compatibility is + // checked separately by population routing. + (Capability::ExactAgg(req), Capability::ExactAgg(have)) => req == have, _ => false, } } @@ -348,30 +334,12 @@ fn sketch_algorithms_compatible( sketch_family_satisfied(required, available) } -/// True when `available` is the multi-population equivalent of -/// `required`'s single-population variant — i.e. a `MultipleSum` -/// policy can serve a `Sum` query (via re-aggregation across keys), -/// `MultipleIncrease` can serve `Increase`, `MultipleMax` can -/// serve `Max`. Asymmetric: this returns `false` for the reverse -/// direction (single-pop can't recover keys that have been collapsed -/// away). -fn multi_pop_satisfies_single(required: AggregationType, available: AggregationType) -> bool { - matches!( - (required, available), - (AggregationType::Sum, AggregationType::MultipleSum) - | (AggregationType::Increase, AggregationType::MultipleIncrease) - | (AggregationType::Min, AggregationType::MultipleMin) - | (AggregationType::Max, AggregationType::MultipleMax) - ) -} - // ── AggIntent → Capability bridge ──────────────────────────────────────────── #[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` for intents that have no ASAP-tier sketch -/// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent -/// — see [`AggIntent::archive_only`]). +/// answer it. Returns `None` when no deployed ASAP-tier capability can +/// satisfy the intent. /// /// This is a runtime routing requirement, not a summary-selection rule. /// ASAPPlanner owns legal implementations and candidate enumeration; this @@ -395,15 +363,17 @@ pub fn capability_for(intent: &AggIntent) -> Option { } match intent { AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Count { accuracy } if is_exact(accuracy) => { + Some(Capability::ExactAgg(AggregationType::Count)) + } // Direction is part of the capability: a stored minimum cannot // answer `max_over_time` and vice versa, so these must not // collapse onto one `ExactAgg` the way they did while Planner // had a single `MinMax` accumulator. AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase | AggIntent::Rate => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } + AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), + AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::QuantileApprox(None)) } @@ -462,7 +432,7 @@ mod tests { #[test] fn capability_for_cardinality_with_epsilon_returns_cardinality_approx() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Epsilon(0.01), }; assert_eq!(capability_for(&intent), Some(Capability::CardinalityApprox)); @@ -471,7 +441,7 @@ mod tests { #[test] fn capability_for_cardinality_with_epsilon_delta_returns_cardinality_approx() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::EpsilonDelta { epsilon: 0.01, delta: 0.001, @@ -483,7 +453,7 @@ mod tests { #[test] fn capability_for_cardinality_with_exact_returns_none() { let intent = AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: AccuracyTarget::Exact, }; assert_eq!(capability_for(&intent), None); @@ -520,7 +490,7 @@ mod tests { accuracy: approximate.clone(), }); let cardinality = capability_for(&AggIntent::Cardinality { - col: None, + cols: vec![], accuracy: approximate, }); assert_eq!(count, Some(Capability::FrequencyEstimate(None))); @@ -532,18 +502,14 @@ mod tests { } #[test] - fn capability_for_count_exact_routes_to_archive() { - // `count_over_time` lowers to `Count{accuracy:Exact}`. The - // PR #200/#201 follow-up briefly routed this to - // `ExactAgg(Sum)`, but the data plane has no count - // accumulator — `SumAccumulator` returns its `sum` for both - // `Statistic::Sum` and `Statistic::Count`, so the result was - // sum-of-values, not sample-count. Reverted to `None` (archive - // routing) until a real `SumCountAccumulator` lands. + fn capability_for_count_exact_preserves_count_family() { let intent = AggIntent::Count { accuracy: AccuracyTarget::Exact, }; - assert_eq!(capability_for(&intent), None); + assert_eq!( + capability_for(&intent), + Some(Capability::ExactAgg(AggregationType::Count)) + ); } #[test] @@ -593,16 +559,16 @@ mod tests { assert!(!Capability::ExactAgg(AggregationType::Max) .is_satisfied_by(&Capability::ExactAgg(AggregationType::Min))); assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleMax))); + .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); } #[test] - fn capability_for_rate_increase_route_to_exact_agg_increase() { - // PR-6 follow-up: Rate and Increase route to ASAP-tier - // ExactAgg(Increase) — the counter-reset-aware exact precompute. - // Pre-follow-up this returned `None`. + fn capability_for_rate_and_increase_preserves_family() { let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!(capability_for(&AggIntent::Rate), exact_inc); + assert_eq!( + capability_for(&AggIntent::Rate), + Some(Capability::ExactAgg(AggregationType::Rate)) + ); assert_eq!(capability_for(&AggIntent::Increase), exact_inc); } @@ -849,10 +815,6 @@ mod tests { AggregationType::Min, AggregationType::Max, AggregationType::DatasketchesKLL, - AggregationType::MultipleSum, - AggregationType::MultipleIncrease, - AggregationType::MultipleMin, - AggregationType::MultipleMax, AggregationType::HydraKLL, AggregationType::CountMinSketch, AggregationType::CountMinSketchWithHeap, @@ -873,21 +835,16 @@ mod tests { fn sum_family_cannot_impersonate_exact_counter_state() { let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); - assert!( - !required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum)) - ); + let required_multi = Capability::ExactAgg(AggregationType::Increase); + assert!(!required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } #[test] fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { - // Same single/multi-population direction as multi_pop_satisfies_single: - // a single-pop available (Sum) can't serve a multi-pop required - // capability (MultipleIncrease) -- it already lost the per-key - // breakdown a multi-pop caller needs. - let required = Capability::ExactAgg(AggregationType::MultipleIncrease); + // A different exact family cannot supply counter state. + let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } @@ -905,30 +862,26 @@ mod tests { // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] - fn exact_agg_routing_covers_sum_rate_increase_only() { - // `Capability::ExactAgg` routing covers the three intents the - // data plane has a real accumulator for: `Sum` (SumAccumulator) - // and `Rate` / `Increase` (IncreaseAccumulator). + fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { assert_eq!( capability_for(&AggIntent::Sum { col: None }), Some(Capability::ExactAgg(AggregationType::Sum)) ); assert_eq!( capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Increase)) + Some(Capability::ExactAgg(AggregationType::Rate)) ); assert_eq!( capability_for(&AggIntent::Increase), Some(Capability::ExactAgg(AggregationType::Increase)) ); - // `Count{Exact}` (count_over_time) and `Avg` both need a real - // count accumulator that doesn't exist yet — they route to - // archive until `SumCountAccumulator` lands. + // Exact count follows the Planner Count family; Avg still needs + // its own composition contract. assert_eq!( capability_for(&AggIntent::Count { accuracy: AccuracyTarget::Exact, }), - None + Some(Capability::ExactAgg(AggregationType::Count)) ); assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); } diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 71f7bfa24..963ebccac 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -9,7 +9,7 @@ mod status; pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; #[cfg(test)] -use super::compiler::PhysicalPlanCompiler; +use super::compiler::DeploymentPlanCompiler; use std::collections::{BTreeMap, BTreeSet}; @@ -1034,7 +1034,7 @@ mod tests { ); entries.push(second); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request.clone(), env) .unwrap(); let costs = manifest(&plan, &request.queries).unwrap(); @@ -1091,7 +1091,7 @@ mod tests { .as_ref() .unwrap() .len(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), environment.clone()) .unwrap(); assert!(identities.insert(plan.envelope.plan_id)); @@ -1146,7 +1146,7 @@ mod tests { let quotes = candidates .iter() .map(|candidate| { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -1186,10 +1186,10 @@ mod tests { let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(candidates.len() >= 2); - let local = PhysicalPlanCompiler + let local = DeploymentPlanCompiler .compile_promql(candidates[0].clone(), environment.clone()) .unwrap(); - let native = PhysicalPlanCompiler + let native = DeploymentPlanCompiler .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert_ne!(local.envelope.plan_id, native.envelope.plan_id); @@ -1245,7 +1245,7 @@ mod tests { .unwrap() .pop() .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(exact.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &exact.queries).unwrap(); @@ -1397,7 +1397,7 @@ mod tests { fn second_consumer_adds_reads_not_another_shared_state() { let (request, env) = fixture().into_physical_compilation_request().unwrap(); let first = manifest( - &PhysicalPlanCompiler + &DeploymentPlanCompiler .compile_promql(request.clone(), env.clone()) .unwrap(), &request.queries, @@ -1438,7 +1438,7 @@ mod tests { &shared.exact_composition_costs, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(shared.clone(), env) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index bf6b73187..838966c05 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -179,7 +179,7 @@ where *input = remap[input]; } } - QueryPlanNode::CandidateTopK { inputs, .. } + QueryPlanNode::MembershipFilter { inputs, .. } | QueryPlanNode::Binary { inputs, .. } | QueryPlanNode::RelationalJoin { inputs, .. } => { for input in inputs { @@ -257,6 +257,7 @@ where right, kind, pred, + pruning: None, } if self.preserve_relational => QueryPlanNode::RelationalJoin { inputs: [self.lower(left)?, self.lower(right)?], join_kind: kind.clone(), @@ -269,9 +270,6 @@ where right_schema: right.schema.clone(), output_schema: node.schema.clone(), }, - SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { - reason: "read-time relational join requires the relational compiler".into(), - }, SummaryExpr::ValueOperation { child, operation, .. } if self.preserve_relational @@ -308,7 +306,7 @@ where .. }, ), - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if measures.len() == 1 => { use planner_types::pre_asap::AggIntent; let operation = match &measures[0] { @@ -367,19 +365,29 @@ where } SummaryExpr::ValueOperation { child: sort, - operation: planner_types::post_asap::ValueOperation::Limit { n, offset: 0 }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + operation: + planner_types::post_asap::ValueOperation::Limit { + n, + offset: 0, + partition_by: limit_partition, + }, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } => { let SummaryExpr::ValueOperation { child, operation: planner_types::post_asap::ValueOperation::Sort { keys, partition_by }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } = &sort.expr else { return Err(QueryPlanError::Invalid( "query-time Limit must consume a query-time Sort".into(), )); }; + if limit_partition != partition_by { + return Err(QueryPlanError::Invalid( + "TopK Limit and Sort grouping differ".into(), + )); + } if keys.len() != 1 || keys[0].ascending { return Err(QueryPlanError::Invalid( "only descending value-ranked TopK is executable".into(), @@ -434,7 +442,7 @@ where SummaryExpr::ValueOperation { child, operation: planner_types::post_asap::ValueOperation::Sort { keys, .. }, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if keys.len() == 1 => QueryPlanNode::Logical { operator: residual::ResidualQueryOperator::Sort { descending: !keys[0].ascending, @@ -444,43 +452,17 @@ where SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { reason: "unsupported post-ASAP value operation".into(), }, - SummaryExpr::CandidateTopK { - candidates, - values, - k, - grouping, - completeness, - } => { - let labels = grouping - .keys() - .iter() - .map(|&column| { - values - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid( - "unresolved CandidateTopK grouping column".into(), - ) - }) - }) - .collect::, _>>()?; + SummaryExpr::RelationalJoin { + right: candidates, + left: values, + kind: planner_types::pre_asap::JoinKind::Semi, + pruning: Some(completeness), + pred, + } if !self.preserve_relational => { + validate_membership_join(pred, &values.schema, &candidates.schema)?; let candidate_input = self.lower(candidates)?; let value_input = if let Some(original) = &self.logical_source { - let parsed = promql_parser::parser::parse(original) - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - let promql_parser::parser::Expr::Aggregate(aggregate) = parsed else { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires a top-level PromQL aggregate".into(), - )); - }; - if aggregate.op.to_string() != "topk" { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires a topk source expression".into(), - )); - } + let exact_expression = residual::selected_native_expression(original, values)?; fn item_label(node: &SummaryNode) -> Option { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => { @@ -500,7 +482,7 @@ where } let item_label = item_label(candidates).ok_or_else(|| { QueryPlanError::Invalid( - "CandidateTopK membership has no named item label".into(), + "MembershipFilter membership has no named item label".into(), ) })?; let value_id = QueryNodeId(self.next_id); @@ -510,7 +492,7 @@ where QueryPlanNode::ExternalExact { request: ExternalExactRequest { language: QueryLanguage::PromQl, - expression: aggregate.expr.to_string(), + expression: exact_expression.to_string(), output: ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, @@ -526,23 +508,19 @@ where } else { self.lower(values)? }; - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [candidate_input, value_input], - k: u64::try_from(*k).map_err(|_| { - QueryPlanError::Invalid("CandidateTopK k exceeds u64".into()) - })?, - grouping: residual::Grouping { - labels, - without: grouping.is_without(), - }, completeness: completeness.clone(), } } + SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { + reason: "read-time relational join requires the relational compiler".into(), + }, SummaryExpr::BinaryOp { lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if self.logical_source.is_some() || operator.checked_relative_division || operator.checked_finite_division => @@ -624,7 +602,7 @@ where lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } if exact_value_executable(node) => { let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind else { @@ -782,7 +760,7 @@ where input: self.lower(summary_input)?, query: query.clone().into(), }, - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { if children.is_empty() { QueryPlanNode::ExactFallback { reason: "empty summary_merge".into(), @@ -881,7 +859,7 @@ pub(crate) fn exact_value_executable(node: &SummaryNode) -> bool { lhs, rhs, operator, - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, } => { matches!( operator.kind, @@ -1029,6 +1007,9 @@ mod catalog_binding_tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: config.policy_fingerprint().into(), + stored_output_reference: catalog + .output_reference(config.policy_fingerprint().into()) + .unwrap(), output_grouping: PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1048,6 +1029,7 @@ mod catalog_binding_tests { plan_id: 7, plan_version: 2, clickhouse_context: None, + selected_dags: Default::default(), entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), }, catalog, @@ -1157,6 +1139,9 @@ mod catalog_binding_tests { SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); let (mut counter_plan, _) = fixture(); binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); + binding(&mut counter_plan).stored_output_reference = counter_catalog + .output_reference(counter.policy_fingerprint().into()) + .unwrap(); as_rate_plan(counter_plan) .validate_against_catalog(&counter_catalog) .unwrap(); @@ -1199,6 +1184,9 @@ mod tests { Ok(MaterializationBinding { full_window_slide_ms: None, materialization: PolicyFingerprint(7).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + PolicyFingerprint(7).into(), + ), output_grouping: PhysicalGrouping::PerEntity, window_ms: 300_000, pane_origin_ms: Some(0), @@ -1326,6 +1314,7 @@ mod tests { tables: Default::default(), accuracy: planner_types::types::AccuracyTarget::Exact, }), + selected_dags: Default::default(), entries: [base, metricsql, clickhouse] .into_iter() .map(|entry| (QueryPlan::catalog_key(entry.language, "shared"), entry)) @@ -1379,7 +1368,7 @@ mod tests { } #[test] - fn candidate_topk_rejects_invalid_completeness_contract() { + fn membership_filter_rejects_invalid_completeness_contract() { let leaf = QueryPlanNode::ExactFallback { reason: "prepared".into(), }; @@ -1394,13 +1383,8 @@ mod tests { (QueryNodeId(1), leaf), ( QueryNodeId(2), - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [QueryNodeId(0), QueryNodeId(1)], - k: 2, - grouping: residual::Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::Certified { guarantee: planner_types::post_asap::ResultGuarantee { metric: planner_types::post_asap::ErrorMetric::Frequency, @@ -1427,3 +1411,130 @@ mod tests { assert!(entry.validate(&BTreeSet::new()).is_err()); } } + +// The vector adapter matches complete label identities. Reject joins whose +// predicate would require a different projection instead of silently widening it. +fn validate_membership_join( + pred: &planner_types::pre_asap::Predicate, + left: &planner_types::post_asap::SummarySchema, + right: &planner_types::post_asap::SummarySchema, +) -> Result<(), QueryPlanError> { + use planner_types::post_asap::SummaryFamilyType; + use planner_types::pre_asap::{CompareOpKind, DataType, QueryExpr}; + use std::collections::BTreeSet; + fn collect( + expr: &QueryExpr, + width: usize, + keys: &mut Vec<(usize, usize)>, + ) -> Result<(), QueryPlanError> { + match expr { + QueryExpr::BoolAnd(parts) => { + for part in parts { + collect(part, width, keys)?; + } + } + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => { + let (QueryExpr::Column(a), QueryExpr::Column(b)) = (left.as_ref(), right.as_ref()) + else { + return Err(QueryPlanError::Invalid( + "membership join requires column equality".into(), + )); + }; + let (a, b) = if a < b { (*a, *b) } else { (*b, *a) }; + if a >= width || b < width { + return Err(QueryPlanError::Invalid( + "membership join requires cross-input keys".into(), + )); + } + keys.push((a, b - width)); + } + _ => { + return Err(QueryPlanError::Invalid( + "unsupported membership join predicate".into(), + )) + } + } + Ok(()) + } + let labels = |schema: &planner_types::post_asap::SummarySchema| { + schema + .fields + .iter() + .filter(|f| { + f.name != "__name__" && matches!(f.dtype, SummaryFamilyType::Plain(DataType::Utf8)) + }) + .map(|f| f.name.clone()) + .collect::>() + }; + let mut keys = Vec::new(); + collect(&pred.0, left.fields.len(), &mut keys)?; + let mut matched = BTreeSet::new(); + for (a, b) in keys { + let Some((a, b)) = left.fields.get(a).zip(right.fields.get(b)) else { + return Err(QueryPlanError::Invalid( + "membership join key out of bounds".into(), + )); + }; + if a.name != b.name { + return Err(QueryPlanError::Invalid( + "membership join requires matching label names".into(), + )); + } + matched.insert(a.name.clone()); + } + if matched.is_empty() || matched != labels(left) || matched != labels(right) { + return Err(QueryPlanError::Invalid( + "membership join must match the complete label identity".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod membership_binding_tests { + use super::*; + use planner_types::{ + post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, + pre_asap::{CompareOpKind, DataType, Predicate, QueryExpr}, + }; + + // This adapter can implement full-label equality only; reject narrower joins. + #[test] + fn membership_binding_rejects_other_join_semantics() { + let schema = |names: &[&str]| SummarySchema { + fields: names + .iter() + .map(|name| SummaryField { + name: (*name).into(), + dtype: SummaryFamilyType::Plain(DataType::Utf8), + nullable: false, + }) + .collect(), + time_index: None, + }; + let eq = |a, b| QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(a)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(b)), + }; + let one = schema(&["job"]); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 1))), &one, &one).is_ok()); + assert!( + validate_membership_join(&Predicate(Rc::new(eq(0, 1))), &one, &schema(&["host"])) + .is_err() + ); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 2))), &one, &one).is_err()); + assert!(validate_membership_join( + &Predicate(Rc::new(QueryExpr::BoolAnd(vec![]))), + &one, + &one + ) + .is_err()); + let two = schema(&["job", "host"]); + assert!(validate_membership_join(&Predicate(Rc::new(eq(0, 2))), &two, &two).is_err()); + } +} diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index 7e64c0089..3c7e51f05 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -361,11 +361,32 @@ pub(super) fn residual_nodes( horizons(residual, &mut intervals); intervals.sort_unstable(); intervals.dedup(); + // Accuracy annotations select a candidate, but exact execution still + // implements that candidate's computation. Reconstruct the same typed IR + // before comparing it; do not erase operators or source predicates. + let accuracy = match residual { + planner_types::pre_asap::QueryExpr::Aggregate { measures, .. } => measures + .iter() + .find_map(|intent| { + use planner_types::pre_asap::AggIntent; + match intent { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } + | AggIntent::FrequencyL2 { accuracy, .. } + | AggIntent::FrequencyEntropy { accuracy, .. } => Some(accuracy.clone()), + _ => None, + } + }) + .unwrap_or(planner_types::types::AccuracyTarget::Exact), + _ => planner_types::types::AccuracyTarget::Exact, + }; for expression in expressions { for interval in &intervals { if let Ok(candidate) = crate::query_parser::parse_query_expr_with_interval( &expression.to_string(), - planner_types::types::AccuracyTarget::Exact, + accuracy.clone(), *interval, ) { if &candidate == residual { @@ -442,11 +463,34 @@ pub(crate) fn selected_residual_nodes( original: &str, selected: &planner_types::post_asap::SummaryNode, ) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { + let expression = selected_native_expression(original, selected)?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expression)?; + Ok((root, lower.nodes)) +} + +/// Resolve the selected exact subtree to a verified native expression before +/// binding an external input. Never substitute the top-level query's child. +pub(super) fn selected_native_expression( + original: &str, + selected: &planner_types::post_asap::SummaryNode, +) -> Result { if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { return Err(invalid( "native residual substitution requires an exact selected value", )); } + let selected = match &selected.expr { + planner_types::post_asap::SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + .. + } => child.as_ref(), + _ => selected, + }; fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { output.push(expr); match expr { @@ -485,11 +529,6 @@ pub(crate) fn selected_residual_nodes( rhs: right, .. } - | SummaryExpr::CandidateTopK { - candidates: left, - values: right, - .. - } | SummaryExpr::RelationalJoin { left, right, .. } | SummaryExpr::SummaryJoin { outer: left, @@ -500,7 +539,7 @@ pub(crate) fn selected_residual_nodes( selected_horizons(left, out); selected_horizons(right, out); } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { selected_horizons(child, out); } @@ -530,12 +569,7 @@ pub(crate) fn selected_residual_nodes( let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) .replacements(&TargetSubDAG::new(&root)); if candidates.iter().any(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if node.as_ref() == selected)) { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(expression)?; - let candidate = (root, lower.nodes); + let candidate = expression.clone(); if matched .as_ref() .is_some_and(|previous| previous != &candidate) @@ -575,6 +609,24 @@ pub(super) fn selected_aggregate_operator( mod hybrid_tests { use super::*; use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; + #[test] + fn external_binding_rejects_an_unrelated_selected_exact_subtree() { + let exact = crate::query_parser::parse_query_expr_with_interval( + "sum_over_time(other_metric[5m])", + planner_types::types::AccuracyTarget::Exact, + 1_000, + ) + .unwrap(); + let selected = crate::planner_selection::plan_test_query(&exact).unwrap(); + assert!(selected_native_expression("topk(2, sum_over_time(m[5m]))", &selected).is_err()); + assert_eq!( + selected_native_expression("sum_over_time(other_metric[5m])", &selected) + .unwrap() + .to_string(), + "sum_over_time(other_metric[5m])" + ); + } + #[test] fn selected_summary_and_filtered_residual_share_installed_binary() { // Both filtered and unfiltered leaves bind independently. @@ -607,6 +659,14 @@ mod hybrid_tests { if spatial_filter.is_empty() { 7 } else { 8 }, ) .into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + asap_types::PolicyFingerprint(if spatial_filter.is_empty() { + 7 + } else { + 8 + }) + .into(), + ), output_grouping: PhysicalGrouping::PerEntity, window_ms: 300_000, pane_origin_ms: Some(0), @@ -668,7 +728,7 @@ mod hybrid_tests { #[cfg(test)] mod planner_workload_tests { use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { let mut fixture: serde_json::Value = serde_json::from_str(include_str!( @@ -683,7 +743,7 @@ mod planner_workload_tests { let (request, environment) = snapshot .into_physical_compilation_request() .unwrap_or_else(|error| panic!("{query}: {error}")); - PhysicalPlanCompiler + DeploymentPlanCompiler .compile_promql(request, environment) .unwrap_or_else(|error| panic!("{query}: {error}")) } @@ -769,7 +829,7 @@ mod planner_workload_tests { let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.allow_mixed_summary_and_exact_execution); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.query_plan.entries.len(), 24); @@ -1077,19 +1137,13 @@ pub fn eligible_materialization_keys( visit(original, left, keys)?; visit(original, right, keys)?; } - SummaryExpr::CandidateTopK { - candidates, values, .. - } => { - visit(original, candidates, keys)?; - visit(original, values, keys)?; - } SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryEstimate { summary_input, .. } | SummaryExpr::SummaryDelete { summary_input, .. } => { visit(original, summary_input, keys)? } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { for child in children { visit(original, child, keys)?; } @@ -1300,7 +1354,7 @@ mod remote_boundary_regressions { use super::*; #[test] - fn summary_definition_identity_is_independent_of_matcher_order() { + fn stored_output_identity_is_independent_of_matcher_order() { let first = LabelMatcher { name: "job".into(), value: "orders".into(), diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 6bb7fc327..d6b74cd45 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -18,8 +18,7 @@ use planner_types::pre_asap::AggIntent; /// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` /// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): /// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Sum`] -/// (rate binds to ExactAgg(Sum)-shaped capability) +/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] /// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] /// /// Before this enum: the `WorkloadStore` was keyed by metric name alone @@ -30,7 +29,7 @@ use planner_types::pre_asap::AggIntent; /// shape). /// /// After: the store is keyed by `(metric, role)` so each shape gets its -/// own plan, its own `AggregationConfig` on the backend's streaming +/// own plan, its own `PrecomputeMaterialization` on the backend's streaming /// config, and its own routing-connector pipeline. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -39,15 +38,15 @@ pub enum AggRole { /// or workload entries with `sketch_family_override: DDSketch | KLL`. /// Routes to a quantile-shaped sketch (DDSketch / KLL). Quantile, - /// Bare counter selector, `sum(...)`, `sum_over_time(...)`, - /// `rate(...)`, `increase(...)`. All bind to ExactAgg(Sum)-shaped - /// capability on the data plane; the streaming-config emits an - /// `aggregation_type: Sum` rather than a sketch. + /// Bare selector or Sum-shaped exact aggregation. Sum, + /// Reset-aware per-second counter rate. + Rate, + /// Reset-aware counter increase over the selected window. + Increase, /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to a Sum-as-count - /// exact-aggregation. + /// to HLL when a sketch is appropriate, otherwise to exact Count. Count, /// `topk(...)`, or workload entries with /// `sketch_family_override: CountSketch | CountMinSketch`. Routes @@ -70,6 +69,8 @@ impl AggRole { match self { AggRole::Quantile => "quantile", AggRole::Sum => "sum", + AggRole::Rate => "rate", + AggRole::Increase => "increase", AggRole::Count => "count", AggRole::Topk => "topk", AggRole::Other => "other", @@ -97,15 +98,16 @@ impl std::fmt::Display for AggRole { /// through the same canonical pipeline the live serving path uses /// (`query_parser::parse_query_expr_canonical` → /// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent (the one bound to the data-plane capability) is matched: +/// intent is matched, except that a Sum wrapping a counter function +/// keeps the inner Rate or Increase role: /// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] /// * [`AggIntent::TopK`] → [`AggRole::Topk`] /// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the /// windowed-Count-as-Frequency extension /// (`intent_algebra::as_frequency`) → [`AggRole::Count`] /// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → [`AggRole::Sum`] -/// * Anything else recognised but not one of the four shapes above +/// → their respective roles +/// * Anything else recognised but not one of the listed shapes above /// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → /// [`AggRole::Other`]. /// * Bare metric selector (no `Aggregate` node at all) → @@ -125,11 +127,7 @@ impl std::fmt::Display for AggRole { /// semantics, whichever the lowerer picks). The Sum-shaped /// alternative is rare in practice; users who want it write /// `sum_over_time(count(...))` which classifies as Sum. -/// * `rate` / `irate` / `increase` — Sum. `irate` folds onto -/// `AggIntent::Rate` at L3 same as `rate`; both bind to -/// ExactAgg(Increase) on the data plane (see -/// `data_plane/src/precompute_engine/ingest_handler.rs`'s handling -/// of `AggKind::ExactAgg { Increase }`). +/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { // 1. `sketch_family_override` wins. if let Some(family) = entry.sketch_family_override.as_ref() { @@ -167,11 +165,30 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { if crate::planner_selection::as_frequency(outer).is_some() { return AggRole::Count; } + // A spatial `sum by (...)` around a counter function still requires the + // counter family's state; using Sum as the registration key would let it + // overwrite a bare Sum workload for the same metric. + if matches!(outer, AggIntent::Sum { .. }) { + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Rate)) + { + return AggRole::Rate; + } + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Increase)) + { + return AggRole::Increase; + } + } match outer { AggIntent::Quantile { .. } => AggRole::Quantile, AggIntent::TopK { .. } => AggRole::Topk, AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } | AggIntent::Rate | AggIntent::Increase => AggRole::Sum, + AggIntent::Sum { .. } => AggRole::Sum, + AggIntent::Rate => AggRole::Rate, + AggIntent::Increase => AggRole::Increase, _ => AggRole::Other, } } @@ -970,13 +987,7 @@ mod tests { #[test] fn agg_role_sum_query_strings() { - for q in [ - "sum by (zone) (m)", - "sum_over_time(m[5m])", - "rate(m[5m])", - "increase(m[5m])", - "sum by (zone) (rate(m[5m]))", - ] { + for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { assert_eq!( derive_agg_role(&entry("m", Some(q), None)), AggRole::Sum, @@ -985,6 +996,18 @@ mod tests { } } + #[test] + fn counter_functions_have_distinct_workload_roles() { + for (query, expected) in [ + ("rate(m[5m])", AggRole::Rate), + ("sum by (zone) (rate(m[5m]))", AggRole::Rate), + ("increase(m[5m])", AggRole::Increase), + ("sum by (zone) (increase(m[5m]))", AggRole::Increase), + ] { + assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); + } + } + #[test] fn agg_role_count_query_strings() { for q in [ @@ -1095,7 +1118,7 @@ mod tests { } #[test] - fn three_synthetic_http_requests_total_entries_classify_to_two_distinct_roles() { + fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { // Synthetic mirror of `deploy/configs/mvp-workload.yaml` // entries 2/3/4 — proves `derive_agg_role` produces distinct // roles for the three http_requests_total shapes. Pre-B2 the @@ -1120,15 +1143,8 @@ mod tests { ), ]; let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Sum, AggRole::Count]); - // The store distinguishes Sum vs Count keys, so two of the - // three entries (the two Sum-shaped ones) still collide - // under (metric, role). That's the documented behaviour — - // two YAML entries with the SAME (metric, role) overwrite, - // which is the legitimate "operator updated their workload" - // path. The fix scope is collisions across DIFFERENT shapes, - // not idempotent re-registers. + assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); + assert_eq!(distinct.len(), 3); } } diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 5e334d261..ded77eed3 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -274,10 +274,25 @@ fn sketch(node: &SummaryNode) -> (&SketchAlgorithm, &SketchParams) { } } -/// The actual control-plane parser/binder selects the lower measured update -/// cost while preserving the selected algorithm's normal parameter sizing. +fn assert_exact_count(node: &SummaryNode) { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => assert_exact_count(summary_input), + SummaryExpr::SummaryAgg { + family: + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + planner_types::post_asap::ExactParams::Count, + ), + .. + } => {} + other => panic!("expected exact total-count state, got {other:?}"), + } +} + +/// Update evidence ranks frequency sketches but does not replace an exact +/// total-count accumulator with a point-frequency sketch. #[test] -fn offline_update_evidence_changes_typed_binding() { +fn offline_update_evidence_preserves_exact_count_binding() { let default = model(); let (artifact, context) = fixture(&default, &intent()); let empirical = @@ -292,12 +307,8 @@ fn offline_update_evidence_changes_typed_binding() { ); let default_bound = bound(&default); let measured_bound = bound(&empirical); - assert_eq!(sketch(&default_bound).0, &SketchAlgorithm::Cms); - assert_eq!(sketch(&measured_bound).0, &SketchAlgorithm::CountSketch); - assert_eq!( - sketch(&measured_bound).1, - &default.size_params(SketchAlgorithm::CountSketch, &intent(), 0.01, 0.01) - ); + assert_exact_count(&default_bound); + assert_exact_count(&measured_bound); assert!(default_bound.guarantee.is_some()); assert!(measured_bound.guarantee.is_some()); // Observed zero point-frequency error has no effect on formal sizing. @@ -334,11 +345,7 @@ fn incompatible_evidence_preserves_deployment_behavior() { candidates(), "{scenario}" ); - assert_eq!( - sketch(&bound(&empirical)).0, - &SketchAlgorithm::Cms, - "{scenario}" - ); + assert_exact_count(&bound(&empirical)); } } @@ -351,7 +358,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: child.clone(), rhs: child.clone(), operator: BinaryOperator { diff --git a/crates/asap_sketch_codec/Cargo.toml b/crates/asap_sketch_codec/Cargo.toml new file mode 100644 index 000000000..c2d728895 --- /dev/null +++ b/crates/asap_sketch_codec/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "asap_sketch_codec" +version.workspace = true +edition.workspace = true + +[dependencies] +asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch = "main" } +prost = "0.13" diff --git a/crates/asap_sketch_codec/src/lib.rs b/crates/asap_sketch_codec/src/lib.rs new file mode 100644 index 000000000..279efe168 --- /dev/null +++ b/crates/asap_sketch_codec/src/lib.rs @@ -0,0 +1,84 @@ +//! Runtime-independent decoding of the sketchlib protobuf envelope. + +use asap_sketchlib::proto::sketchlib::{ + sketch_envelope::SketchState, DdSketchState, KllState, SketchEnvelope, +}; +use asap_sketchlib::DdSketch; +use prost::Message; + +pub fn envelope_state(bytes: &[u8]) -> Result, String> { + SketchEnvelope::decode(bytes) + .map(|envelope| envelope.sketch_state) + .map_err(|error| format!("decode SketchEnvelope: {error}")) +} + +pub fn ddsketch_state(bytes: &[u8]) -> Result<(DdSketchState, f64), String> { + let envelope = + SketchEnvelope::decode(bytes).map_err(|error| format!("decode SketchEnvelope: {error}"))?; + match envelope.sketch_state { + Some(SketchState::Ddsketch(state)) => Ok((state, envelope.sample_p)), + _ => Err("SketchEnvelope contains no DDSketch state".into()), + } +} + +pub fn reconstruct_ddsketch(bytes: &[u8]) -> Result<(DdSketch, f64), String> { + let (state, sample_p) = ddsketch_state(bytes)?; + if !state.alpha.is_finite() || !(0.0..1.0).contains(&state.alpha) || state.alpha == 0.0 { + return Err("DDSketch alpha must be finite and between zero and one".into()); + } + Ok(( + DdSketch::from_raw(state.alpha, state.store_counts, state.store_offset), + sample_p, + )) +} + +pub fn kll_state(bytes: &[u8]) -> Result { + let envelope = + SketchEnvelope::decode(bytes).map_err(|error| format!("decode SketchEnvelope: {error}"))?; + match envelope.sketch_state { + Some(SketchState::Kll(state)) => Ok(state), + _ => Err("SketchEnvelope contains no KLL state".into()), + } +} + +pub fn encode_ddsketch(sketch: &DdSketch) -> Vec { + let envelope = SketchEnvelope { + format_version: 1, + producer: None, + hash_spec: None, + sample_p: 0.0, + sketch_state: Some(SketchState::Ddsketch(DdSketchState { + alpha: sketch.wire_alpha(), + store_counts: sketch.store_counts.clone(), + store_offset: sketch.store_offset, + })), + }; + envelope.encode_to_vec() +} + +pub fn encode_kll(sketch: &asap_sketchlib::sketches::kll::KLL) -> Vec { + use asap_sketchlib::proto::sketchlib::CoinState; + let (state, bit_cache, remaining_bits) = sketch.wire_coin(); + SketchEnvelope { + format_version: 1, + producer: None, + hash_spec: None, + sample_p: 0.0, + sketch_state: Some(SketchState::Kll(KllState { + k: sketch.wire_k(), + m: sketch.wire_m(), + num_levels: sketch.wire_num_levels(), + levels: sketch.wire_levels(), + items: sketch.wire_items(), + coin: Some(CoinState { + state, + bit_cache, + remaining_bits, + }), + offset: 0.0, + value_scale: 0, + residuals: Vec::new(), + })), + } + .encode_to_vec() +} diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 482ef6d67..bdf5949b4 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,68 +1,12 @@ -//! Typed accumulator dispatch derived from legacy streaming config. +//! Validate stored materialization descriptors against Planner summary families. //! -//! The semantic identity is ASAPPlanner's [`SummaryFamilyType`]. This module -//! only adds the backend execution concern of keyed versus unkeyed state and -//! adapts the stable legacy wire fields into that canonical representation. -//! -//! ## This is an additive representation, not a replacement (yet) -//! -//! `AggregationConfig` keeps its `aggregation_type` / `aggregation_sub_type` -//! / `parameters` fields untouched. Two hard constraints ruled out full -//! removal in this pass: -//! -//! 1. **`PolicyFingerprint` hash stability.** [`crate::policy_fingerprint`] -//! hashes `aggregation_type` / `aggregation_sub_type` / `parameters` -//! directly, and its own module doc is explicit that the byte layout -//! it produces is a stability *contract* ("Don't reorder fields... -//! any such change invalidates every deployed fingerprint and forces -//! a cold-start rebuild"). Changing what feeds that hash — even by -//! routing it through an equivalent typed shape — risks producing a -//! different byte sequence for the same logical policy, which strands -//! on-disk sids after a deploy. `policy_fingerprint.rs` is -//! deliberately **not touched** by this module; it keeps reading the -//! original three fields, unchanged. -//! 2. **Consumer fan-out.** `AggregationType` is read by ~40 files across -//! `data_plane` and `asap_types` — persistence (`sid_metadata.json` -//! round-trip), query-time capability matching -//! (`capability_matching.rs`, unrelated to accumulator dispatch), -//! the query engine, reconciliation, index maintenance — not just -//! `accumulator_factory.rs` (the single highest-risk consumer, and -//! the one this module targets). Migrating all of them in one PR was -//! judged too large to land and review safely; that's tracked as -//! follow-up, not done here. -//! -//! So: `AccumulatorSpec` is *computed from* `AggregationConfig`'s -//! existing fields via [`AggregationConfig::accumulator_spec`], and -//! consumed by `data_plane::precompute_engine::accumulator_factory` -//! instead of the raw fields. The wire format (`aggregationType` / -//! `aggregationSubType` / `parameters` JSON/YAML keys) is completely -//! unaffected — nothing here changes how `AggregationConfig::from_yaml` -//! / `from_json` parse or how `serialize_to_json` emits. -//! -//! Backend-specific execution details remain deliberately separate: -//! -//! - **Min/max direction.** Direction is part of the family now, not a -//! string riding alongside it: `AggregationType::{Min, Max}` (and the -//! keyed `{MultipleMin, MultipleMax}`) map to `ExactKind::Min` and -//! `ExactKind::Max` respectively — upstream still spells its -//! maximum accumulator `MinMax`, but it is a maximum. Nothing reads -//! `AggregationConfig::aggregation_sub_type` for the direction any -//! more, so a min state can no longer content-address onto a max one. -//! - **HydraKLL's `(row, col)` tiling.** `SketchParams::Kll` carries -//! only `k` — upstream has no concept of the CMS-like grid-of-KLL-cells -//! layout `HydraKllSketchAccumulator` uses to parallelize a keyed KLL -//! across many populations. `accumulator_factory.rs` calls -//! [`cms_params`] directly for keyed KLL execution -//! arm, same extraction the plain CMS arms use, because `w`/`d` are -//! genuinely the same wire keys for both. -//! - **Top-k ranking mode (`weight_mode`).** Not a sketch structural -//! parameter — a data_plane-only "what to accumulate" axis -//! (`accumulator_factory::TopkWeight`) with no upstream equivalent. -//! Stays a raw-`parameters`-reading helper in `accumulator_factory.rs`. +//! This projection supports catalog identity and imported state metadata. It is +//! not an execution program. Raw and maintenance execution dispatch directly +//! on the selected post-ASAP DAG payload; the descriptor must agree with it. use serde_json::Value; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; use planner_types::post_asap::{ @@ -75,8 +19,8 @@ use planner_types::post_asap::{ /// accumulator to run (`kind`), with what tuning (`params`), and /// whether it's keyed by a group-by label set (`grouping`). /// -/// Computed on demand from an [`AggregationConfig`] via -/// [`AggregationConfig::accumulator_spec`] — not stored on the config +/// Computed on demand from an [`PrecomputeMaterialization`] via +/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config /// itself, so there is exactly one source of truth for the fields that /// feed [`crate::policy_fingerprint::PolicyFingerprint`]. #[derive(Debug, Clone, PartialEq)] @@ -85,10 +29,7 @@ pub struct AccumulatorSpec { /// validated `SketchKind` (category + algorithm + params), following the /// ASAP-aware-mapping vocabulary. pub family: SummaryFamilyType, - /// `Some(labels)` for a keyed (multi-population) accumulator, - /// `None` for a single-population one. This is the axis - /// `AggregationType` wrongly folded into identity (`Sum` vs - /// `MultipleSum`) — here it's a sibling field instead. + /// Physical keyed-state layout, independent of semantic family. pub grouping: Option, } @@ -96,8 +37,8 @@ pub struct AccumulatorSpec { /// /// This is execution semantics, separate from the summary family: the same /// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Legacy streaming artifacts still encode the rule in -/// `parameters`; callers use [`AggregationConfig::sample_update_rule`] so the +/// counter deltas. Stored descriptors encode the rule in +/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the /// runtime does not branch on ad-hoc strings. #[derive(Debug, Clone, Copy, PartialEq)] pub enum SampleUpdateRule { @@ -134,7 +75,7 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) } -impl AggregationConfig { +impl PrecomputeMaterialization { pub fn sample_update_rule(&self) -> SampleUpdateRule { let scale = self .parameters @@ -157,23 +98,14 @@ impl AggregationConfig { } } -/// Why [`AggregationConfig::accumulator_spec`] couldn't resolve a config -/// into an [`AccumulatorSpec`]. Each variant matches one of the three -/// distinct fallback paths `accumulator_factory::create_accumulator_updater` -/// took pre-Step-5 — preserved verbatim (including which default -/// updater and which warning text each one produced) so this refactor -/// changes *how* the dispatch is expressed, not what it does for any -/// input. +/// A storage descriptor cannot be resolved to a supported Planner family. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccumulatorSpecError { /// `aggregation_type` was `SingleSubpopulation` with an /// `aggregation_sub_type` string not in the recognized alias list. - /// Pre-Step-5 this defaulted to `SumAccumulatorUpdater`. UnknownSingleSubpopulationSubType(String), /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. Pre-Step-5 this defaulted - /// to `MultipleSumAccumulatorUpdater` (note: a *different* default - /// than the `SingleSubpopulation` case). + /// unrecognized `aggregation_sub_type`. UnknownMultipleSubpopulationSubType(String), /// `aggregation_type` itself has no accumulator-dispatch mapping. /// Also returned for an invalid HLL precision. A resolved family identifies @@ -185,36 +117,24 @@ impl std::fmt::Display for AccumulatorSpecError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::UnknownSingleSubpopulationSubType(s) => { - write!( - f, - "Unknown SingleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown SingleSubpopulation sub_type '{s}'") } Self::UnknownMultipleSubpopulationSubType(s) => { - write!( - f, - "Unknown MultipleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") } - Self::UnmappedAggregationType(t) => write!( - f, - "Unknown aggregation_type '{t:?}', defaulting to SingleSubpopulation Sum" - ), + Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), } } } impl std::error::Error for AccumulatorSpecError {} -impl AggregationConfig { +impl PrecomputeMaterialization { /// Resolve this config's `(aggregation_type, aggregation_sub_type, /// parameters)` triple into a typed [`AccumulatorSpec`]. /// - /// Mirrors `accumulator_factory::create_accumulator_updater`'s - /// pre-Step-5 dispatch exactly — same sub_type alias lists, same - /// numeric defaults, same three fallback paths (see - /// [`AccumulatorSpecError`]) — just re-expressed as data instead of - /// as a 14-arm match baked into the accumulator constructor. + /// Unsupported descriptors return an error; this projection never chooses + /// a fallback family and cannot authorize DAG execution. pub fn accumulator_spec(&self) -> Result { use AggregationType::*; @@ -227,21 +147,9 @@ impl AggregationConfig { ) }; let (family, keyed) = match self.aggregation_type { - Sum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - Increase => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - Min => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - Max => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, + Sum | Count | Increase | Rate | Min | Max => ( + self.aggregation_type.planner_exact_family().unwrap(), + !self.aggregated_labels.is_empty(), ), DatasketchesKLL => ( independent_sketch( @@ -252,22 +160,6 @@ impl AggregationConfig { ), false, ), - MultipleSum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - MultipleIncrease => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - MultipleMin => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - MultipleMax => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), HydraKLL => { let k = kll_k_param(self) as u32; ( @@ -497,7 +389,7 @@ impl AggregationConfig { /// Extract the KLL `k` parameter. Capital `"K"` takes precedence over /// lowercase `"k"` to match the convention used by the top-level /// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &AggregationConfig) -> u16 { +pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { config .parameters .get("K") @@ -513,7 +405,7 @@ pub fn kll_k_param(config: &AggregationConfig) -> u16 { /// matches what the control plane's `sketch_params_to_json` emits and /// what `sketch_config_to_params` uses for OTLP policy_fp content /// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { +pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { let row_num = config .parameters .get("d") @@ -530,7 +422,7 @@ pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { /// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` /// from `parameters`; defaults to 20 (the heap holds the top-k /// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &AggregationConfig) -> usize { +pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { config .parameters .get("heap_size") @@ -545,7 +437,7 @@ pub fn heap_size_param(config: &AggregationConfig) -> usize { /// Pull `relativeAccuracy` (or canonical aliases) out of a /// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, /// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { +pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { let parsed = param_f64(config, "relativeAccuracy") .or_else(|| param_f64(config, "relative_accuracy")) .or_else(|| param_f64(config, "alpha")) @@ -561,7 +453,7 @@ pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { } } -fn param_f64(config: &AggregationConfig, key: &str) -> Option { +fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { config.parameters.get(key).and_then(Value::as_f64) } @@ -599,8 +491,8 @@ mod tests { sub_type: &str, params: HashMap, grouping_labels: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), params, @@ -630,13 +522,9 @@ mod tests { } #[test] - fn multiple_sum_is_keyed_sum() { - let cfg = make_config( - AggregationType::MultipleSum, - "", - HashMap::new(), - vec!["zone"], - ); + fn keyed_layout_preserves_sum_family() { + let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); + cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); let spec = cfg.accumulator_spec().expect("resolves"); assert_exact(&spec, ExactKind::Sum); assert_eq!( @@ -862,16 +750,16 @@ mod tests { assert_eq!( AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown SingleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown MultipleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL', defaulting to SingleSubpopulation Sum" + "Unknown aggregation_type 'HLL'" ); } @@ -905,7 +793,7 @@ mod tests { // ---- PolicyFingerprint stability guard --------------------------- /// `accumulator_spec()` must be a pure, additional *read* of - /// `AggregationConfig` — it must not change what + /// `PrecomputeMaterialization` — it must not change what /// `PolicyFingerprint::from_config` hashes. This locks in a fixed /// fingerprint for a fixed config as a tripwire: if this test ever /// needs its expected constant updated, `policy_fingerprint.rs` diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 92dfd41dc..a170f3cd7 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -5,7 +5,6 @@ use std::collections::HashMap; use crate::enums::{QueryLanguage, WindowKind}; use crate::policy_fingerprint::PolicyFingerprint; -use crate::traits::SerializableToSink; use crate::utils::normalize_spatial_filter; use crate::AggregationType; use crate::KeyByLabelNames; @@ -87,10 +86,18 @@ impl WindowMaterializationLayout { } } -/// Per-aggregation policy with content-derived [`PolicyFingerprint`] identity. -/// An `aggregationId` field in input YAML is ignored for compatibility. +/// Physical materialization metadata with content-derived identity. +/// This descriptor cannot authorize execution: the enclosing PrecomputePlan +/// must bind it to a compatible Planner DAG producer. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct PrecomputeMaterialization { + /// Explicit deployment output allocation; independent of semantic identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stored_output_id: Option, + /// Planner-selected dependency closure ending at the persisted output. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub semantic_fragment: Option, pub aggregation_type: AggregationType, pub aggregation_sub_type: String, pub parameters: HashMap, @@ -114,13 +121,9 @@ pub struct PrecomputeMaterialization { pub window_type: WindowKind, // Tumbling or Sliding pub window_layout: WindowMaterializationLayout, /// Unix millisecond timestamp on the pane-boundary grid selected from - /// the consuming query workload. Missing on legacy definitions, which - /// must not be used for certified pane-only reads. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + /// the consuming query workload. An absent origin cannot authorize certified + /// pane-only reads. + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, pub spatial_filter: String, @@ -130,26 +133,12 @@ pub struct PrecomputeMaterialization { // SQL-specific fields (optional, used when query_language=sql) pub table_name: Option, // SQL mode: table name - #[serde( - default, - alias = "value_column", - alias = "valueColumn", - alias = "valueProjection", - deserialize_with = "crate::sds::deserialize_optional_value_projection" - )] + #[serde(default)] pub value_projection: Option, /// Table timestamp projection, in Unix milliseconds. - #[serde( - default, - alias = "tableTimestampColumn", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub table_timestamp_column: Option, - #[serde( - default, - alias = "tablePopulation", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub table_population: Option, /// Producer typing for a SQL table value projection: the source column's /// declared type and nullability. @@ -166,11 +155,7 @@ pub struct PrecomputeMaterialization { /// /// `None` ⇒ PromQL-mode materializations and legacy SQL definitions, which /// keep the pre-typed behaviour (read the column as it comes). - #[serde( - default, - alias = "valueSourceColumn", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub value_source_column: Option, } @@ -196,10 +181,6 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} -/// Compatibility name for legacy streaming-config and precompute call sites. -/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. -pub type AggregationConfig = PrecomputeMaterialization; - impl PrecomputeMaterialization { pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection @@ -296,6 +277,8 @@ impl PrecomputeMaterialization { let spatial_filter_normalized = normalize_spatial_filter(&spatial_filter); Self { + stored_output_id: None, + semantic_fragment: None, aggregation_type, aggregation_sub_type, parameters, @@ -338,171 +321,13 @@ impl PrecomputeMaterialization { /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s - /// `HashMap` keys). **Always** equal to + /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. pub fn policy_fp_u64(&self) -> u64 { self.policy_fingerprint().as_u64() } - pub fn deserialize_from_json( - data: &Value, - ) -> Result> { - if [ - "valueColumn", - "value_column", - "valueProjection", - "value_projection", - ] - .iter() - .filter(|key| data.get(**key).is_some_and(|value| !value.is_null())) - .count() - > 1 - { - return Err("multiple value projection fields are not allowed".into()); - } - // `aggregationId` is silently ignored — identity is - // content-addressed via PolicyFingerprint (PR 5). - - let aggregation_type: AggregationType = data["aggregationType"] - .as_str() - .ok_or("Missing aggregationType")? - .parse() - .map_err(|e: String| e)?; - - let aggregation_sub_type = data["aggregationSubType"] - .as_str() - .ok_or("Missing aggregationSubType")? - .to_string(); - - let parameters = data["parameters"] - .as_object() - .ok_or("Missing parameters")? - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - // Note: In Python, eval(data["originalYaml"]) is used, but this is unsafe - // Using the string value directly instead - let original_yaml = data["originalYaml"].as_str().unwrap_or("").to_string(); - - // Deserialize KeyByLabelNames - assuming they have deserialize_from_json methods - let grouping_labels = - crate::GroupingProjection::deserialize_from_json(&data["groupingLabels"])?; - let aggregated_labels = KeyByLabelNames::deserialize_from_json(&data["aggregatedLabels"])?; - let rollup_labels = KeyByLabelNames::deserialize_from_json(&data["rollupLabels"])?; - - let window_size = data["windowSize"].as_u64().ok_or("Missing windowSize")?; - - let window_type = data - .get("windowType") - .and_then(|v| v.as_str()) - .unwrap_or("tumbling") - .parse::() - .unwrap_or_default(); - - let slide_interval = data - .get("slideInterval") - .and_then(|v| v.as_u64()) - .unwrap_or(window_size); - - let pane_origin_ms = data - .get("paneOriginMs") - .or_else(|| data.get("pane_origin_ms")) - .and_then(|v| v.as_i64()); - - let spatial_filter = data["spatialFilter"].as_str().unwrap_or("").to_string(); - - let metric = data["metric"].as_str().ok_or("Missing metric")?.to_string(); - - let num_aggregates_to_retain = data.get("numAggregatesToRetain").and_then(|v| v.as_u64()); - - // SQL-specific fields (optional) - let table_name = data - .get("tableName") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - let value_column = data - .get("valueColumn") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let mut config = Self::new( - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels, - aggregated_labels, - rollup_labels, - original_yaml, - window_size, - slide_interval, - window_type, - spatial_filter, - metric, - num_aggregates_to_retain, - table_name, - value_column, - ); - if data.get("windowLayout").is_some() && data.get("window_layout").is_some() { - return Err("multiple window layout fields are not allowed".into()); - } - if let Some(layout) = data - .get("windowLayout") - .or_else(|| data.get("window_layout")) - { - config.window_layout = serde_json::from_value(layout.clone())?; - config - .window_layout - .validate(config.window_size, config.slide_interval)?; - } - config.population_key_encoding = data - .get("population_key_encoding") - .map(|value| serde_json::from_value(value.clone())) - .transpose()? - .unwrap_or_default(); - config.derived_input = data - .get("derived_input") - .filter(|v| !v.is_null()) - .map(|v| serde_json::from_value(v.clone())) - .transpose()?; - config.partitioning = data - .get("partitioning") - .filter(|value| !value.is_null()) - .map(|value| serde_json::from_value(value.clone())) - .transpose()?; - if let Some(projection) = data - .get("valueProjection") - .or_else(|| data.get("value_projection")) - .filter(|value| !value.is_null()) - { - config.value_projection = Some(serde_json::from_value(projection.clone())?); - } - config.pane_origin_ms = pane_origin_ms; - config.table_timestamp_column = data - .get("tableTimestampColumn") - .or_else(|| data.get("table_timestamp_column")) - .and_then(Value::as_str) - .map(str::to_owned); - config.table_population = data - .get("tablePopulation") - .or_else(|| data.get("table_population")) - .filter(|value| !value.is_null()) - .cloned() - .map(serde_json::from_value) - .transpose()?; - config.population_filter_canonical()?; - Ok(config) - } - - pub fn deserialize_from_bytes( - bytes: &[u8], - ) -> Result> { - let data_str = std::str::from_utf8(bytes)?.trim(); - let data: Value = serde_json::from_str(data_str)?; - Self::deserialize_from_json(&data) - } - pub fn from_yaml_data( aggregation_data: &serde_yaml::Value, num_aggregates_to_retain: Option, @@ -714,60 +539,6 @@ impl PrecomputeMaterialization { } } -impl SerializableToSink for PrecomputeMaterialization { - fn serialize_to_json(&self) -> Value { - // PR 5: `aggregationId` is no longer emitted — readers derive it - // from content via `PolicyFingerprint::from_config(...).as_u64()`. - let mut json = serde_json::json!({ - "aggregationType": self.aggregation_type, - "aggregationSubType": self.aggregation_sub_type, - "parameters": self.parameters, - "partitioning": self.partitioning, - "originalYaml": self.original_yaml, - "windowSize": self.window_size, - "slideInterval": self.slide_interval, - "windowLayout": self.window_layout, - "windowType": self.window_type.to_string(), - "spatialFilter": self.spatial_filter, - "metric": self.metric, - }); - - if !self.population_key_encoding.is_legacy() { - json["population_key_encoding"] = serde_json::json!(self.population_key_encoding); - } - if let Some(input) = &self.derived_input { - json["derived_input"] = serde_json::json!(input); - } - // Only include numAggregatesToRetain if it's Some - if let Some(num_aggregates) = self.num_aggregates_to_retain { - json["numAggregatesToRetain"] = serde_json::json!(num_aggregates); - } - if let Some(pane_origin_ms) = self.pane_origin_ms { - json["paneOriginMs"] = serde_json::json!(pane_origin_ms); - } - - // SQL-specific fields (only include if present) - if let Some(ref table_name) = self.table_name { - json["tableName"] = serde_json::json!(table_name); - } - if let Some(ref projection) = self.value_projection { - json["valueProjection"] = serde_json::json!(projection); - } - if let Some(ref population) = self.table_population { - json["tablePopulation"] = serde_json::json!(population); - } - if let Some(ref column) = self.table_timestamp_column { - json["tableTimestampColumn"] = serde_json::json!(column); - } - - json - } - - fn serialize_to_bytes(&self) -> Vec { - self.original_yaml.as_bytes().to_vec() - } -} - #[cfg(test)] mod window_layout_tests { use super::WindowMaterializationLayout; @@ -818,12 +589,18 @@ mod tests { /// SAME config as a fixture without it. #[test] fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = - AggregationConfig::from_yaml_data(&sample_yaml(true), None, QueryLanguage::PromQl) - .expect("parse ok"); - let without = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse ok"); + let with = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(true), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); + let without = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); assert_eq!( with.policy_fingerprint(), without.policy_fingerprint(), @@ -834,10 +611,18 @@ mod tests { /// Round-tripping the same content yields the same fingerprint. #[test] fn fingerprint_is_deterministic_per_content() { - let a = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse a"); - let b = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse b"); + let a = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse a"); + let b = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse b"); assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); assert_ne!( a.policy_fingerprint().as_u64(), @@ -862,44 +647,47 @@ mod tests { ] { yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); let config = - AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) + .unwrap(); assert_eq!(config.window_layout, layout); - let mut wire = config.serialize_to_json(); - wire["groupingLabels"] = serde_json::to_value(&config.grouping_labels).unwrap(); - wire["aggregatedLabels"] = - serde_json::to_value(&config.aggregated_labels.labels).unwrap(); - wire["rollupLabels"] = serde_json::to_value(&config.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&wire).unwrap(); + let mut wire = serde_json::to_value(&config).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(decoded.window_layout, layout); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); - wire["window_layout"] = wire["windowLayout"].clone(); - assert!(AggregationConfig::deserialize_from_json(&wire).is_err()); + wire["windowLayout"] = wire["window_layout"].clone(); + assert!(serde_json::from_value::(wire).is_err()); } yaml.as_mapping_mut() .unwrap() .remove(serde_yaml::Value::from("windowLayout")); - let legacy = AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + let legacy = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); assert_eq!( legacy.window_layout, WindowMaterializationLayout::Pane { pane_secs: 10 } ); yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!(AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err()); + assert!( + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() + ); } #[test] fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let mut epoch = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let unknown = epoch.policy_fingerprint(); epoch.pane_origin_ms = Some(7_000); let planned = epoch.policy_fingerprint(); assert_ne!(unknown, planned); - let wire = epoch.serialize_to_json(); - assert_eq!(wire["paneOriginMs"], serde_json::json!(7_000)); + let wire = serde_json::to_value(&epoch).unwrap(); + assert_eq!(wire["pane_origin_ms"], serde_json::json!(7_000)); let mut derived = serde_json::to_value(&epoch).unwrap(); let origin = derived .as_object_mut() @@ -910,48 +698,52 @@ mod tests { .as_object_mut() .unwrap() .insert("paneOriginMs".into(), origin); - let decoded: AggregationConfig = serde_json::from_value(derived.clone()).unwrap(); + assert!(serde_json::from_value::(derived).is_err()); + let decoded: PrecomputeMaterialization = + serde_json::from_value(serde_json::to_value(&epoch).unwrap()).unwrap(); assert_eq!(decoded.pane_origin_ms, Some(7_000)); - - let mut legacy = derived; - legacy.as_object_mut().unwrap().remove("paneOriginMs"); - assert_eq!( - serde_json::from_value::(legacy) - .expect("decode legacy wire") - .pane_origin_ms, - None - ); } /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. #[test] fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); } - /// PR 5: `serialize_to_json` no longer emits `aggregationId`. + /// Installed materializations reject the removed externally assigned identity. #[test] - fn serialize_to_json_omits_aggregation_id() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); - let json = cfg.serialize_to_json(); + fn canonical_wire_rejects_aggregation_id() { + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); + let mut json = serde_json::to_value(&cfg).unwrap(); assert!( json.get("aggregationId").is_none(), - "PR 5: aggregationId must not appear on the wire — readers derive it from content" + "aggregationId must not appear in the canonical wire format" ); + json["aggregationId"] = serde_json::json!(42); + assert!(serde_json::from_value::(json).is_err()); } #[test] - fn typed_projection_roundtrips_and_legacy_column_keeps_identity() { + fn typed_projection_roundtrips_and_untyped_column_is_rejected() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .unwrap(); + let mut config = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .unwrap(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -960,43 +752,33 @@ mod tests { let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); - let decoded: AggregationConfig = serde_json::from_value(legacy).unwrap(); - assert_eq!(decoded.policy_fingerprint(), column_identity); + assert!(serde_json::from_value::(legacy).is_err()); + let mut untyped = serde_json::to_value(&config).unwrap(); + untyped["value_projection"] = serde_json::json!("value"); + assert!(serde_json::from_value::(untyped).is_err()); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), }); - let mut wire = config.serialize_to_json(); - // The legacy JSON and YAML readers receive their labels from the - // enclosing streaming config, in their respective wire shapes. - wire["groupingLabels"] = config.grouping_labels.serialize_to_json(); - wire["aggregatedLabels"] = config.aggregated_labels.serialize_to_json(); - wire["rollupLabels"] = config.rollup_labels.serialize_to_json(); - wire["labels"] = serde_json::json!({ - "grouping": config.grouping_labels.serialize_to_json(), - "aggregated": config.aggregated_labels.serialize_to_json(), - "rollup": config.rollup_labels.serialize_to_json(), - }); - assert!(wire.get("valueColumn").is_none()); - let json = AggregationConfig::deserialize_from_json(&wire).unwrap(); - let yaml = AggregationConfig::from_yaml_data( - &serde_yaml::to_value(&wire).unwrap(), - None, - QueryLanguage::ClickHouseSql, - ) - .unwrap(); + let wire = serde_json::to_value(&config).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!( - json.effective_value_projection(), + decoded.effective_value_projection(), config.effective_value_projection() ); + let mut yaml = sample_yaml(false); + yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); + yaml["valueProjection"] = + serde_yaml::to_value(serde_json::to_value(&config.value_projection).unwrap()).unwrap(); + let decoded = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) + .unwrap(); assert_eq!( - yaml.effective_value_projection(), + decoded.effective_value_projection(), config.effective_value_projection() ); - let mut conflicting = wire; - conflicting["valueColumn"] = serde_json::json!("other_column"); - assert!(AggregationConfig::deserialize_from_json(&conflicting).is_err()); - assert!(AggregationConfig::from_yaml_data( - &serde_yaml::to_value(conflicting).unwrap(), + yaml["valueColumn"] = serde_yaml::to_value("other_column").unwrap(); + assert!(PrecomputeMaterialization::from_yaml_data( + &yaml, None, QueryLanguage::ClickHouseSql ) diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ccdcbec0d..647f7604f 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -14,15 +14,13 @@ use std::str::FromStr; pub enum AggregationType { // ---------- single-population (non-keyed) ---------- Sum, + Count, Increase, + Rate, Min, Max, DatasketchesKLL, // ---------- multi-population (keyed) ---------- - MultipleSum, - MultipleIncrease, - MultipleMin, - MultipleMax, HydraKLL, CountMinSketch, CountMinSketchWithHeap, @@ -38,17 +36,31 @@ pub enum AggregationType { } impl AggregationType { + /// Adapt a storage/processor tag to Planner's exact family. Keyed storage + /// changes the payload layout, not the semantic family. + pub fn planner_exact_family(self) -> Option { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + _ => return None, + }; + Some(SummaryFamilyType::ExactAggregate(kind, params)) + } + pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", + AggregationType::Count => "Count", AggregationType::Increase => "Increase", + AggregationType::Rate => "Rate", AggregationType::Min => "Min", AggregationType::Max => "Max", AggregationType::DatasketchesKLL => "DatasketchesKLL", - AggregationType::MultipleSum => "MultipleSum", - AggregationType::MultipleIncrease => "MultipleIncrease", - AggregationType::MultipleMin => "MultipleMin", - AggregationType::MultipleMax => "MultipleMax", AggregationType::HydraKLL => "HydraKLL", AggregationType::CountMinSketch => "CountMinSketch", AggregationType::CountMinSketchWithHeap => "CountMinSketchWithHeap", @@ -67,10 +79,6 @@ impl AggregationType { matches!( self, AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax | AggregationType::CountMinSketch | AggregationType::CountMinSketchWithHeap | AggregationType::CountSketch @@ -93,14 +101,12 @@ impl FromStr for AggregationType { match s { // Canonical names "Sum" => Ok(AggregationType::Sum), + "Count" => Ok(AggregationType::Count), "Increase" => Ok(AggregationType::Increase), + "Rate" => Ok(AggregationType::Rate), "Min" => Ok(AggregationType::Min), "Max" => Ok(AggregationType::Max), "DatasketchesKLL" => Ok(AggregationType::DatasketchesKLL), - "MultipleSum" => Ok(AggregationType::MultipleSum), - "MultipleIncrease" => Ok(AggregationType::MultipleIncrease), - "MultipleMin" => Ok(AggregationType::MultipleMin), - "MultipleMax" => Ok(AggregationType::MultipleMax), "HydraKLL" => Ok(AggregationType::HydraKLL), "CountMinSketch" => Ok(AggregationType::CountMinSketch), "CountMinSketchWithHeap" => Ok(AggregationType::CountMinSketchWithHeap), @@ -121,12 +127,6 @@ impl FromStr for AggregationType { "DatasketchesKLLAccumulator" | "KLL" | "kll" | "datasketches_kll" => { Ok(AggregationType::DatasketchesKLL) } - "MultipleSumAccumulator" | "multiple_sum" => Ok(AggregationType::MultipleSum), - "MultipleIncreaseAccumulator" | "multiple_increase" => { - Ok(AggregationType::MultipleIncrease) - } - "MultipleMinAccumulator" | "multiple_min" => Ok(AggregationType::MultipleMin), - "MultipleMaxAccumulator" | "multiple_max" => Ok(AggregationType::MultipleMax), "HydraKllSketchAccumulator" | "hydra_kll" => Ok(AggregationType::HydraKLL), "CountMinSketchAccumulator" | "CMS" | "cms" | "count_min_sketch" => { Ok(AggregationType::CountMinSketch) @@ -149,7 +149,7 @@ impl FromStr for AggregationType { | "MultipleMinMaxAccumulator" | "multiple_min_max" => Err(format!( "Retired aggregation type: '{s}' -- min and max are separate types now, \ - use 'Min'/'Max' (or 'MultipleMin'/'MultipleMax')" + use 'Min'/'Max'" )), _ => Err(format!("Unknown aggregation type: '{s}'")), } @@ -168,3 +168,48 @@ impl<'de> Deserialize<'de> for AggregationType { s.parse().map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + /// Removed layout tags cannot be installed as semantic families. + #[test] + fn rejects_keyed_family_aliases() { + for name in [ + "MultipleSum", + "MultipleIncrease", + "MultipleMin", + "MultipleMax", + ] { + assert!(name.parse::().is_err(), "{name}"); + } + } + + #[test] + fn storage_layout_tags_do_not_create_planner_families() { + for (storage, expected) in [ + (AggregationType::Sum, ExactKind::Sum), + (AggregationType::Count, ExactKind::Count), + (AggregationType::Increase, ExactKind::Increase), + (AggregationType::Rate, ExactKind::Rate), + ] { + let family = storage.planner_exact_family().unwrap(); + assert!( + matches!(family, SummaryFamilyType::ExactAggregate(kind, _) if kind == expected) + ); + } + assert_eq!( + AggregationType::Rate.planner_exact_family(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Rate, + ExactParams::Rate + )) + ); + assert_ne!( + AggregationType::Rate.planner_exact_family(), + AggregationType::Increase.planner_exact_family() + ); + } +} diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index 018635c1a..b3556e69d 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -6,12 +6,12 @@ use planner_types::post_asap::PostAsapNodeId; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::{executable_plan::OwnedPostAsapDag, sds::SummaryDefinitionId}; +use crate::{executable_plan::OwnedPostAsapDag, sds::StoredOutputId}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct DerivedInputIdentity { - pub inputs: BTreeSet, + pub inputs: BTreeSet, pub program_sha256: String, } @@ -35,9 +35,14 @@ impl DerivedInputIdentity { pub fn from_dag( document: &OwnedPostAsapDag, root: PostAsapNodeId, - frontiers: &BTreeMap, + frontiers: &BTreeMap, ) -> Result { - if document.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + if ![ + crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, + crate::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION, + ] + .contains(&document.schema_version) + { return Err("unsupported derived program document version".into()); } let decoded = document.decode()?; @@ -106,7 +111,7 @@ impl DerivedInputIdentity { } edges.sort(); let bytes = serde_json::to_vec(&serde_json::json!({ - "version": 1, "payload": node.payload, + "version": 2, "payload": node.payload, "state": node.output_state, "schema": node.output_schema, "guarantee": node.guarantee, "inputs": edges, })) @@ -158,7 +163,7 @@ mod tests { #[test] fn derived_source_is_distinct_and_generation_independent() { let raw = config(); - let raw_id = SummaryDefinitionId::from(raw.policy_fingerprint()); + let raw_id = StoredOutputId::from(raw.policy_fingerprint()); let mut derived = raw.clone(); derived.derived_input = Some(DerivedInputIdentity { inputs: BTreeSet::from([raw_id]), @@ -168,7 +173,7 @@ mod tests { let a = SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); - assert_eq!(a.materializations, b.materializations); + assert_eq!(a.outputs, b.outputs); assert_eq!(a.data_descriptors, b.data_descriptors); let mut renamed = derived.clone(); renamed.metric = "output_alias".into(); @@ -182,7 +187,7 @@ mod tests { fn raw_utf8_metric_cannot_impersonate_derived_policy_domain() { let mut derived = config(); derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "d".repeat(64), }); let mut raw = derived.clone(); @@ -198,7 +203,7 @@ mod tests { fn catalog_rejects_missing_derived_dependencies_and_raw_source_conflicts() { let mut derived = config(); derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "b".repeat(64), }); assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); @@ -210,7 +215,7 @@ mod tests { use planner_types::post_asap::{ EdgeRole, ExecutionDataState, GroupingEdgeCompatibility, WindowEdgeCompatibility, }; - let state = ExecutionDataState::MAINTENANCE_SUMMARY; + let state = ExecutionDataState::INGESTION_SUMMARY; OwnedPostAsapDag { schema_version: crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, query_id: "query-a".into(), @@ -239,7 +244,7 @@ mod tests { #[test] fn semantic_signature_ignores_node_and_query_numbering_but_not_inputs() { - let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let source = StoredOutputId::from(config().policy_fingerprint()); let a = program(1, 2); let first = DerivedInputIdentity::from_dag( &a, @@ -302,7 +307,7 @@ mod tests { }; let config = config(); let input = DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), program_sha256: "f".repeat(64), }; let data = DataDescriptor::new_typed( @@ -319,7 +324,6 @@ mod tests { config.policy_fingerprint(), SummaryDescriptor::from_config(&config).unwrap(), data, - config.window_layout )] ) .is_err()); @@ -329,7 +333,7 @@ mod tests { use crate::precompute_plan::{PlanEnvelope, PrecomputePlan}; let mut config = config(); config.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), program_sha256: "c".repeat(64), }); let envelope = PlanEnvelope { @@ -365,7 +369,7 @@ mod tests { let mut edge = dag.edges[0].clone(); edge.producer = PostAsapNodeId(3); dag.edges.push(edge); - let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let source = StoredOutputId::from(config().policy_fingerprint()); let frontiers = BTreeMap::from([(PostAsapNodeId(1), source)]); let first = DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap(); assert_eq!(first.inputs, BTreeSet::from([source])); diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index 5e7cb103b..df23f6f1b 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -1,6 +1,6 @@ //! Versioned runtime evidence about the inputs of one installed summary. //! Shape is generic so the transport contract does not depend on a planner. -use crate::sds::{CatalogGeneration, SummaryDefinitionId, SummaryInstanceId}; +use crate::sds::{CatalogGeneration, StoredOutputId, SummaryInstanceId}; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -22,7 +22,7 @@ pub struct ErpPopulationObservation { pub struct ErpPopulationObservations { pub schema_version: u32, pub catalog_generation: CatalogGeneration, - pub summary_definition_id: SummaryDefinitionId, + pub stored_output_id: StoredOutputId, pub observed_at_unix_ms: u64, pub window_start_ms: i64, pub window_end_ms: i64, @@ -44,13 +44,13 @@ impl ErpPopulationObservations { pub fn validate_identity_and_freshness( &self, expected_generation: &CatalogGeneration, - expected_definition: SummaryDefinitionId, + expected_definition: StoredOutputId, now_ms: u64, freshness: ErpObservationFreshness, ) -> Result<(), &'static str> { if self.schema_version != 1 || &self.catalog_generation != expected_generation - || self.summary_definition_id != expected_definition + || self.stored_output_id != expected_definition { return Err("ERP observation belongs to a different catalog or summary"); } @@ -189,7 +189,7 @@ impl ErpPopulationObservations { Some(ErpPopulationObservations { schema_version: self.schema_version, catalog_generation: self.catalog_generation, - summary_definition_id: self.summary_definition_id, + stored_output_id: self.stored_output_id, observed_at_unix_ms: self.observed_at_unix_ms, window_start_ms: self.window_start_ms, window_end_ms: self.window_end_ms, diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index ed40942aa..03e2193f9 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -8,7 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::sds::SummaryDefinitionId; +use crate::sds::StoredOutputId; use planner_types::post_asap::{ EdgeRole, ExecutableDag, ExecutableDagEdge, ExecutableDagNode, ExecutionDataState, ExecutionTiming, GroupingEdgeCompatibility, PostAsapNodeId, WindowEdgeCompatibility, @@ -20,7 +20,8 @@ use serde::{Deserialize, Serialize}; #[serde(transparent)] pub struct QueryNodeId(pub u64); -pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 2; +pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 3; +pub const MAINTENANCE_DAG_SCHEMA_VERSION: u32 = 4; /// Versioned, language-neutral Planner DAG persisted with an installed plan. /// Plan lifecycle belongs to the enclosing `PrecomputePlan`; this document @@ -183,12 +184,55 @@ pub struct InstalledPostAsapDag { impl InstalledPostAsapDag { pub fn validate(&self) -> Result<(), String> { - if self.document.schema_version != OWNED_POST_ASAP_DAG_SCHEMA_VERSION - || self.document.query_id.trim().is_empty() - { + if self.document.query_id.trim().is_empty() { return Err("invalid post-ASAP DAG document identity/version".into()); } - self.binding.validate(&self.document.decode()?) + if self.document.schema_version != MAINTENANCE_DAG_SCHEMA_VERSION { + return Err("unsupported maintenance DAG document version".into()); + } + self.binding.validate_maintenance(&self.document.decode()?) + } + + /// Project the selected semantic DAG onto the maintenance ancestors of its + /// stored outputs. + pub fn maintenance_projection(mut self) -> Result { + if self.document.schema_version != OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + return Err("selected DAG has an unsupported document version".into()); + } + self.binding.validate(&self.document.decode()?)?; + if self.binding.precompute_sinks.is_empty() { + return Err("cannot project a DAG without maintenance sinks".into()); + } + let mut included = self + .binding + .precompute_sinks + .iter() + .copied() + .collect::>(); + let mut frontier = self.binding.precompute_sinks.clone(); + while let Some(consumer) = frontier.pop() { + for edge in self + .document + .edges + .iter() + .filter(|edge| edge.consumer == consumer) + { + if included.insert(edge.producer) { + frontier.push(edge.producer); + } + } + } + self.document + .nodes + .retain(|node| included.contains(&node.id)); + self.document + .edges + .retain(|edge| included.contains(&edge.producer) && included.contains(&edge.consumer)); + self.binding.nodes.retain(|id, _| included.contains(id)); + self.document.root = *self.binding.precompute_sinks.first().unwrap(); + self.document.schema_version = MAINTENANCE_DAG_SCHEMA_VERSION; + self.validate()?; + Ok(self) } } @@ -215,7 +259,7 @@ pub enum BackendNodeBinding { QueryInput, MaintenanceInput, Materialization { - summary_definition: SummaryDefinitionId, + stored_output: StoredOutputId, }, } @@ -224,6 +268,55 @@ impl BackendExecutableBinding { self.nodes.get(&id) } + pub fn validate_maintenance(&self, dag: &ExecutableDag) -> Result<(), String> { + let ids = dag + .nodes + .iter() + .map(|node| node.id) + .collect::>(); + if ids.is_empty() || self.nodes.keys().copied().collect::>() != ids { + return Err("maintenance binding does not cover its projected DAG".into()); + } + if self.precompute_sinks.is_empty() + || self.precompute_sinks.iter().any(|id| !ids.contains(id)) + { + return Err("maintenance projection has missing sinks".into()); + } + if self.precompute_sinks.iter().any(|id| { + !matches!( + self.node(*id), + Some(BackendNodeBinding::Materialization { .. }) + ) + }) { + return Err("maintenance sink lacks a stored-output binding".into()); + } + for node in &dag.nodes { + match (node.output_state.timing, self.node(node.id)) { + ( + ExecutionTiming::IngestionTime, + Some( + BackendNodeBinding::MaintenanceInput + | BackendNodeBinding::Materialization { .. }, + ), + ) => {} + _ => { + return Err(format!( + "query-owned node {} in maintenance projection", + node.id.0 + )) + } + } + } + if dag + .edges + .iter() + .any(|edge| !ids.contains(&edge.producer) || !ids.contains(&edge.consumer)) + { + return Err("maintenance projection has dangling edges".into()); + } + Ok(()) + } + pub fn validate(&self, dag: &ExecutableDag) -> Result<(), String> { let semantic = dag .nodes @@ -238,11 +331,10 @@ impl BackendExecutableBinding { } for node in &dag.nodes { match (node.output_state.timing, self.node(node.id).unwrap()) { - (ExecutionTiming::ReadTime, BackendNodeBinding::Query { .. }) - | (ExecutionTiming::ReadTime, BackendNodeBinding::QueryInput) - | (ExecutionTiming::MaintenanceTime, BackendNodeBinding::MaintenanceInput) - | (ExecutionTiming::MaintenanceTime, BackendNodeBinding::Materialization { .. }) => { - } + (ExecutionTiming::QueryTime, BackendNodeBinding::Query { .. }) + | (ExecutionTiming::QueryTime, BackendNodeBinding::QueryInput) + | (ExecutionTiming::IngestionTime, BackendNodeBinding::MaintenanceInput) + | (ExecutionTiming::IngestionTime, BackendNodeBinding::Materialization { .. }) => {} _ => { return Err(format!( "backend placement disagrees with node {} mode", @@ -277,10 +369,33 @@ mod tests { fn send_sync() {} send_sync::(); let wire = serde_json::json!({ - "schema_version": 2, "query_id": "q", "nodes": [], "edges": [], "root": 0 + "schema_version": 1, "query_id": "q", "nodes": [], "edges": [], "root": 0 }); let document: OwnedPostAsapDag = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(serde_json::to_value(document).unwrap(), wire); assert_eq!(serde_json::to_value(QueryNodeId(9)).unwrap(), 9); } + + #[test] + fn installed_maintenance_dag_rejects_complete_dag_version() { + let installed = InstalledPostAsapDag { + document: OwnedPostAsapDag { + schema_version: OWNED_POST_ASAP_DAG_SCHEMA_VERSION, + query_id: "q".into(), + nodes: Vec::new(), + edges: Vec::new(), + root: PostAsapNodeId(0), + }, + binding: BackendExecutableBinding { + nodes: BTreeMap::new(), + query_sink: PostAsapNodeId(0), + query_plan_sink: QueryNodeId(0), + precompute_sinks: Vec::new(), + }, + }; + assert!(installed + .validate() + .unwrap_err() + .contains("unsupported maintenance DAG")); + } } diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index deb7fe3f1..5cd902b7d 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -2,7 +2,7 @@ //! //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` -//! (`AggregationConfig::grouping_labels`, `PolicyFingerprint`, +//! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, //! `PolicyRegistry`, `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 0cccd182e..1d4af2fef 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -16,8 +16,10 @@ pub mod producer_plan; pub mod query_requirements; pub mod routing_index; pub mod sds; +pub mod semantic_fragment; pub mod storage_backend; pub mod summary_catalog; +pub mod summary_semantics; pub mod table_population; pub mod traits; pub mod utils; diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index 535ec3ef0..a22352be6 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -44,7 +44,7 @@ impl MonitorFunctional { /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both -/// sides can reach — same reasoning as `AggregationConfig`/`PolicyFingerprint`. +/// sides can reach — same reasoning as `PrecomputeMaterialization`/`PolicyFingerprint`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct MonitorSpec { pub agg_id: u64, diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs index 3631b6cef..d678a23d5 100644 --- a/crates/asap_types/src/plan_publication.rs +++ b/crates/asap_types/src/plan_publication.rs @@ -33,6 +33,109 @@ pub struct PhysicalPlanInstallRequest { pub adaptation_evidence: Vec, } +/// A projected writer must refer to the query entry installed in the same +/// generation. +pub fn validate_maintenance_query_bindings( + precompute: &PrecomputePlan, + query: &QueryPlan, +) -> Result<(), String> { + for (query_id, installed) in &precompute.executable_dags { + installed.validate()?; + let mut entries = query + .entries + .values() + .filter(|entry| &entry.query_id == query_id); + let entry = entries + .next() + .ok_or("maintenance projection has no query entry")?; + if entries.next().is_some() { + return Err("maintenance projection has ambiguous query entries".into()); + } + if entry.root != installed.binding.query_plan_sink { + return Err("maintenance projection and query entry have different roots".into()); + } + let selected = query + .selected_dags + .get(query_id) + .ok_or("maintenance projection has no selected semantic provenance")?; + if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION + || selected.query_id != *query_id + { + return Err("selected semantic provenance has invalid identity/version".into()); + } + selected.decode()?; + if installed + .document + .nodes + .iter() + .any(|node| !selected.nodes.contains(node)) + || installed + .document + .edges + .iter() + .any(|edge| !selected.edges.contains(edge)) + { + return Err("maintenance projection differs from its selected DAG".into()); + } + } + Ok(()) +} + +/// Every query read must target the installed writer's exact stored output and +/// physical window contract. The catalog describes semantics; these choices +/// belong to the executable plans. +pub fn validate_stored_output_references( + precompute: &PrecomputePlan, + query: &QueryPlan, +) -> Result<(), String> { + let writers = precompute + .schemas + .iter() + .map(|schema| (schema.materialization, schema)) + .collect::>(); + let configs = precompute + .materializations + .iter() + .map(|config| (config.policy_fingerprint().into(), config)) + .collect::>(); + for entry in query.entries.values() { + for binding in entry.materialization_bindings() { + let writer = writers.get(&binding.materialization).ok_or_else(|| { + format!( + "query binding for definition {} has no precompute stored-summary writer", + binding.materialization.as_u64() + ) + })?; + if binding.stored_output_reference != writer.stored_output_reference { + return Err( + "query read and precompute writer have different stored outputs".into(), + ); + } + let config = configs.get(&binding.materialization).ok_or_else(|| { + format!( + "query binding for definition {} has no precompute configuration", + binding.materialization.as_u64() + ) + })?; + let full_slide = matches!( + config.window_layout, + crate::WindowMaterializationLayout::FullWindow + ) + .then_some(config.slide_interval.saturating_mul(1_000)); + if binding.full_window_slide_ms != full_slide { + return Err("query full-window cadence differs from precompute definition".into()); + } + if config.stored_window_ms() != binding.window_ms { + return Err("query pane differs from precompute stored window".into()); + } + if config.pane_origin_ms != binding.pane_origin_ms { + return Err("query pane origin differs from precompute definition".into()); + } + } + } + Ok(()) +} + impl PhysicalPlanPublication { /// Validate every plan against the shared catalog snapshot. pub fn validate(&self) -> Result<(), String> { @@ -49,36 +152,8 @@ impl PhysicalPlanPublication { self.query_plan .validate_against_catalog(catalog) .map_err(|e| e.to_string())?; - let materializations = self - .precompute_plan - .materializations - .iter() - .map(|config| (config.policy_fingerprint(), config)) - .collect::>(); - for entry in self.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("query binding has no precompute materialization")?; - let full_slide = matches!( - config.window_layout, - crate::WindowMaterializationLayout::FullWindow - ) - .then_some(config.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_slide { - return Err( - "query full-window cadence differs from precompute definition".into(), - ); - } - if config.stored_window_ms() != binding.window_ms { - return Err("query pane differs from precompute stored window".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } + validate_maintenance_query_bindings(&self.precompute_plan, &self.query_plan)?; + validate_stored_output_references(&self.precompute_plan, &self.query_plan)?; let mut collectors = std::collections::BTreeSet::new(); for collector in &self.collector_plans { if collector.envelope != self.precompute_plan.envelope diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index ca7353aa1..42fa2e475 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -1,62 +1,20 @@ -//! Content-addressed policy identity. +//! Legacy routing wrapper for a deployed stored output. //! -//! `PolicyFingerprint` is the merged-sid-identity-chain replacement for -//! the controller-allocated `aggregation_id: u64`. Where `aggregation_id` -//! is a counter the control plane mints and ships in the streaming-config -//! YAML, `PolicyFingerprint` is derived deterministically from the -//! `AggregationConfig`'s content — so two control planes producing the -//! same policy independently produce the same fingerprint, and the data -//! plane can index without a separate id allocation. -//! -//! ## Identity contract -//! -//! `PolicyFingerprint = h(metric, agg_type, sub_type, parameters, -//! grouping_labels, aggregated_labels, rollup_labels, window_size, -//! slide_interval, window_type, pane_origin_ms, spatial_filter_normalized)` -//! -//! The hash includes **every** field of `AggregationConfig` that -//! determines what the policy does — sketch / exact-agg shape, -//! group-by + rollup layout, window cadence, spatial filter. Two -//! configs that compare equal on these dimensions produce the same -//! fingerprint; two that differ produce different fingerprints. -//! -//! Fields *excluded* from the fingerprint: -//! - `aggregation_id` itself (the thing we're replacing — it's a -//! downstream label, not part of identity). -//! - `original_yaml` (incidental serialization artifact). -//! - `num_aggregates_to_retain` (retention policy, not aggregation -//! semantics — two policies with the same shape but different -//! retention are *the same policy* for ingest/query routing -//! purposes; retention is a separate concern). -//! -//! SQL source table, value projection, timestamp projection, and typed -//! population are included explicitly; the output metric is not a substitute -//! for these source semantics. -//! -//! ## Hash function -//! -//! `xxh64` keyed at 0, matching the existing `compute_agg_config_id` -//! helper this replaces. 64-bit gives ~4B-policy birthday bound -//! (collision probability ~10⁻¹¹ at 100K live policies); ample for -//! foreseeable workloads. Bump to sha256 if the control plane ever -//! manages >10⁶ live policies and we want deterministic uniqueness. -//! -//! The fingerprint is **stable across hosts and versions**: the byte -//! layout this module produces is the contract. Don't reorder fields, -//! don't change separator bytes — any such change invalidates every -//! deployed fingerprint and forces a cold-start rebuild. +//! An explicit `PrecomputeMaterialization::stored_output_id` takes precedence. +//! Otherwise the compiler allocates a deterministic default from the existing +//! policy fields (including pane layout and cadence). This identifier is not +//! semantic identity: `SummaryDefinitionId` hashes the versioned semantic +//! definition, and several deployed outputs may share that definition. +//! Catalog installation checks that an output is never assigned conflicting +//! computation or format contracts. use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use xxhash_rust::xxh64::xxh64; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; -/// Stable, content-addressed handle for an `AggregationConfig`. -/// -/// Wrap a `u64` so callers can't accidentally swap a `PolicyFingerprint` -/// with an `aggregation_id` — they're both u64-shaped but they index -/// different things (content-addressed vs. controller-allocated). +/// Routing handle for one deployed stored output. See the module contract. #[derive( Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, )] @@ -75,14 +33,17 @@ impl PolicyFingerprint { } impl PolicyFingerprint { - /// Compute the fingerprint of an [`AggregationConfig`]. + /// Compute the fingerprint of an [`PrecomputeMaterialization`]. /// /// Hash inputs are concatenated with `\0` byte separators and /// canonicalized so that map/iteration order can't affect the /// outcome. Parameter values are rendered via `serde_json::to_string` /// for nested-shape determinism (matches the existing /// `parameters_canonical` form used in `AggKind::ExactAgg`). - pub fn from_config(cfg: &AggregationConfig) -> Self { + pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { + if let Some(output) = cfg.stored_output_id { + return output.fingerprint(); + } let mut buf: Vec = Vec::with_capacity(512); if !cfg.population_key_encoding.is_legacy() { @@ -265,8 +226,8 @@ mod tests { group_by: Vec<&str>, window_size: u64, spatial_filter: &str, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -298,7 +259,7 @@ mod tests { ); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); let mut canonical = legacy.clone(); @@ -306,22 +267,8 @@ mod tests { assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); - use crate::traits::SerializableToSink; - let mut sink = canonical.serialize_to_json(); - // Transport wrappers supply the three label projections separately. - sink["groupingLabels"] = serde_json::to_value(&canonical.grouping_labels).unwrap(); - sink["aggregatedLabels"] = - serde_json::to_value(&canonical.aggregated_labels.labels).unwrap(); - sink["rollupLabels"] = serde_json::to_value(&canonical.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&sink).unwrap(); - assert_eq!( - decoded.population_key_encoding, - canonical.population_key_encoding - ); - - assert!(serde_json::from_str::("\"canonical_labels_v2\"").is_err()); } #[test] @@ -462,7 +409,7 @@ mod tests { ); } - /// Pre-PR-5 the `aggregation_id` field on `AggregationConfig` was + /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was /// excluded from the fingerprint hash. PR 5 deletes the field /// entirely — identity *is* the fingerprint — so this is now /// vacuously true. Kept as a doc-comment anchor; no runtime test @@ -525,7 +472,7 @@ mod tests { fn spatial_filter_canonicalization_drives_fingerprint() { // Two filters that differ only in matcher ordering produce the // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `AggregationConfig::new` (via + // canonicalization step in `PrecomputeMaterialization::new` (via // `normalize_spatial_filter`) sorts matchers by key. let a = cfg( "http_lat", diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 4b4f9e95c..dde5e5cce 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -1,7 +1,7 @@ //! Content-addressed policy registry. //! -//! Derived view over a collection of `AggregationConfig`s that maps -//! [`PolicyFingerprint`] → [`AggregationConfig`]. This is the +//! Derived view over a collection of `PrecomputeMaterialization`s that maps +//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated //! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` //! carries (see `data_plane::storage_engines::types::streaming_config`'s @@ -20,7 +20,7 @@ //! //! ## Identity invariants //! -//! Two `AggregationConfig`s that produce the same `PolicyFingerprint` +//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* //! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write @@ -30,13 +30,13 @@ use std::collections::HashMap; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; /// Content-addressed lookup table for active aggregation policies. #[derive(Debug, Clone, Default)] pub struct PolicyRegistry { - policies: HashMap, + policies: HashMap, } impl PolicyRegistry { @@ -46,7 +46,7 @@ impl PolicyRegistry { /// them. pub fn from_configs(configs: I) -> Self where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); for cfg in configs { @@ -63,7 +63,7 @@ impl PolicyRegistry { /// surfacing. pub fn from_configs_with_collisions(configs: I) -> (Self, usize) where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); let mut collisions = 0usize; @@ -77,12 +77,12 @@ impl PolicyRegistry { } /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.policies.get(&fp) } /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.policies.iter() } @@ -110,11 +110,11 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(_id: u64, metric: &str) -> AggregationConfig { + fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is derived from // content. Kept as a parameter so existing call sites in the // tests below don't churn. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 746649fc1..f360ab4b9 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -101,11 +101,10 @@ pub struct PlanEnvelope { pub capability_snapshot_id: String, } -/// Backend-side materialization projection consumed by the streaming -/// precompute engine. This is deliberately config-driven: it contains no -/// PromQL string or ad-hoc scheduler job. The aggregation definitions are -/// emitted to `/api/v1/streaming-config`, where the runtime matches incoming -/// series, maintains windows, and writes content-addressed materializations. +/// DAG-format precompute installation. Planner node payloads and dependency +/// edges define execution; materializations attach storage/window placement. +/// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths +/// execute through the maintenance DAG scheduler at stored-state frontiers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -115,8 +114,7 @@ pub struct PrecomputePlan { pub schemas: Vec, pub producers: Vec, pub materializations: Vec, - /// Planner semantic DAGs and backend-owned placement for this generation. - /// Empty only for legacy/config-only construction paths. + /// Maintenance projections ending at stored outputs. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub executable_dags: BTreeMap, } @@ -140,12 +138,13 @@ pub enum TimestampUnit { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct IngestContract { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dataset_identity: Option, pub protocol: IngestProtocol, pub endpoint_path: String, pub timestamp_unit: TimestampUnit, pub require_plan_identity: bool, - #[serde(alias = "require_materialization_identity")] - pub require_summary_definition_identity: bool, + pub require_stored_output_identity: bool, pub require_registered_producer: bool, } @@ -155,6 +154,8 @@ pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, ExactAccumulatorV1, + /// Persisted backend state with explicit Planner family and population layout. + PlannerExactAccumulatorV1, ExactCounterAccumulatorV2, } @@ -219,15 +220,12 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StateSchemaContract { + pub stored_output_reference: crate::sds::StoredOutputReference, pub schema_id: String, pub schema_version: u32, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub family: StateFamilyContract, pub source: Source, - #[serde( - alias = "value_column", - deserialize_with = "crate::sds::deserialize_state_value_projection" - )] pub value_projection: crate::sds::ValueProjectionIdentity, pub group_by: crate::GroupingProjection, pub window: StateWindowContract, @@ -240,11 +238,7 @@ pub struct StateWindowContract { pub kind: crate::WindowKind, pub size_ms: u64, pub slide_ms: Option, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, } @@ -255,11 +249,10 @@ pub struct StateWindowContract { pub struct ProducerContract { pub producer_id: String, pub collector_id: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub schema_id: String, - /// Authoritative partitions for completion barriers. Empty legacy contracts - /// authorize state ingestion only, never completion claims. - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + /// Authoritative partitions for completion barriers. An explicitly empty + /// roster cannot authorize completion claims. pub partition_ids: BTreeSet, } @@ -333,6 +326,9 @@ impl PrecomputePlan { ); let value_projection = materialization.effective_value_projection().clone(); Ok(StateSchemaContract { + stored_output_reference: crate::sds::StoredOutputReference::for_output( + fingerprint.into(), + ), schema_id: state_schema_id(fingerprint), schema_version: 1, materialization: fingerprint.into(), @@ -368,25 +364,31 @@ impl PrecomputePlan { }) }) .collect(); + let dataset_identity = materializations + .iter() + .filter_map(|m| m.semantic_fragment.as_ref()?.dataset_identity.clone()) + .next(); let plan = Self { summary_catalog: None, envelope, ingest: if backend_local { IngestContract { + dataset_identity: dataset_identity.clone(), protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, } } else { IngestContract { + dataset_identity: dataset_identity.clone(), protocol: IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: TimestampUnit::UnixNanoseconds, require_plan_identity: true, - require_summary_definition_identity: true, + require_stored_output_identity: true, require_registered_producer: true, } }, @@ -434,7 +436,7 @@ impl PrecomputePlan { /// monotonic progress. The runtime must enforce those before accepting it. pub fn validate_watermark_scope( &self, - materialization: crate::sds::SummaryDefinitionId, + materialization: crate::sds::StoredOutputId, barrier: &crate::sds::SummaryWatermarkBarrier, ) -> Result<(), PrecomputePlanError> { self.validate()?; @@ -464,19 +466,36 @@ impl PrecomputePlan { } pub fn validate(&self) -> Result<(), PrecomputePlanError> { + if let Some(dataset) = &self.ingest.dataset_identity { + dataset + .validate() + .map_err(PrecomputePlanError::CatalogContract)?; + } + for materialization in &self.materializations { + if let Some(fragment) = &materialization.semantic_fragment { + fragment + .validate() + .map_err(PrecomputePlanError::CatalogContract)?; + if fragment.dataset_identity != self.ingest.dataset_identity { + return Err(PrecomputePlanError::CatalogContract( + "semantic dataset differs from installed input binding".into(), + )); + } + } + } let valid_ingest = match self.ingest.protocol { IngestProtocol::ModifiedOtlpMetricsV1 => { self.ingest.endpoint_path == "/v1/metrics" && self.ingest.timestamp_unit == TimestampUnit::UnixNanoseconds && self.ingest.require_plan_identity - && self.ingest.require_summary_definition_identity + && self.ingest.require_stored_output_identity && self.ingest.require_registered_producer } IngestProtocol::PrometheusRemoteWriteV1 => { self.ingest.endpoint_path == "/api/v1/write" && self.ingest.timestamp_unit == TimestampUnit::UnixMilliseconds && !self.ingest.require_plan_identity - && !self.ingest.require_summary_definition_identity + && !self.ingest.require_stored_output_identity && !self.ingest.require_registered_producer } }; @@ -559,8 +578,8 @@ impl PrecomputePlan { .map_err(PrecomputePlanError::CatalogContract)?; for sink in &installed.binding.precompute_sinks { if !matches!(installed.binding.node(*sink), - Some(crate::executable_plan::BackendNodeBinding::Materialization { summary_definition }) - if summary_definition.fingerprint() == config.policy_fingerprint()) + Some(crate::executable_plan::BackendNodeBinding::Materialization { stored_output }) + if stored_output.fingerprint() == config.policy_fingerprint()) { continue; } @@ -585,9 +604,9 @@ impl PrecomputePlan { .iter() .filter_map(|(node, binding)| match binding { crate::executable_plan::BackendNodeBinding::Materialization { - summary_definition, - } if derived.inputs.contains(summary_definition) => { - Some((*node, *summary_definition)) + stored_output, + } if derived.inputs.contains(stored_output) => { + Some((*node, *stored_output)) } _ => None, }) @@ -618,42 +637,39 @@ impl PrecomputePlan { .filter(|edge| edge.consumer == id) .collect(); use planner_types::post_asap::{ - ExecutableOperatorPayload as Payload, ExecutionTiming, ValueOperation, + ExecutableOperatorPayload as Payload, ValueOperation, }; if node.output_state - != planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS + != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS { return Err(invalid()); } match &node.payload { Payload::Value { operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::MaintenanceTime, } if children.len() == 1 && frontiers.contains_key(&children[0].producer) => {} - Payload::Binary { - operator, - timing: ExecutionTiming::MaintenanceTime, - } if children.len() == 2 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Left - }) - .count() - == 1 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Right - }) - .count() - == 1 - && operator.vector_match.is_none() - && matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) => + Payload::Binary { operator } + if children.len() == 2 + && children + .iter() + .filter(|edge| { + edge.role == planner_types::post_asap::EdgeRole::Left + }) + .count() + == 1 + && children + .iter() + .filter(|edge| { + edge.role == planner_types::post_asap::EdgeRole::Right + }) + .count() + == 1 + && operator.vector_match.is_none() + && matches!( + operator.kind, + planner_types::pre_asap::BinaryOpKind::Arithmetic(_) + ) => { pending.extend(children.iter().map(|edge| edge.producer)); } @@ -682,7 +698,7 @@ impl PrecomputePlan { .map_err(PrecomputePlanError::CatalogContract)?; for node in &dag.nodes { let Some(crate::executable_plan::BackendNodeBinding::Materialization { - summary_definition, + stored_output, }) = installed.binding.node(node.id) else { continue; @@ -690,7 +706,7 @@ impl PrecomputePlan { let Some(config) = self .materializations .iter() - .find(|config| config.policy_fingerprint() == summary_definition.fingerprint()) + .find(|config| config.policy_fingerprint() == stored_output.fingerprint()) else { return Err(PrecomputePlanError::CatalogContract( "DAG materialization has no runtime configuration".into(), @@ -787,11 +803,14 @@ impl PrecomputePlan { } } let mut schema_ids = BTreeSet::new(); + let mut stored_outputs = BTreeSet::new(); for schema in &self.schemas { if schema.schema_id.trim().is_empty() || !schema_ids.insert(schema.schema_id.as_str()) + || !stored_outputs.insert(schema.stored_output_reference.stored_output_id) || schema.schema_version == 0 || schema.encodings.is_empty() + || schema.stored_output_reference.stored_output_id != schema.materialization { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), @@ -904,8 +923,14 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate, _, - ) => vec![StateEncoding::ExactCounterAccumulatorV2], - SummaryFamilyType::ExactAggregate(..) => vec![StateEncoding::ExactAccumulatorV1], + ) => vec![ + StateEncoding::ExactCounterAccumulatorV2, + StateEncoding::PlannerExactAccumulatorV1, + ], + SummaryFamilyType::ExactAggregate(..) => vec![ + StateEncoding::ExactAccumulatorV1, + StateEncoding::PlannerExactAccumulatorV1, + ], SummaryFamilyType::Sketch(kind, _) if matches!( kind.algorithm(), @@ -967,9 +992,11 @@ mod source_window_cohort_tests { "producer_id":"p", "collector_id":"c", "materialization":materialization, "schema_id":plan.schemas[0].schema_id, }); - let producer: ProducerContract = serde_json::from_value(legacy.clone()).unwrap(); + assert!(serde_json::from_value::(legacy.clone()).is_err()); + let mut current = legacy; + current["partition_ids"] = serde_json::json!([]); + let producer: ProducerContract = serde_json::from_value(current).unwrap(); assert!(producer.partition_ids.is_empty()); - assert_eq!(serde_json::to_value(&producer).unwrap(), legacy); plan.producers.push(producer); let barrier = crate::sds::SummaryWatermarkBarrier { catalog_generation: generation, @@ -1008,7 +1035,7 @@ mod source_window_cohort_tests { .validate_watermark_scope(materialization, &wrong) .is_err()); } - let other_materialization = crate::sds::SummaryDefinitionId(crate::PolicyFingerprint( + let other_materialization = crate::sds::StoredOutputId::from(crate::PolicyFingerprint( materialization.as_u64().wrapping_add(1), )); assert!(restored @@ -1041,7 +1068,7 @@ mod source_window_cohort_tests { reduction: Reduction::by(vec![]), grouping: Default::default(), }, - output_state: ExecutionDataState::MAINTENANCE_SUMMARY, + output_state: ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: vec![], time_index: None, diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 201b9a5fa..bad95e7e5 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -1,6 +1,6 @@ //! Catalog consistency checks for the precompute execution plan. use super::*; -use crate::sds::{SummaryDefinitionId, SummaryDescriptor}; +use crate::sds::{StoredOutputId, SummaryDescriptor}; use crate::summary_catalog::SummaryCatalog; use planner_types::pre_asap::Source; use std::collections::BTreeSet; @@ -13,11 +13,16 @@ impl PrecomputePlan { /// only the immutable snapshot reference; descriptors are installed once. pub fn bind_catalog(&mut self, catalog: &SummaryCatalog) -> Result<(), PrecomputePlanError> { for config in &self.materializations { - let id = SummaryDefinitionId::from(config.policy_fingerprint()); - catalog.materializations.get(&id).ok_or_else(|| { + let id = StoredOutputId::from(config.policy_fingerprint()); + catalog.outputs.get(&id).ok_or_else(|| { invalid(format!("missing catalog materialization {}", id.as_u64())) })?; } + for schema in &mut self.schemas { + schema.stored_output_reference = catalog + .output_reference(schema.materialization) + .map_err(|e| invalid(e.to_string()))?; + } self.summary_catalog = Some( catalog .reference() @@ -39,6 +44,51 @@ impl PrecomputePlan { catalog: &SummaryCatalog, ) -> Result<(), PrecomputePlanError> { catalog.validate().map_err(|e| invalid(e.to_string()))?; + let expected = SummaryCatalog::from_materializations( + catalog.plan_id, + catalog.plan_version, + &self.materializations, + ) + .map_err(|e| invalid(e.to_string()))?; + if catalog.outputs != expected.outputs || catalog.definitions != expected.definitions { + return Err(invalid( + "stored output semantics differ from installed writer computation", + )); + } + for config in &self.materializations { + if config.derived_input.is_some() && config.semantic_fragment.is_none() { + return Err(invalid( + "derived stored output requires its complete Planner semantic closure", + )); + } + if let Some(expected) = &config.semantic_fragment { + let mut found = false; + for installed in self.executable_dags.values() { + let dag = installed.document.decode().map_err(invalid)?; + for (id, binding) in &installed.binding.nodes { + if matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } + if stored_output.fingerprint() == config.policy_fingerprint()) + { + found = true; + let actual = match &self.ingest.dataset_identity { + Some(dataset) => crate::semantic_fragment::SemanticFragment::from_stored_output_in_dataset(&dag, *id, dataset.clone()), + None => crate::semantic_fragment::SemanticFragment::from_stored_output(&dag, *id), + }.map_err(invalid)?; + if &actual != expected { + return Err(invalid( + "semantic definition differs from Planner-selected producer", + )); + } + } + } + } + if !found { + return Err(invalid( + "semantic definition has no Planner-selected producer", + )); + } + } + } let expected_reference = catalog .reference() .map_err(|error| invalid(error.to_string()))?; @@ -51,14 +101,14 @@ impl PrecomputePlan { let ids: BTreeSet<_> = self .materializations .iter() - .map(|m| SummaryDefinitionId::from(m.policy_fingerprint())) + .map(|m| StoredOutputId::from(m.policy_fingerprint())) .collect(); - if ids != catalog.materializations.keys().copied().collect() { + if ids != catalog.outputs.keys().copied().collect() { return Err(invalid("catalog/reference/materialization sets differ")); } for config in &self.materializations { - let id = SummaryDefinitionId::from(config.policy_fingerprint()); - let binding = &catalog.materializations[&id]; + let id = StoredOutputId::from(config.policy_fingerprint()); + let binding = &catalog.outputs[&id]; let expected = SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { @@ -66,9 +116,6 @@ impl PrecomputePlan { "summary operator/update contract differs from catalog", )); } - if binding.pane_origin_ms != config.pane_origin_ms { - return Err(invalid("pane origin differs from catalog definition")); - } let data = &catalog.data_descriptors[&binding.data_descriptor_id]; let expected_source = config.source_identity(); if config.table_name.is_some() @@ -141,7 +188,11 @@ impl PrecomputePlan { return Err(invalid("session lifecycle is not supported")) } }; - if schema.schema_id != state_schema_id(id.fingerprint()) + if schema.stored_output_reference + != catalog + .output_reference(id) + .map_err(|e| invalid(e.to_string()))? + || schema.schema_id != state_schema_id(id.fingerprint()) || schema.family != expected_family || schema.source != source || &schema.value_projection != projection diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 0d9e58457..846c4df83 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -8,23 +8,20 @@ use std::collections::BTreeSet; use thiserror::Error; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] pub struct CollectorMaterialization { pub query_id: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub metric: String, pub algorithm: String, pub parameters: Value, pub group_by: Vec, pub window_secs: u64, pub abstract_window_framework: SummaryWindowFramework, - #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + #[serde(rename = "window_implementation_id")] pub window_realization_id: String, pub slide_secs: u64, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, pub window_layout: crate::WindowMaterializationLayout, pub evidence_source: Option, @@ -41,8 +38,9 @@ pub struct CollectorLifecycle { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] pub struct CollectorPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. + /// Set when binding the plan; required for catalog-authorized installation. #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, pub collector_id: String, @@ -76,7 +74,7 @@ pub struct FrameIdentityContract { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct TransmissionRule { - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub producer_id: String, pub schema_id: String, pub mode: TransmissionMode, @@ -194,7 +192,7 @@ pub struct RuntimeRulePolicy { pub struct RuntimeAdaptationEvidence { pub plan_id: u64, pub plan_version: u64, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, pub producer_id: String, pub schema_id: String, pub producer_version: String, @@ -205,7 +203,7 @@ pub struct RuntimeAdaptationEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct TransmissionPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. + /// Set when binding the plan; required for catalog-authorized installation. #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, pub envelope: PlanEnvelope, @@ -230,7 +228,7 @@ pub struct SummaryFrameIdentity { pub plan_id: u64, pub plan_version: u64, pub backend_compat: String, - pub materialization: crate::sds::SummaryDefinitionId, + pub materialization: crate::sds::StoredOutputId, /// Canonical producer-side identity for one concrete retained-label group. pub series_identity: String, pub schema_id: String, @@ -275,7 +273,7 @@ pub enum TransmissionPlanError { fn validate_catalog_projection( reference: Option<&crate::sds::CatalogGeneration>, envelope: &PlanEnvelope, - materializations: impl IntoIterator, + materializations: impl IntoIterator, catalog: &crate::summary_catalog::SummaryCatalog, ) -> Result<(), TransmissionPlanError> { let expected = catalog @@ -290,7 +288,7 @@ fn validate_catalog_projection( )); } for id in materializations { - if !catalog.materializations.contains_key(&id) { + if !catalog.outputs.contains_key(&id) { return Err(TransmissionPlanError::Catalog(format!( "unknown materialization {}", id.as_u64() diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 7595ee934..f4eb6e76d 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; pub use crate::QueryLanguage; -use crate::{sds::SummaryDefinitionId, PolicyFingerprint}; +use crate::{sds::StoredOutputId, PolicyFingerprint}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -24,6 +24,10 @@ pub struct QueryPlan { pub plan_version: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub clickhouse_context: Option, + /// Selected semantic roots retained for provenance; serving executes + /// `entries` and never reconstructs a plan from these documents. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub selected_dags: BTreeMap, pub entries: BTreeMap, } @@ -52,6 +56,7 @@ impl QueryPlan { plan_id: 0, plan_version: 0, clickhouse_context: None, + selected_dags: BTreeMap::new(), entries: BTreeMap::new(), } } @@ -88,6 +93,25 @@ impl QueryPlan { self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) } + pub fn bind_catalog( + &mut self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + for entry in self.entries.values_mut() { + for node in entry.nodes.values_mut() { + if let QueryPlanNode::ReadMaterialization { binding } = node { + binding.stored_output_reference = catalog + .output_reference(binding.materialization) + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + } + } + } + self.validate_against_catalog(catalog) + } + /// Validate semantic bindings against the authoritative snapshot before use. pub fn validate_against_catalog( &self, @@ -101,43 +125,32 @@ impl QueryPlan { "QueryPlan and SummaryCatalog have different plan identity/version".into(), )); } - let available = catalog - .materializations - .keys() - .copied() - .map(Into::into) - .collect(); + let available = catalog.outputs.keys().copied().map(Into::into).collect(); self.validate(&available)?; for entry in self.entries.values() { for binding in entry.materialization_bindings() { let identity = catalog - .materializations + .outputs .get(&binding.materialization) .ok_or_else(|| { QueryPlanError::Invalid( "query binding references absent catalog materialization".into(), ) })?; - let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; - if binding.window_ms == 0 { + if binding.stored_output_reference.definition_id != identity.definition_id { return Err(QueryPlanError::Invalid( - "zero physical pane duration".into(), + "read definition differs from installed output".into(), )); } - if binding.full_window_slide_ms.is_some() - != matches!( - identity.window_layout, - crate::WindowMaterializationLayout::FullWindow - ) - || binding.full_window_slide_ms == Some(0) - { + let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; + if binding.window_ms == 0 { return Err(QueryPlanError::Invalid( - "query storage layout differs from catalog definition".into(), + "zero physical pane duration".into(), )); } - if binding.pane_origin_ms != identity.pane_origin_ms { + if binding.full_window_slide_ms == Some(0) { return Err(QueryPlanError::Invalid( - "query pane origin differs from catalog definition".into(), + "query full-window cadence must be nonzero".into(), )); } } @@ -154,7 +167,7 @@ impl QueryPlan { "counter readout must directly consume one catalog materialization".into(), )); }; - let identity = &catalog.materializations[&binding.materialization]; + let identity = &catalog.outputs[&binding.materialization]; let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; if !matches!( descriptor.fidelity, @@ -178,6 +191,34 @@ impl QueryPlan { "non-bootstrap QueryPlan has zero plan_version".into(), )); } + for (query_id, selected) in &self.selected_dags { + if query_id != &selected.query_id { + return Err(QueryPlanError::Invalid(format!( + "selected DAG map key `{query_id}` differs from document query ID `{}`", + selected.query_id + ))); + } + if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION + { + return Err(QueryPlanError::Invalid(format!( + "selected DAG `{query_id}` has unsupported schema version {}", + selected.schema_version + ))); + } + selected.decode().map_err(|error| { + QueryPlanError::Invalid(format!("selected DAG `{query_id}` is invalid: {error}")) + })?; + let matching_entries = self + .entries + .values() + .filter(|entry| entry.query_id == *query_id) + .count(); + if matching_entries != 1 { + return Err(QueryPlanError::Invalid(format!( + "selected DAG `{query_id}` must correspond to exactly one query entry; found {matching_entries}" + ))); + } + } if let Some(context) = &self.clickhouse_context { for (template, identities) in &context.window_templates { if !template.starts_with("moving-window-v1:") || identities.is_empty() { @@ -250,7 +291,6 @@ pub struct QueryPlanEntry { #[serde(default)] pub language: QueryLanguage, pub query_id: String, - #[serde(alias = "canonical_promql")] pub canonical_query: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub fixed_evaluation: Option, @@ -369,15 +409,7 @@ impl QueryPlanEntry { )); } } - if let QueryPlanNode::CandidateTopK { - k, completeness, .. - } = node - { - if *k == 0 { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires k > 0".into(), - )); - } + if let QueryPlanNode::MembershipFilter { completeness, .. } = node { if matches!( completeness, CandidateCompleteness::Certified { guarantee } @@ -387,7 +419,7 @@ impl QueryPlanEntry { || guarantee.failure_probability.evaluate().is_none() ) { return Err(QueryPlanError::Invalid( - "invalid CandidateTopK completeness certificate".into(), + "invalid MembershipFilter completeness certificate".into(), )); } } @@ -400,6 +432,11 @@ impl QueryPlanEntry { } } if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.stored_output_reference.stored_output_id != binding.materialization { + return Err(QueryPlanError::Invalid( + "read binding has invalid stored output or definition".into(), + )); + } if binding.readout_lookback_ms == Some(0) { return Err(QueryPlanError::Invalid( "zero semantic readout lookback".into(), @@ -436,24 +473,21 @@ pub enum FallbackPolicy { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MaterializationBinding { + pub stored_output_reference: crate::sds::StoredOutputReference, /// Complete-window storage advances independently of its stored extent. /// None denotes disjoint pane storage. #[serde(default, skip_serializing_if = "Option::is_none")] pub full_window_slide_ms: Option, - pub materialization: SummaryDefinitionId, + pub materialization: StoredOutputId, /// Query operator grouping applied while folding those SIDs. pub output_grouping: PhysicalGrouping, /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub item_labels: Vec, pub window_ms: u64, /// Unix millisecond timestamp on the materialized pane-boundary grid. /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, /// Semantic query lookback, independent of the physical pane duration. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -582,13 +616,11 @@ pub enum QueryPlanNode { SummaryMerge { inputs: Vec, }, - /// Use an approximate heap only as a membership sidecar, then rerank the - /// matching exact counter readouts. `inputs[0]` is candidate membership; - /// `inputs[1]` is the authoritative exact value vector. - CandidateTopK { + /// Semijoin value rows against membership identities, preserving their values + /// and order. Inputs are membership and authoritative values respectively. + /// Ranking, grouping and limiting are separate downstream operators. + MembershipFilter { inputs: [QueryNodeId; 2], - k: u64, - grouping: residual::Grouping, completeness: CandidateCompleteness, }, /// An exact subtree evaluated outside ASAP. Its results enter the query DAG @@ -616,7 +648,7 @@ impl QueryPlanNode { Self::SummaryMerge { inputs } | Self::Logical { inputs, .. } | Self::ExternalExact { inputs, .. } => inputs, - Self::CandidateTopK { inputs, .. } => inputs, + Self::MembershipFilter { inputs, .. } => inputs, } } } @@ -634,6 +666,22 @@ pub enum ExactReadout { Max, } +impl ExactReadout { + /// Planner family required by this installed DAG readout node. + pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + }; + SummaryFamilyType::ExactAggregate(kind, params) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryReadout { diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 4e0dd5153..40a90fe36 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `AggregationConfig`s, so it represents planned policy +//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -32,7 +32,7 @@ use std::collections::{BTreeSet, HashMap}; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; use crate::policy_registry::PolicyRegistry; @@ -62,7 +62,7 @@ impl RoutingIndex { /// Tier 1 — exact fingerprint lookup. Delegates to the underlying /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.registry.get(fp) } @@ -136,8 +136,8 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(metric: &str) -> AggregationConfig { - AggregationConfig::new( + fn cfg(metric: &str) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -172,7 +172,7 @@ mod tests { fn multiple_policies_for_the_same_metric_all_bucket_together() { // Same metric, distinct group-by shapes -> distinct fingerprints, // same bucket. - let a = AggregationConfig::new( + let a = PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -220,8 +220,9 @@ mod tests { #[test] fn len_and_is_empty_match_registry() { - let idx = - RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let idx = RoutingIndex::build(PolicyRegistry::from_configs( + Vec::::new(), + )); assert!(idx.is_empty()); assert_eq!(idx.len(), 0); @@ -234,7 +235,7 @@ mod tests { fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { let mut parameters = StdHashMap::new(); parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::DDSketch, String::new(), parameters, diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index c06216148..9433816d1 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1,5 +1,6 @@ -//! Shared SDS metadata contracts. Summary payload bytes remain storage-engine -//! owned; catalogs and inventories contain identities and state references only. +//! Shared contracts for summary definitions, stored-summary metadata, and plan +//! output bindings. Payload bytes remain storage-engine owned and logically +//! belong to the stored summary identified by this metadata. pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; use crate::{AggregationType, PrecomputeMaterialization}; @@ -29,27 +30,68 @@ macro_rules! descriptor_id { } }; } -/// Semantic materialization reference. Wire-compatible with PolicyFingerprint, -/// but distinct from descriptor IDs and concrete [`SummaryInstanceId`] identity. +/// Identity of one deployed producer output. Runtime routing uses this identity, +/// never the semantic definition hash. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(transparent)] -pub struct SummaryDefinitionId(pub crate::PolicyFingerprint); -impl SummaryDefinitionId { +pub struct StoredOutputId(pub u64); +impl StoredOutputId { pub fn fingerprint(self) -> crate::PolicyFingerprint { - self.0 + crate::PolicyFingerprint(self.0) } pub fn as_u64(self) -> u64 { - self.0 .0 + self.0 } } -impl From for SummaryDefinitionId { +impl From for StoredOutputId { fn from(value: crate::PolicyFingerprint) -> Self { - Self(value) + Self(value.0) + } +} +impl From for crate::PolicyFingerprint { + fn from(value: StoredOutputId) -> Self { + Self(value.0) + } +} + +descriptor_id!(SummaryDefinitionId); +impl SummaryDefinitionId { + pub(crate) fn from_semantics(bytes: &[u8]) -> Self { + use sha2::{Digest, Sha256}; + Self(format!("sds-v1:{:x}", Sha256::digest(bytes))) + } + pub fn validate(&self) -> Result<(), SdsError> { + let hash = self.0.strip_prefix("sds-v1:").unwrap_or(""); + if hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + Ok(()) + } else { + Err(SdsError("invalid semantic definition ID".into())) + } } } -impl From for crate::PolicyFingerprint { - fn from(value: SummaryDefinitionId) -> Self { - value.0 + +/// The installed writer/reader binding joins deployment identity and semantics. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoredOutputReference { + pub stored_output_id: StoredOutputId, + pub definition_id: SummaryDefinitionId, +} +impl StoredOutputReference { + /// Internal compilation placeholder. Catalog binding must replace its empty + /// semantic identity before installation; it cannot authorize a read/write. + pub fn for_output(stored_output_id: StoredOutputId) -> Self { + Self { + stored_output_id, + definition_id: SummaryDefinitionId(String::new()), + } + } + pub fn validate(&self) -> Result<(), SdsError> { + self.definition_id.validate() } } @@ -83,7 +125,6 @@ pub struct CatalogGeneration { pub schema_version: u32, pub plan_id: u64, pub plan_version: u64, - #[serde(alias = "snapshot_digest")] pub snapshot_sha256: String, } @@ -112,7 +153,7 @@ pub struct SummarySourcePartition { #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryInstanceCoordinates { - pub summary_definition_id: SummaryDefinitionId, + pub stored_output_id: StoredOutputId, pub time_range: HalfOpenTimeRange, pub group_values: BTreeMap, } @@ -123,7 +164,7 @@ impl SummaryInstanceCoordinates { serde_json::to_vec(&self.group_values).map_err(|error| SdsError(error.to_string()))?; SummaryInstanceId::new(format!( "summary-instance:v1:{}:{}:{}:{}", - self.summary_definition_id.as_u64(), + self.stored_output_id.as_u64(), self.time_range.start_ms, self.time_range.end_ms, xxhash_rust::xxh64::xxh64(&bytes, 0) @@ -275,7 +316,7 @@ pub enum InstanceLifecycle { #[serde(deny_unknown_fields)] pub struct SummaryInstance { pub instance_id: SummaryInstanceId, - #[serde(alias = "materialization_id")] + pub stored_output_id: StoredOutputId, pub summary_definition_id: SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, pub data_descriptor_id: DataDescriptorId, @@ -292,6 +333,7 @@ pub struct SummaryInstance { impl SummaryInstance { pub fn validate(&self) -> Result<(), SdsError> { + self.summary_definition_id.validate()?; if self.time_range.start_ms >= self.time_range.end_ms { return Err(SdsError( "summary instance time range must be non-empty".into(), @@ -312,6 +354,7 @@ impl SummaryInstance { if self.state_reference.store.is_empty() || self.state_reference.key.is_empty() || self.state_reference.state_schema_version == 0 + || self.state_reference.generation != self.catalog_generation.plan_version { return Err(SdsError( "summary instance has invalid state reference".into(), @@ -364,6 +407,39 @@ impl ObservedSummaryInventory { } Ok(()) } + + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), SdsError> { + self.validate()?; + let generation = catalog + .reference() + .map_err(|error| SdsError(error.to_string()))?; + for instance in self.instances.values() { + if instance.catalog_generation != generation { + return Err(SdsError( + "summary instance belongs to another plan generation".into(), + )); + } + let definition = catalog + .outputs + .get(&instance.stored_output_id) + .ok_or_else(|| SdsError("summary instance has no catalog definition".into()))?; + if instance.summary_definition_id != definition.definition_id + || instance.summary_descriptor_id != definition.summary_descriptor_id + || instance.data_descriptor_id != definition.data_descriptor_id + || instance.state_reference.state_schema_version + != catalog.summary_descriptors[&definition.summary_descriptor_id] + .state_schema_version + { + return Err(SdsError( + "summary instance differs from its catalog definition".into(), + )); + } + } + Ok(()) + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -383,6 +459,9 @@ pub enum SummaryOperator { /// Complete planner materialization configuration, including heap/Hydra /// dimensions and readout/update subtype. Never equal to a legacy projection. Configured { + /// Planner-selected semantic family; grouping and pane layout live in + /// the data descriptor and summary definition, respectively. + family: planner_types::post_asap::SummaryFamilyType, aggregation_type: AggregationType, aggregation_sub_type: String, parameters: BTreeMap, @@ -598,13 +677,48 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if let SummaryOperator::Configured { + family, + aggregation_type, + .. + } = &operator + { + if let Some(expected) = aggregation_type.planner_exact_family() { + if family != &expected { + return Err(SdsError( + "configured storage type disagrees with Planner family".into(), + )); + } + } else { + use AggregationType as A; + let expected = match aggregation_type { + A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), + A::CountMinSketch => Some(SketchAlgorithm::Cms), + A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), + A::CountSketch => Some(SketchAlgorithm::CountSketch), + A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), + A::DDSketch => Some(SketchAlgorithm::DDSketch), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let Some(expected) = expected { + if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) + { + return Err(SdsError( + "configured sketch storage disagrees with Planner family".into(), + )); + } + } + } + } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), )); } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); + let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); Ok(Self { id, operator, @@ -640,6 +754,10 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { + family: config + .accumulator_spec() + .map_err(|error| SdsError(error.to_string()))? + .family, aggregation_type: config.aggregation_type, aggregation_sub_type: config.aggregation_sub_type.clone(), parameters: config @@ -667,11 +785,8 @@ impl FidelityGuarantee { matches!( (aggregation_type, self), (A::UnivMon, UnivMonFrequency { .. }) - | ( - A::Sum | A::MultipleSum | A::Min | A::Max | A::MultipleMin | A::MultipleMax, - Exact - ) - | (A::Increase | A::MultipleIncrease, ExactCounter { .. }) + | (A::Sum | A::Count | A::Min | A::Max, Exact) + | (A::Increase | A::Rate, ExactCounter { .. }) | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) | (A::DDSketch, DdSketchRelativeError { .. }) | (A::HLL, HllCardinalityError { .. }) @@ -809,33 +924,6 @@ impl ValueProjectionIdentity { } } -/// Compatibility adapter for old config column strings; storage is always typed. -pub(crate) fn deserialize_optional_value_projection<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - let value = Option::::deserialize(deserializer)?; - value - .map(|value| match value { - Value::String(name) => Ok(ValueProjectionIdentity::Column { name }), - value => serde_json::from_value(value).map_err(serde::de::Error::custom), - }) - .transpose() -} - -/// Read legacy StateSchema ColumnRef values without retaining a parallel field. -pub(crate) fn deserialize_state_value_projection<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result { - let value = Value::deserialize(deserializer)?; - if value == "SampleValue" { - return Ok(ValueProjectionIdentity::SampleValue); - } - if let Some(name) = value.get("Named").and_then(Value::as_str) { - return Ok(ValueProjectionIdentity::Column { name: name.into() }); - } - serde_json::from_value(value).map_err(serde::de::Error::custom) -} - /// Whether a materialization preserves source entities or pools a population. /// Grouped label names remain in `DataDescriptor::group_by_keys`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -1136,7 +1224,7 @@ mod tests { }, instance_id: SummaryInstanceId::new("instance-1").unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(crate::PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(crate::PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 1_000, end_ms: 2_000, @@ -1201,7 +1289,8 @@ mod tests { fn observed_instance(lifecycle: InstanceLifecycle) -> SummaryInstance { SummaryInstance { instance_id: SummaryInstanceId::new("instance-1").unwrap(), - summary_definition_id: SummaryDefinitionId(crate::PolicyFingerprint(7)), + stored_output_id: StoredOutputId(7), + summary_definition_id: SummaryDefinitionId::from_semantics(b"fixture"), summary_descriptor_id: descriptor( 200, FidelityGuarantee::KllRankError { @@ -1231,7 +1320,7 @@ mod tests { store: "summary-store".into(), key: "state/1".into(), state_schema_version: 1, - generation: 1, + generation: 2, sequence: 3, checksum: None, }, @@ -1260,6 +1349,51 @@ mod tests { inventory.validate().unwrap(); } + #[test] + fn stored_output_and_payload_version_must_match_instance_definition() { + let mut instance = observed_instance(InstanceLifecycle::Persistent); + instance.stored_output_id = StoredOutputId(8); + assert!(instance.validate().is_ok()); + instance.stored_output_id = StoredOutputId(7); + instance.state_reference.generation = 3; + assert!(instance.validate().is_err()); + let mut reference = StoredOutputReference::for_output(instance.stored_output_id); + reference.stored_output_id = StoredOutputId(8); + assert!(reference.validate().is_err()); + } + + // Old aliases and numeric semantic IDs must not authorize the new format. + #[test] + fn stored_output_reference_rejects_legacy_identity() { + assert!(serde_json::from_value::(json!({ + "state_slot_id": 7, "definition_id": 7 + })) + .is_err()); + let reference = StoredOutputReference { + stored_output_id: StoredOutputId(8), + definition_id: SummaryDefinitionId::from_semantics(b"fixture"), + }; + reference.validate().unwrap(); + assert_eq!( + serde_json::from_value::( + serde_json::to_value(&reference).unwrap() + ) + .unwrap(), + reference + ); + } + + #[test] + fn new_generation_rejects_cross_version_payload_adoption() { + let mut instance = observed_instance(InstanceLifecycle::Persistent); + let mut wire = serde_json::to_value(&instance).unwrap(); + wire["reused_from_generation"] = + serde_json::to_value(&instance.catalog_generation).unwrap(); + assert!(serde_json::from_value::(wire).is_err()); + instance.state_reference.generation -= 1; + assert!(instance.validate().is_err()); + } + #[test] fn invalid_range_and_lease_are_rejected() { let mut instance = observed_instance(InstanceLifecycle::Ephemeral { @@ -1395,19 +1529,14 @@ mod tests { } #[test] - fn catalog_generation_accepts_legacy_digest_name() { - let generation: CatalogGeneration = serde_json::from_value(json!({ + fn catalog_generation_rejects_legacy_digest_name() { + assert!(serde_json::from_value::(json!({ "schema_version": 1, "plan_id": 2, "plan_version": 3, "snapshot_digest": "abc" })) - .unwrap(); - assert_eq!(generation.snapshot_sha256, "abc"); - assert!(serde_json::to_value(generation) - .unwrap() - .get("snapshot_digest") - .is_none()); + .is_err()); } #[test] fn wire_roundtrip_and_tampered_id_validation() { @@ -1444,6 +1573,7 @@ mod tests { .is_err()); assert!(SummaryDescriptor::new( SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::new(), @@ -1468,6 +1598,24 @@ mod tests { ) .is_err()); } + + #[test] + fn configured_descriptor_rejects_family_storage_disagreement() { + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + family: AggregationType::Rate.planner_exact_family().unwrap(), + aggregation_type: AggregationType::Increase, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, + 2, + ) + .is_err()); + } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); @@ -1524,11 +1672,13 @@ mod tests { #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), }; let b = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), @@ -1566,9 +1716,9 @@ mod tests { assert_ne!(first.id, second.id); } #[test] - fn summary_definition_id_preserves_legacy_wire_identity() { + fn stored_output_id_preserves_legacy_wire_identity() { let fingerprint = crate::PolicyFingerprint(42); - let id = SummaryDefinitionId::from(fingerprint); + let id = StoredOutputId::from(fingerprint); assert_eq!(id.fingerprint(), fingerprint); assert_eq!(id.as_u64(), 42); assert_eq!(crate::PolicyFingerprint::from(id), fingerprint); @@ -1576,10 +1726,7 @@ mod tests { serde_json::to_value(id).unwrap(), serde_json::to_value(fingerprint).unwrap() ); - assert_eq!( - serde_json::from_str::("42").unwrap(), - id - ); + assert_eq!(serde_json::from_str::("42").unwrap(), id); } /// Every supplied alias must agree with the declared fidelity, including runtime w/d keys. #[test] diff --git a/crates/asap_types/src/semantic_fragment.rs b/crates/asap_types/src/semantic_fragment.rs new file mode 100644 index 000000000..390354bd9 --- /dev/null +++ b/crates/asap_types/src/semantic_fragment.rs @@ -0,0 +1,2 @@ +//! Planner owns the versioned semantic description and its normalization. +pub use planner_types::post_asap::SummarySemanticFragment as SemanticFragment; diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4d1ac0628..ec924e704 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -1,39 +1,27 @@ //! Authoritative descriptor snapshot for a compiled physical plan. //! -//! Execution plans keep their compatibility fields during migration. This -//! catalog owns semantic definitions, not producer placement or pane state. +//! The catalog owns semantic definitions; executable plans own writer and +//! reader bindings, while runtime inventory owns placement and pane state. use std::collections::BTreeMap; use crate::sds::{ - CatalogGeneration, DataDescriptor, DataDescriptorId, DataSourceIdentity, SummaryDefinitionId, + CatalogGeneration, DataDescriptor, DataDescriptorId, DataSourceIdentity, StoredOutputId, SummaryDescriptor, SummaryDescriptorId, }; use crate::PolicyFingerprint; -use crate::WindowMaterializationLayout; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 2; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 6; -/// Stable materialization identity binds operator and population descriptors. -/// Concrete intervals, groups and completeness belong to runtime instances. +/// Canonical definition binds operator and population descriptors. Writer +/// layout and concrete state belong to installed plans and runtime instances. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct SummaryDefinitionIdentity { +pub struct StoredOutputDefinition { + pub definition_id: crate::sds::SummaryDefinitionId, pub summary_descriptor_id: SummaryDescriptorId, pub data_descriptor_id: DataDescriptorId, - /// Backend-selected physical representation. It is catalog-visible so - /// producers, readers, lifecycle management, and recovery agree on the - /// concrete state being referenced. - pub window_layout: WindowMaterializationLayout, - /// Pane boundary selected from the shared consumer workload. Legacy - /// snapshots deserialize as unknown and fail closed at pane-only reads. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -44,7 +32,9 @@ pub struct SummaryCatalog { pub plan_version: u64, pub summary_descriptors: BTreeMap, pub data_descriptors: BTreeMap, - pub materializations: BTreeMap, + pub outputs: BTreeMap, + pub definitions: + BTreeMap, } #[derive(Debug, thiserror::Error)] @@ -53,9 +43,9 @@ pub enum SummaryCatalogError { SchemaVersion(u32), #[error("invalid summary catalog descriptor: {0}")] Descriptor(String), - #[error("materialization {0} has conflicting descriptor bindings")] - ConflictingMaterialization(u64), - #[error("materialization {0} references a missing descriptor")] + #[error("definition {0} has conflicting descriptor bindings")] + ConflictingDefinition(u64), + #[error("definition {0} references a missing descriptor")] MissingDescriptor(u64), #[error("catalog reference does not identify the supplied snapshot")] ReferenceMismatch, @@ -80,6 +70,20 @@ impl CatalogGeneration { } impl SummaryCatalog { + pub fn output_reference( + &self, + id: StoredOutputId, + ) -> Result { + let output = self + .outputs + .get(&id) + .ok_or(SummaryCatalogError::MissingDescriptor(id.as_u64()))?; + Ok(crate::sds::StoredOutputReference { + stored_output_id: id, + definition_id: output.definition_id.clone(), + }) + } + pub fn reference(&self) -> Result { use sha2::{Digest, Sha256}; self.validate()?; @@ -123,53 +127,47 @@ impl SummaryCatalog { .with_partitioning(config.partitioning) .with_population_key_encoding(config.population_key_encoding) .with_timestamp_column(config.table_timestamp_column.clone()); - Ok(( - config.policy_fingerprint(), - summary, - data, - config.window_layout.clone(), - config.pane_origin_ms, - )) + Ok((config.policy_fingerprint(), summary, data)) }) .collect::, SummaryCatalogError>>()?; - Self::build_with_origins(plan_id, plan_version, entries) + let mut catalog = Self::build(plan_id, plan_version, entries)?; + let mut semantic_bindings = BTreeMap::new(); + for config in materializations { + let output = catalog + .outputs + .get_mut(&StoredOutputId::from(config.policy_fingerprint())) + .unwrap(); + let definition = crate::summary_semantics::SummaryDefinition::from_descriptors( + &catalog.summary_descriptors[&output.summary_descriptor_id], + &catalog.data_descriptors[&output.data_descriptor_id], + )? + .with_config(config); + let id = definition.id()?; + if semantic_bindings + .insert(config.policy_fingerprint(), id.clone()) + .is_some_and(|old| old != id) + { + return Err(SummaryCatalogError::ConflictingDefinition( + config.policy_fingerprint().0, + )); + } + output.definition_id = id.clone(); + catalog.definitions.insert(id, definition); + } + let used: std::collections::BTreeSet<_> = catalog + .outputs + .values() + .map(|o| o.definition_id.clone()) + .collect(); + catalog.definitions.retain(|id, _| used.contains(id)); + catalog.validate()?; + Ok(catalog) } pub fn build( plan_id: u64, plan_version: u64, - entries: impl IntoIterator< - Item = ( - PolicyFingerprint, - SummaryDescriptor, - DataDescriptor, - WindowMaterializationLayout, - ), - >, - ) -> Result { - Self::build_with_origins( - plan_id, - plan_version, - entries - .into_iter() - .map(|(fingerprint, summary, data, layout)| { - (fingerprint, summary, data, layout, None) - }), - ) - } - - fn build_with_origins( - plan_id: u64, - plan_version: u64, - entries: impl IntoIterator< - Item = ( - PolicyFingerprint, - SummaryDescriptor, - DataDescriptor, - WindowMaterializationLayout, - Option, - ), - >, + entries: impl IntoIterator, ) -> Result { let mut catalog = Self { schema_version: SUMMARY_CATALOG_SCHEMA_VERSION, @@ -177,28 +175,32 @@ impl SummaryCatalog { plan_version, summary_descriptors: BTreeMap::new(), data_descriptors: BTreeMap::new(), - materializations: BTreeMap::new(), + outputs: BTreeMap::new(), + definitions: BTreeMap::new(), }; - for (fingerprint, summary, data, window_layout, pane_origin_ms) in entries { - let materialization = SummaryDefinitionId::from(fingerprint); + for (fingerprint, summary, data) in entries { + let definition = StoredOutputId::from(fingerprint); summary .validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; data.validate() .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let binding = SummaryDefinitionIdentity { + let semantics = + crate::summary_semantics::SummaryDefinition::from_descriptors(&summary, &data)?; + let semantic_id = semantics.id()?; + catalog.definitions.insert(semantic_id.clone(), semantics); + let binding = StoredOutputDefinition { + definition_id: semantic_id, summary_descriptor_id: summary.id().clone(), data_descriptor_id: data.id().clone(), - window_layout, - pane_origin_ms, }; if catalog - .materializations - .get(&materialization) + .outputs + .get(&definition) .is_some_and(|old| old != &binding) { - return Err(SummaryCatalogError::ConflictingMaterialization( - materialization.as_u64(), + return Err(SummaryCatalogError::ConflictingDefinition( + definition.as_u64(), )); } catalog @@ -207,7 +209,7 @@ impl SummaryCatalog { catalog .data_descriptors .insert(binding.data_descriptor_id.clone(), data); - catalog.materializations.insert(materialization, binding); + catalog.outputs.insert(definition, binding); } catalog.validate()?; Ok(catalog) @@ -217,6 +219,13 @@ impl SummaryCatalog { if self.schema_version != SUMMARY_CATALOG_SCHEMA_VERSION { return Err(SummaryCatalogError::SchemaVersion(self.schema_version)); } + for (id, definition) in &self.definitions { + if &definition.id()? != id { + return Err(SummaryCatalogError::Descriptor( + "semantic definition content hash mismatch".into(), + )); + } + } for (key, descriptor) in &self.summary_descriptors { descriptor .validate() @@ -237,7 +246,12 @@ impl SummaryCatalog { )); } } - for (id, binding) in &self.materializations { + for (id, binding) in &self.outputs { + if !self.definitions.contains_key(&binding.definition_id) { + return Err(SummaryCatalogError::Descriptor( + "output references absent semantic definition".into(), + )); + } if !self .summary_descriptors .contains_key(&binding.summary_descriptor_id) @@ -248,6 +262,31 @@ impl SummaryCatalog { return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); } } + for output in self.outputs.values() { + let expected = crate::summary_semantics::SummaryDefinition::from_descriptors( + &self.summary_descriptors[&output.summary_descriptor_id], + &self.data_descriptors[&output.data_descriptor_id], + )?; + if let ( + crate::summary_semantics::SummarySemantics::Configured { + computation: actual, + .. + }, + crate::summary_semantics::SummarySemantics::Configured { + computation: expected, + .. + }, + ) = ( + &self.definitions[&output.definition_id].semantics, + &expected.semantics, + ) { + if actual != expected { + return Err(SummaryCatalogError::Descriptor( + "output descriptors disagree with semantic definition".into(), + )); + } + } + } if !self .data_descriptors .values() @@ -259,13 +298,13 @@ impl SummaryCatalog { let mut pending = std::collections::BTreeMap::new(); let mut consumers: std::collections::BTreeMap<_, Vec<_>> = std::collections::BTreeMap::new(); - for (id, binding) in &self.materializations { + for (id, binding) in &self.outputs { let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { DataSourceIdentity::Derived { input } => input.inputs.clone(), _ => Default::default(), }; for source in &dependencies { - if !self.materializations.contains_key(source) { + if !self.outputs.contains_key(source) { return Err(SummaryCatalogError::Descriptor( "derived input references missing summary".into(), )); @@ -377,7 +416,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); assert_eq!(catalog.data_descriptors.len(), 3); - assert_eq!(catalog.materializations.len(), 3); + assert_eq!(catalog.outputs.len(), 3); } #[test] @@ -409,10 +448,54 @@ mod tests { SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); + assert_eq!(catalog.outputs.len(), 2); assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); } + // Stored pane duration identifies a deployment output, not the summary meaning. + #[test] + fn semantic_definition_is_shared_across_deployed_pane_outputs() { + let catalog = SummaryCatalog::from_materializations( + 1, + 1, + &[config("requests", "", 60), config("requests", "", 120)], + ) + .unwrap(); + assert_eq!(catalog.definitions.len(), 1); + assert_eq!(catalog.outputs.len(), 2); + } + + // A hot and rebuild output share meaning but remain independently bound. + #[test] + fn hot_and_rebuild_have_one_definition_and_two_bound_outputs() { + let mut hot = config("latency", "", 60); + hot.stored_output_id = Some(StoredOutputId(41)); + let mut rebuild = hot.clone(); + rebuild.stored_output_id = Some(StoredOutputId(42)); + let catalog = SummaryCatalog::from_materializations(7, 42, &[hot, rebuild]).unwrap(); + assert_eq!(catalog.definitions.len(), 1); + let hot = catalog.output_reference(StoredOutputId(41)).unwrap(); + let rebuild = catalog.output_reference(StoredOutputId(42)).unwrap(); + assert_eq!(hot.definition_id, rebuild.definition_id); + assert_ne!(hot, rebuild); + let mut forged = catalog.clone(); + let crate::summary_semantics::SummarySemantics::Configured { computation, .. } = + &mut forged.definitions.values_mut().next().unwrap().semantics + else { + panic!("fixture") + }; + computation.predicate = "service=other".into(); + assert!(forged.validate().is_err()); + let mut unknown = catalog.clone(); + unknown + .definitions + .values_mut() + .next() + .unwrap() + .semantic_format_version += 1; + assert!(unknown.validate().is_err()); + } + // Source/population changes never alias, while the operator can be reused. #[test] fn separates_population_and_operator_identity() { @@ -441,7 +524,7 @@ mod tests { let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 2); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); + assert_eq!(catalog.outputs.len(), 2); } // Construction order cannot affect the published snapshot bytes. @@ -459,20 +542,22 @@ mod tests { } #[test] - fn catalog_persists_definition_pane_origin() { + fn catalog_definitions_do_not_store_writer_layout() { let mut materialization = config("requests", "", 60); materialization.pane_origin_ms = Some(7_000); - let id = SummaryDefinitionId::from(materialization.policy_fingerprint()); + let id = StoredOutputId::from(materialization.policy_fingerprint()); let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); - assert_eq!(catalog.materializations[&id].pane_origin_ms, Some(7_000)); + assert!(catalog.outputs.contains_key(&id)); + assert!( + !serde_json::to_value(&catalog).unwrap()["outputs"][id.as_u64().to_string()] + .as_object() + .unwrap() + .contains_key("pane_origin_ms") + ); - let mut legacy = serde_json::to_value(&catalog).unwrap(); - legacy["materializations"][id.as_u64().to_string()] - .as_object_mut() - .unwrap() - .remove("pane_origin_ms"); - let decoded: SummaryCatalog = serde_json::from_value(legacy).unwrap(); - assert_eq!(decoded.materializations[&id].pane_origin_ms, None); + let mut invalid = serde_json::to_value(&catalog).unwrap(); + invalid["outputs"][id.as_u64().to_string()]["pane_origin_ms"] = serde_json::json!(7_000); + assert!(serde_json::from_value::(invalid).is_err()); } // The same materialization cannot silently rebind to another population. @@ -492,24 +577,14 @@ mod tests { 1, 1, [ - ( - first.policy_fingerprint(), - summary.clone(), - data[0].clone(), - first.window_layout.clone(), - ), - ( - first.policy_fingerprint(), - summary, - data[1].clone(), - first.window_layout.clone(), - ), + (first.policy_fingerprint(), summary.clone(), data[0].clone()), + (first.policy_fingerprint(), summary, data[1].clone()), ], ) .unwrap_err(); assert!(matches!( error, - SummaryCatalogError::ConflictingMaterialization(_) + SummaryCatalogError::ConflictingDefinition(_) )); } @@ -557,7 +632,7 @@ mod tests { #[test] fn empty_catalog_is_valid() { let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - assert!(catalog.materializations.is_empty()); + assert!(catalog.outputs.is_empty()); catalog.validate().unwrap(); } } diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs new file mode 100644 index 000000000..757a7e15b --- /dev/null +++ b/crates/asap_types/src/summary_semantics.rs @@ -0,0 +1,129 @@ +//! Versioned, content-addressed meaning of a stored result. Runtime routing, +//! pane placement, codecs, retention and plan generations are deliberately absent. +use crate::sds::{ + DataDescriptor, DataSourceIdentity, SummaryDescriptor, SummaryOperator, ValueProjectionIdentity, +}; +use crate::summary_catalog::SummaryCatalogError; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryDefinition { + pub semantic_format_version: u32, + pub semantics: SummarySemantics, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SummarySemantics { + Planner { + fragment: crate::semantic_fragment::SemanticFragment, + }, + /// Restricted raw-input adapter for explicit native summary configurations. + /// General expressions must use the Planner fragment variant. + Configured { + computation: Box, + value_source_column: Option, + aggregated_labels: crate::KeyByLabelNames, + rollup_labels: crate::KeyByLabelNames, + }, +} + +/// Typed semantic contract of the supported summary input. This is not a +/// deployment plan: it contains neither physical nodes nor storage references. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryComputation { + pub operator: SummaryOperator, + pub source: DataSourceIdentity, + pub value: ValueProjectionIdentity, + pub predicate: String, + pub grouping: crate::GroupingProjection, + pub timestamp_column: Option, + pub observation_semantics: String, +} + +impl SummaryDefinition { + pub fn from_descriptors( + summary: &SummaryDescriptor, + data: &DataDescriptor, + ) -> Result { + summary + .validate() + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + data.validate() + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + Ok(Self { + semantic_format_version: 1, + semantics: SummarySemantics::Configured { + value_source_column: None, + aggregated_labels: crate::KeyByLabelNames::empty(), + rollup_labels: crate::KeyByLabelNames::empty(), + computation: Box::new(SummaryComputation { + operator: summary.operator.clone(), + source: data.source.clone(), + value: data.value_projection.clone(), + predicate: data.population_filter_canonical.clone(), + grouping: data.group_by_keys.clone(), + timestamp_column: data.timestamp_column.clone(), + observation_semantics: data.observation_semantics.clone(), + }), + }, + }) + } + + pub fn with_config(mut self, config: &crate::PrecomputeMaterialization) -> Self { + let fragment = config.semantic_fragment.as_ref(); + if let Some(fragment) = fragment { + self.semantics = SummarySemantics::Planner { + fragment: fragment.clone(), + }; + } else if let SummarySemantics::Configured { + value_source_column, + aggregated_labels, + rollup_labels, + .. + } = &mut self.semantics + { + *value_source_column = config.value_source_column.clone(); + *aggregated_labels = config.aggregated_labels.clone(); + *rollup_labels = config.rollup_labels.clone(); + } + self + } + + pub fn id(&self) -> Result { + if self.semantic_format_version != 1 { + return Err(SummaryCatalogError::Descriptor( + "unsupported semantic format version".into(), + )); + } + if let SummarySemantics::Planner { fragment } = &self.semantics { + fragment + .validate() + .map_err(SummaryCatalogError::Descriptor)?; + } + // Object keys are recursively sorted, independent of serde_json features. + fn canonical(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(values) => serde_json::Value::Object( + values + .into_iter() + .map(|(k, v)| (k, canonical(v))) + .collect::>() + .into_iter() + .collect(), + ), + serde_json::Value::Array(values) => { + serde_json::Value::Array(values.into_iter().map(canonical).collect()) + } + value => value, + } + } + let value = serde_json::to_value(self) + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + let bytes = serde_json::to_vec(&canonical(value)) + .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; + Ok(crate::sds::SummaryDefinitionId::from_semantics(&bytes)) + } +} diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 60d4549a0..7482c5210 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] # Internal crates (workspace) asap_types.workspace = true +asap_sketch_codec = { path = "../crates/asap_sketch_codec" } # Phase 9: the control plane is now an in-process library inside the # backend binary. Wiring up the in-process OpAMP server + capability-map # exposure is a follow-up after Phase 4 (centralized series_id @@ -72,9 +73,6 @@ asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch # existing PUBLIC `from_legacy_matrix`/`sketch_matrix`/`topk_heap_items` API. # Already in the lock as a transitive dep of `asap_sketchlib`. rmp-serde = "1.3" -# Shared collector ingest implementation. This follows ASAPCollector's -# current main branch alongside the direct Sketchlib dependency above. -asap-precompute-rs = { git = "https://github.com/ProjectASAP/ASAPCollector", branch = "main" } # Persistence layer (SketchStore parts / manifest / Tier-2 cache) moka = { version = "0.12", features = ["sync"] } memmap2 = "0.9" diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 01162475e..11cbfa4a0 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -395,13 +395,13 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; use std::collections::HashMap; - let cfg = AggregationConfig::new( + let cfg = PrecomputeMaterialization::new( AT::Sum, String::new(), HashMap::new(), diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 81781851b..22eefcd16 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -73,7 +73,7 @@ async fn main() { let reference = catalog.reference().unwrap(); let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); - precompute_plan.summary_catalog = Some(reference.clone()); + precompute_plan.bind_catalog(&catalog).unwrap(); let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, @@ -89,6 +89,7 @@ async fn main() { tables: HashMap::from([("raw_samples".into(), schema)]), accuracy: AccuracyTarget::Epsilon(0.01), }), + selected_dags: Default::default(), entries: BTreeMap::new(), }; let active = validate_and_build_runtime_plan( diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 8de15bfd4..fea49a0a2 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -582,7 +582,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t } /// Resolve the bucket sid (and `policy_fp`) for a single data point -/// against a single matching `AggregationConfig`. +/// against a single matching `PrecomputeMaterialization`. /// /// B7.6 — sid is the bucket identity in the precompute engine; this /// helper folds `(config, grouping-label-values)` into a single u64 via @@ -602,7 +602,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t /// their separate wire-level identity protocol. fn resolve_bucket_sid_for_agg_config( ingest_state: &Arc, - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, point_labels: &HashMap, captured_generation: Option<&asap_types::sds::CatalogGeneration>, ) -> Result<(u64, asap_types::PolicyFingerprint), String> { @@ -1146,7 +1146,11 @@ async fn route_modified_otlp_sketches_to_precompute( } else { None }; - let series_key = format_series_key(&canonical_name, &dp.attrs); + let series_key = bound_sketch_series_key( + &canonical_name, + &dp.attrs, + frame_identity.as_ref(), + ); let ts_ms = (dp.time_unix_nano / 1_000_000) as i64; // Sid resolution — registry-allocated, NOT content- @@ -1260,7 +1264,16 @@ async fn route_modified_otlp_sketches_to_precompute( // policy". spatial_filter_canonical: String::new(), }; - let agg_kind_canonical = agg_kind.canonical_string(); + let agg_kind_canonical = match frame_identity.as_ref() { + Some(frame) => format!( + "{}|output:{}:{}:{}", + agg_kind.canonical_string(), + frame.plan_id, + frame.plan_version, + frame.materialization.as_u64() + ), + None => agg_kind.canonical_string(), + }; let definition = frame_identity .as_ref() .map(|frame| frame.materialization) @@ -1340,6 +1353,7 @@ async fn route_modified_otlp_sketches_to_precompute( sketch_algorithm_for(&dp), &dp.container_config, &dp.attrs.keys().cloned().collect(), + Some(frame.materialization), ) }); if observed_policy != frame.materialization.fingerprint() { @@ -1418,6 +1432,7 @@ async fn route_modified_otlp_sketches_to_precompute( algorithm.clone(), &cfg, &group_by_keys, + frame_identity.as_ref().map(|frame| frame.materialization), ); // Per-item dimension (item_label) the controller threaded // into the matched policy's parameters — recorded on the sid @@ -1783,15 +1798,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Detection is independent of the legacy dual-write // (it only drives the routed/unconfigured accounting), // so we walk it whether or not the worker push fires. - let matching_configs: Vec<&asap_types::aggregation_config::AggregationConfig> = - agg_configs - .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) - .collect(); + let matching_configs: Vec< + &asap_types::aggregation_config::PrecomputeMaterialization, + > = agg_configs + .values() + .filter(|config| { + config.metric == canonical_name + || config.spatial_filter_normalized == canonical_name + || config.spatial_filter == canonical_name + }) + .collect(); let matched_any = !matching_configs.is_empty(); // CQ-2 — only pay the worker push (and the per-config @@ -1869,7 +1885,7 @@ async fn route_modified_otlp_sketches_to_precompute( routed += 1; } else { // CQ-6 — a decoded sketch that matched no - // AggregationConfig in the running streaming config. + // PrecomputeMaterialization in the running streaming config. ingest_state .observability .dropped_unconfigured @@ -1914,7 +1930,7 @@ async fn route_modified_otlp_sketches_to_precompute( /// `AggregationType`. Inverse direction is in /// `sketch_algorithm_for` above. Used by /// [`derive_sketch_policy_fp`] to find the policy whose -/// `AggregationConfig.aggregation_type` matches a freshly-ingested +/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested /// sketch. /// /// `Any` is a control-plane analysis-time wildcard — it doesn't @@ -1991,38 +2007,47 @@ fn sketch_config_to_params( params } -/// Look up the policy fingerprint for a freshly-ingested OTLP sketch -/// by content-matching against the streaming-config registry. -/// -/// Sketches arrive with `(metric, attrs, sketch_kind, sketch_config)` -/// embedded in the DP but no policy reference. The matching pass: -/// snapshots the current streaming config, derives a -/// `PolicyRegistry`, and asks `find_policy_by_content` for the -/// fingerprint of a policy whose contents match. Returns -/// `PolicyFingerprint::UNSET` when: -/// 1. An unsupported planner algorithm reached this path -/// (defensive — shouldn't happen). -/// 2. No policy in the registry matches. -/// 3. Multiple policies match (would-have-been-a-bug case; -/// `find_policy_by_content` returns `None` on ambiguity). -/// -/// Callers register the sid with the returned fp regardless of -/// success — UNSET sids are simply absent from the policy_fp → -/// {sids} reverse index, and remain reachable via the legacy -/// `instances_matching(metric, gbk)` walk. +// Snapshot and delta bases are scoped to the producer, never only its semantics. +fn bound_sketch_series_key( + name: &str, + labels: &HashMap, + frame: Option<&asap_types::producer_plan::SummaryFrameIdentity>, +) -> String { + let key = format_series_key(name, labels); + match frame { + Some(frame) => format!( + "{}:{}:{}:{key}", + frame.plan_id, + frame.plan_version, + frame.materialization.as_u64() + ), + None => key, + } +} + +/// Resolve the framed deployed output, then verify its content contract. +/// Unframed input uses legacy content matching and must have exactly one match. +/// An absent or ambiguous match returns UNSET and cannot authorize bound writes. fn derive_sketch_policy_fp( ingest_state: &IngestState, metric: &str, kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, cfg: &crate::storage_engines::sketch_db::data::SketchConfig, group_by_keys: &std::collections::BTreeSet, + bound_output: Option, ) -> asap_types::PolicyFingerprint { let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { return asap_types::PolicyFingerprint::UNSET; }; let params = sketch_config_to_params(cfg); let snap = ingest_state.config_snapshot(); - let index = asap_types::RoutingIndex::build(snap.policy_registry()); + let registry = snap.policy_registry(); + let registry = if let Some(output) = bound_output { + asap_types::PolicyRegistry::from_configs(registry.get(output.fingerprint()).cloned()) + } else { + registry + }; + let index = asap_types::RoutingIndex::build(registry); index .find_policy_by_content(metric, group_by_keys, agg_type, ¶ms) .unwrap_or(asap_types::PolicyFingerprint::UNSET) @@ -2303,7 +2328,7 @@ fn preflight_summary_frames( decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; } else { - let series_key = format_series_key(canonical_name, &dp.attrs); + let series_key = bound_sketch_series_key(canonical_name, &dp.attrs, Some(&frame)); let (mut base, base_window_start) = ingest_state .sketch_snapshots .get(&series_key) @@ -2352,6 +2377,7 @@ fn preflight_summary_frames( sketch_algorithm_for(&dp), &dp.container_config, &dp.attrs.keys().cloned().collect(), + Some(frame.materialization), ) }; if observed != frame.materialization.fingerprint() { @@ -2565,71 +2591,20 @@ fn decode_modified_otlp_sketch_bytes( match encoding { ENCODING_PROTO => match algorithm { - // Phase 3 step 3: DDSketch and KLL envelope-parsing / - // sketch reconstruction route through the shared - // `edge_runtime_adapter`, which delegates to - // `asap-precompute-rs`'s `Sketch` trait. Backend's - // accumulator wraps the result. Byte parity with Go is - // covered by `asap_sketchlib` PRs #40 (DDSketch) and #41 - // (KLL). - // - // HLL / CountSketch / CountMinSketch byte parity is - // tracked under ProjectASAP/ASAPCollector#243 — until it - // lands those three sketches keep using the backend's - // existing per-accumulator decoder. + // The neutral codec accepts both full envelopes and supported bare + // states. Query accumulators retain their family-specific readouts. SketchAlgorithm::DDSketch => { - use crate::precompute_engine::operators::edge_runtime_adapter::{ - reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, - }; - // Prefer the asap-precompute-rs runtime path (envelope- - // wrapped bytes, the canonical edge-framework wire format). - // If the input is a bare `DdSketchState` (as some unit-test - // / pre-envelope agent payloads still emit, mirrored by the - // PR #14 contract on `from_sketchlib_proto_bytes`), the - // adapter returns an error decoding the envelope — fall - // back to the backend's native decoder which already - // accepts both shapes. - match reconstruct_via_runtime(RtSketchType::DDSketch, bytes) { - Ok(ReconstructedSketch::DdSketch(inner)) => { - // The runtime reconstruction discards the envelope's - // sample_p; re-read it from the same full-frame bytes - // so a sampled series rescales its Count by 1/p. - let sample_p = DDSketchAccumulator::sample_p_from_envelope_bytes(bytes); - Ok(Box::new(DDSketchAccumulator { inner, sample_p })) - } - Ok(_) => { - Err("edge_runtime_adapter returned non-DDSketch reconstruction".into()) - } - Err(_) => Ok(Box::new(DDSketchAccumulator::from_sketchlib_proto_bytes( - bytes, - )?)), - } - } - SketchAlgorithm::Kll => { - use crate::precompute_engine::operators::edge_runtime_adapter::{ - reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, + let (inner, sample_p) = asap_sketch_codec::reconstruct_ddsketch(bytes)?; + let sample_p = if sample_p.is_finite() && sample_p > 0.0 && sample_p < 1.0 { + sample_p + } else { + 1.0 }; - // Same envelope-vs-bare-state handling as DDSketch above. - // Backend's KLL accumulator owns the wire-format-aligned - // `KllSketch` rather than the high-throughput `KLL` - // that asap-precompute-rs's `KLLWrapper` wraps internally - // — when the adapter succeeds, bridge by re-feeding the - // wrapper's snapshot bytes through backend's existing - // decoder. The envelope work (decode + state extraction - // + reconstruction) has already happened in the runtime - // adapter; this final step just reshapes into backend's - // accumulator type. On envelope-decode failure (bare - // state bytes) fall through to the native decoder. - match reconstruct_via_runtime(RtSketchType::KLLSketch, bytes) { - Ok(ReconstructedSketch::Kll { snapshot_bytes }) => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&snapshot_bytes)?, - )), - Ok(_) => Err("edge_runtime_adapter returned non-KLL reconstruction".into()), - Err(_) => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - } + Ok(Box::new(DDSketchAccumulator { inner, sample_p })) } + SketchAlgorithm::Kll => Ok(Box::new( + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, + )), SketchAlgorithm::Cms => Ok(Box::new( CountMinSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), @@ -3466,7 +3441,7 @@ mod policy_fp_lookup_tests { fn sketch_config_to_params_uses_canonical_keys() { // The param-name vocabulary must match what the control plane // writes in streaming-config YAML (see - // `asap_types::aggregation_config::AggregationConfig::from_yaml_data`). + // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). // Drift surfaces as `find_policy_by_content` missing matches. let dd = sketch_config_to_params(&SketchConfig::DDSketch { relative_accuracy: 0.01, @@ -3849,7 +3824,7 @@ mod sid_resolution_tests { async fn delta_apply_rotates_per_series_base_at_window_boundary() { use crate::precompute_engine::operators::DDSketchAccumulator; use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; - use asap_sketchlib::proto::sketchlib::DdSketchState; + use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; let (state, drain) = make_state().await; @@ -3881,10 +3856,13 @@ mod sid_resolution_tests { ); // ── Window 1: full frame. Base buckets [10, 0, 5]. ── - let full_w1 = DdSketchState { - alpha: 0.01, - store_counts: vec![10, 0, 5], - store_offset: 0, + let full_w1 = SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(DdSketchState { + alpha: 0.01, + store_counts: vec![10, 0, 5], + store_offset: 0, + })), + ..Default::default() } .encode_to_vec(); route_modified_otlp_sketches_to_precompute( @@ -4564,7 +4542,7 @@ mod sid_bucketing_tests { metric::Data, number_data_point::Value as NumberValue, Gauge as PbGauge, Metric as PbMetric, NumberDataPoint, ResourceMetrics, ScopeMetrics, }; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -4581,8 +4559,8 @@ mod sid_bucketing_tests { } } - fn sum_agg_config(metric: &str, grouping: &[&str]) -> AggregationConfig { - AggregationConfig::new( + fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index b6d3bfd73..3ddd40607 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -433,9 +433,9 @@ impl PrometheusRemoteWriteReceiver { } for start in starts { let (start_ms, end_ms) = manager.stored_bucket_bounds(start); - for summary_definition_id in &affected { + for stored_output_id in &affected { coordinates.insert(asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: *summary_definition_id, + stored_output_id: *stored_output_id, time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values: labels.clone(), }); @@ -717,11 +717,9 @@ fn route_messages( && matches!( config.aggregation_type, asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + | asap_types::AggregationType::Rate | asap_types::AggregationType::Min | asap_types::AggregationType::Max - | asap_types::AggregationType::MultipleMin - | asap_types::AggregationType::MultipleMax )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -988,11 +986,12 @@ mod tests { summary_catalog: Some(generation), envelope: envelope.clone(), ingest: IngestContract { + dataset_identity: None, protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), @@ -1040,8 +1039,10 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; - let aggregation = AggregationConfig { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; + let aggregation = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -1165,10 +1166,12 @@ mod tests { #[test] fn global_topk_cms_routes_once_while_counters_remain_per_series() { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let config = - |aggregation_type, grouping: Vec, aggregated: Vec| AggregationConfig { + let config = |aggregation_type, grouping: Vec, aggregated: Vec| { + PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -1203,7 +1206,8 @@ mod tests { table_timestamp_column: None, partitioning: None, value_source_column: None, - }; + } + }; let cms = config( AggregationType::CountMinSketchWithHeap, vec![], @@ -1628,6 +1632,12 @@ mod tests { let binding = asap_types::query_plan::MaterializationBinding { full_window_slide_ms: None, materialization: asap_types::PolicyFingerprint(policy).into(), + stored_output_reference: ingest + .summary_store + .summary_catalog_snapshot() + .unwrap() + .output_reference(asap_types::PolicyFingerprint(policy).into()) + .unwrap(), output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 382ec0b43..8c03df23f 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1004,9 +1004,9 @@ fn metric_has_exact_agg_sum_sid( cap, Capability::ExactAgg( AggregationType::Sum - | AggregationType::MultipleSum + | AggregationType::Count | AggregationType::Increase - | AggregationType::MultipleIncrease + | AggregationType::Rate ) ) { return true; @@ -2524,11 +2524,12 @@ mod tests { summary_catalog: Some(generation.clone()), envelope: envelope.clone(), ingest: IngestContract { + dataset_identity: None, protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), timestamp_unit: TimestampUnit::UnixMilliseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), @@ -2552,6 +2553,7 @@ mod tests { plan_id: 7, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: Default::default(), }), storage_routing: Arc::new( @@ -2847,100 +2849,18 @@ mod tests { /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and /// the GET snapshot emission. #[tokio::test] - async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; - let client = Client::new(); - - // Initial GET: empty config, 0 entries. - let initial = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .send() - .await - .expect("GET failed"); - assert!(initial.status().is_success()); - let initial_body: serde_json::Value = initial.json().await.unwrap(); - assert_eq!(initial_body["aggregation_count"], 0); - - // POST a new config with two aggregation_ids. The YAML shape - // matches what `StreamingConfig::from_yaml_data` parses — see - // `asap-common/dependencies/rs/asap_types/src/streaming_config.rs`. - let new_config_yaml = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 102 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host, region] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 120 - windowType: tumbling - spatialFilter: '' -"#; - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(new_config_yaml.to_string()) - .send() - .await - .expect("POST failed"); - let post_status = post_resp.status(); - let post_body: serde_json::Value = post_resp.json().await.unwrap(); - assert!( - post_status.is_success(), - "POST returned {post_status}: {post_body}" - ); - assert_eq!(post_body["status"], "success"); - assert_eq!(post_body["new_aggregation_count"], 2); - // PR 5: the YAML's `aggregationId` fields are silently - // dropped — `agg_ids_added` carries fingerprint u64s. - let added = post_body["agg_ids_added"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(added.len(), 2, "exactly two distinct aggs were added"); - assert!(added.iter().all(|id| *id != 0), "fingerprints are non-zero"); - - // GET again: should reflect the two new ids. - let after = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) + async fn flat_streaming_config_is_rejected_without_mutating_active_state() { + let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; + let before = handle.snapshot(); + let response = Client::new() + .post(format!("http://127.0.0.1:{port}/api/v1/streaming-config")) + .body("aggregations: [{aggregationType: Sum, metric: m}]") .send() .await - .expect("GET after swap failed"); - assert!(after.status().is_success()); - let after_body: serde_json::Value = after.json().await.unwrap(); - assert_eq!(after_body["aggregation_count"], 2); - - // The underlying StreamingConfigHandle handle (cloned into - // the server at setup) also reflects the swap — proving that - // downstream consumers that re-snapshot would see the new - // state. PR 5: the map is keyed on fingerprints, so just - // assert the entry count. - let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert!(Arc::ptr_eq(&before, &handle.snapshot())); } #[tokio::test] @@ -2983,7 +2903,7 @@ aggregations: .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), reqwest::StatusCode::GONE); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "error"); } @@ -3048,192 +2968,6 @@ aggregations: }); } - #[tokio::test] - async fn test_streaming_config_swap_drives_sid_reconcile() { - // Schema retirement final cut: the swap handler now drives a - // single sid-level reconcile (no `SchemaRegistry`). Sids that - // already exist in the catalog and whose content signature - // does not appear in the new config get force-retired; the - // response surfaces them under `sids_retired`. There is no - // `sids_added` — sids are minted lazily by the ingest path, - // not by the swap handler. - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::sketch_db::AggStatus; - - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let summary_store = Arc::new(SketchStore::new()); - // Pre-register two Active sids whose signatures match the - // first config below; only sid 1 will survive the second - // swap. - register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); - register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); - let server_port = setup_test_server_with_hot_reload_and_sketch_index( - hot_reload.clone(), - summary_store.clone(), - ) - .await; - let client = Client::new(); - - // POST a config whose signatures cover both pre-registered - // sids. Nothing should retire. - let yaml_two = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 202 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_two.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - let retired_ids = body["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert!( - retired_ids.is_empty(), - "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", - ); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Active - ); - - // Swap to a config that drops `mem_usage`. Sid 2's signature - // is now orphaned; the handler must force-retire it. - let yaml_one = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp2 = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .expect("POST failed"); - let body2: serde_json::Value = resp2.json().await.unwrap(); - let retired = body2["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(retired, vec![2u64]); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Retired - ); - } - - #[tokio::test] - async fn test_streaming_config_swap_response_shape_with_empty_catalog() { - // With no registered sids, the swap still works — it just - // produces an empty `sids_retired` array. The `agg_ids_added` - // / `agg_ids_removed` / `new_aggregation_count` fields are - // driven purely by the diff of the two configs and are - // independent of the sid catalog. - // - // PR 5: the YAML's `aggregationId: 42` is silently dropped at - // parse time — the backend identity is content-addressed via - // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 - // in the HTTP response is the fingerprint's `as_u64()` form, - // NOT the literal `42` the YAML once spelled out. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; - let client = Client::new(); - - let yaml = r#" -aggregations: - - aggregationType: Sum - aggregationSubType: '' - metric: m - labels: - grouping: [] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - assert_eq!(body["new_aggregation_count"], 1); - let added = body["agg_ids_added"].as_array().expect("array"); - assert_eq!(added.len(), 1, "exactly one agg was added"); - assert_ne!( - added[0].as_u64().unwrap(), - 0, - "agg id is not the 0 sentinel" - ); - assert_eq!(body["agg_ids_removed"], serde_json::json!([])); - // No pre-registered sids → nothing to retire. - assert_eq!(body["sids_retired"].as_array().unwrap().len(), 0); - } - #[tokio::test] async fn test_get_schemas_returns_active_and_retired_sids_with_status_filter() { // Schema retirement final cut: `/api/v1/db/schemas` now @@ -3253,29 +2987,12 @@ aggregations: .await; let client = Client::new(); - // Retire sid 2 by pushing a config covering only `m1`. - let yaml_one = r#" -aggregations: - - aggregationId: 1 - aggregationType: Sum - aggregationSubType: '' - metric: m1 - labels: { grouping: [], rollup: [], aggregated: [] } - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .unwrap(); - assert!(resp.status().is_success()); + assert!(summary_store + .force_retire( + 2, + crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION + ) + .is_some()); // GET /api/v1/db/schemas (no filter = all). let resp = client @@ -3512,7 +3229,7 @@ aggregations: registry: Arc, active_agg_ids: &[u64], ) -> (u16, std::collections::HashMap) { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -3534,7 +3251,9 @@ aggregations: let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -5617,96 +5336,11 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res (StatusCode::OK, axum::Json(body)).into_response() } -async fn handle_post_streaming_config( - State(state): State, - body: axum::body::Bytes, -) -> axum::response::Response { - use axum::http::StatusCode; +async fn handle_post_streaming_config() -> axum::response::Response { use axum::response::IntoResponse; - use std::collections::HashSet; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - - let yaml_text = match std::str::from_utf8(&body) { - Ok(s) => s, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("request body is not valid UTF-8: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let yaml_value: serde_yaml::Value = match serde_yaml::from_str(yaml_text) { - Ok(v) => v, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("YAML parse error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let new_config = - match crate::storage_engines::types::StreamingConfig::from_yaml_data(&yaml_value) { - Ok(c) => c, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("StreamingConfig build error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - - let new_ids: HashSet = new_config - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let added: Vec = new_ids.difference(&old_ids).copied().collect(); - let removed: Vec = old_ids.difference(&new_ids).copied().collect(); - - if !removed.is_empty() { - warn!( - "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ - precompute worker groups for these ids will continue with their \ - construction-time config until they close naturally (phase 1 \ - limitation; see StreamingConfigHandle module doc)", - removed - ); - } - - // Schema retirement final cut: the sid catalog is the only - // lifecycle registry. The legacy per-`agg_id` `SchemaRegistry` is - // gone, so the swap handler now drives a single sid-level - // reconcile (`reconcile_from_streaming_config`) which force-retires - // any sid whose content signature no longer appears in the new - // config. There is no "added" set: sids are minted lazily at the - // first ingest write under the new config (see - // `SketchStore::ingest_precompute_for_agg_config`). - let snap = handle.snapshot(); - let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.summary_store.as_ref(), - snap.as_ref(), - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - - let body = serde_json::json!({ - "status": "success", - "agg_ids_added": added, - "agg_ids_removed": removed, - "new_aggregation_count": new_ids.len(), - "sids_retired": sid_summary.retired}); - (StatusCode::OK, axum::Json(body)).into_response() + (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ + "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" + }))).into_response() } pub use asap_types::plan_publication::PhysicalPlanInstallRequest; @@ -5772,7 +5406,11 @@ pub fn validate_and_build_runtime_plan( } } } - let runtime_materializations = request + asap_types::plan_publication::validate_stored_output_references( + &request.precompute_plan, + &request.query_plan, + )?; + let _runtime_materializations = request .precompute_plan .runtime_materializations() .map_err(|error| format!("PrecomputePlan validation error: {error}"))?; @@ -5787,8 +5425,10 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = - crate::storage_engines::types::StreamingConfig::new(runtime_materializations); + let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; let typed_fps: BTreeSet<_> = streaming_config .materializations_by_policy_fingerprint .keys() @@ -5799,6 +5439,10 @@ pub fn validate_and_build_runtime_plan( .query_plan .validate(&typed_fps) .map_err(|error| format!("QueryPlan validation error: {error}"))?; + asap_types::plan_publication::validate_maintenance_query_bindings( + &request.precompute_plan, + &request.query_plan, + )?; let storage_routing = match request.storage_routing.as_ref() { Some(value) => Arc::new( crate::storage_engines::types::BackendStorageRouting::from_json_payload(value) @@ -6046,12 +5690,30 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon .producers .iter() .map(|producer| { - ( - asap_types::sds::SummaryDefinitionId::from(producer.materialization), - producer.producer_id.clone(), - ) + let definition = asap_types::sds::StoredOutputId::from(producer.materialization); + active + .precompute_plan + .schemas + .iter() + .find(|schema| schema.materialization == definition) + .map(|schema| { + ( + definition, + ( + schema.stored_output_reference.stored_output_id, + producer.producer_id.clone(), + ), + ) + }) }) - .collect(); + .collect::>>(); + let Some(producers) = producers else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"status":"error","error":"precompute producer has no stored-output binding"})), + ) + .into_response(); + }; let reporter = std::env::var("HOSTNAME").unwrap_or_else(|_| "asapquery-backend".into()); match state.summary_store.observed_summary_inventory( &reporter, @@ -6841,6 +6503,48 @@ mod catalog_install_tests { ) } + #[test] + fn complete_dag_cannot_be_installed_as_maintenance() { + let mut request = request(); + let query_id = request + .precompute_plan + .executable_dags + .iter() + .next() + .map(|(id, _)| id.clone()) + .expect("fixture has a maintained summary"); + request + .precompute_plan + .executable_dags + .get_mut(&query_id) + .unwrap() + .document = request.query_plan.selected_dags[&query_id].clone(); + assert!(install(request) + .unwrap_err() + .contains("unsupported maintenance DAG")); + } + + #[test] + fn selected_dag_identity_is_validated_even_without_using_its_projection() { + let mut request = request(); + let query_id = request + .query_plan + .selected_dags + .keys() + .next() + .cloned() + .expect("fixture has selected DAG provenance"); + request + .query_plan + .selected_dags + .get_mut(&query_id) + .unwrap() + .query_id = "different-query".into(); + assert!(install(request) + .unwrap_err() + .contains("differs from document query ID")); + } + #[test] fn invalid_clickhouse_entry_cannot_change_active_generation() { let active = install(request()).expect("baseline plan installs"); @@ -6949,7 +6653,11 @@ mod catalog_install_tests { }) .expect("demo has maintained summaries"); binding.window_ms += 1; - assert!(install(request).unwrap_err().contains("pane duration")); + let error = install(request).unwrap_err(); + assert!( + error.contains("query") && error.contains("pane") && error.contains("differs"), + "{error}" + ); } #[test] diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2721bc68c..8c3ac483c 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -37,13 +37,13 @@ pub mod utils; // Re-export commonly used types to avoid glob import conflicts pub use storage_engines::types::{ - AggregateCore, AggregationConfig, KeyByLabelValues, Measurement, MergeableAccumulator, - MultipleSubpopulationAggregate, PrecomputedOutput, SerializableToSink, - SingleSubpopulationAggregate, + AggregateCore, KeyByLabelValues, Measurement, MergeableAccumulator, + MultipleSubpopulationAggregate, PrecomputeMaterialization, PrecomputedOutput, + SerializableToSink, SingleSubpopulationAggregate, }; pub use precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; pub use storage_engines::StoreResult; diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 91c109e81..1448cc337 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -52,7 +52,7 @@ struct Args { /// Versioned canonical QueryWorkload + DataWorkload and backend-local /// implementation evidence. The ASAPQuery profile invokes the pinned - /// Planner and PhysicalPlanCompiler at startup when this is supplied. + /// Planner and DeploymentPlanCompiler at startup when this is supplied. #[arg(long)] planning_snapshot: Option, @@ -746,11 +746,12 @@ async fn main() -> Result<()> { capability_snapshot_id: "bootstrap".into(), }, ingest: asap_types::precompute_plan::IngestContract { + dataset_identity: None, protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: false, - require_summary_definition_identity: false, + require_stored_output_identity: false, require_registered_producer: false, }, schemas: Vec::new(), diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 43a779fc6..3346f6059 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,30 +1,19 @@ use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, - HydraKllSketchAccumulator, IncreaseAccumulator, MaxAccumulator, MinAccumulator, - MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, - MultipleSumAccumulator, SumAccumulator, + HydraKllSketchAccumulator, IncreaseAccumulator, KeyedCounterState, KeyedMaxState, + KeyedMinState, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::{ AggregateCore, AggregationType, KeyByLabelValues, Measurement, }; -use asap_types::aggregation_config::AggregationConfig; -// Step 5 (sketch-identity unification, see -// scratchpad/artifacts/enum-unification-plan.md): dispatch below is -// driven by `AccumulatorSpec` (SummaryFamilyType + typed family parameters + -// keyed-axis grouping) instead of raw `AggregationType` + -// `aggregation_sub_type` string matching. Numeric params come straight -// off the committed family's typed params (no HashMap lookups) except -// `cms_params`, kept as a raw-`parameters` read for the one case Planner's -// family parameters have no field for: HydraKLL's `(row, col)` tiling grid (see -// `asap_types::accumulator_spec`'s module doc for why). `cms_params` -// now lives there — the only place that still needs the other three -// former local helpers (`kll_k_param`, `heap_size_param`, -// `ddsketch_alpha_param`) is that module's own `AccumulatorSpec` -// construction, so they aren't re-imported here. +use asap_types::aggregation_config::PrecomputeMaterialization; +// Production dispatch consumes Planner SummaryAgg payloads directly. The +// config adapter below is compiled only for isolated historical kernel tests. use super::operators::hll_sketch_accumulator::HllSketchAccumulator; use super::operators::univmon_accumulator::UnivMonAccumulator; -use asap_types::accumulator_spec::{cms_params, AccumulatorSpecError}; +#[cfg(test)] +use asap_types::accumulator_spec::cms_params; use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; /// Generate the two boilerplate clone-based `AccumulatorUpdater` methods @@ -379,28 +368,32 @@ impl AccumulatorUpdater for DDSketchAccumulatorUpdater { } // --------------------------------------------------------------------------- -// MultipleSumAccumulatorUpdater +// KeyedSumCountAccumulatorUpdater // --------------------------------------------------------------------------- -pub struct MultipleSumAccumulatorUpdater { - acc: MultipleSumAccumulator, +pub struct KeyedSumCountAccumulatorUpdater { + acc: KeyedSumCountAccumulator, } -impl MultipleSumAccumulatorUpdater { +impl KeyedSumCountAccumulatorUpdater { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { Self { - acc: MultipleSumAccumulator::new(), + acc: KeyedSumCountAccumulator::for_family(family), } } } -impl Default for MultipleSumAccumulatorUpdater { +impl Default for KeyedSumCountAccumulatorUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { +impl AccumulatorUpdater for KeyedSumCountAccumulatorUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -415,7 +408,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleSumAccumulator::new(); + self.acc = KeyedSumCountAccumulator::for_family(self.acc.family.clone()); } fn is_keyed(&self) -> bool { @@ -423,13 +416,13 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() - + self.acc.sums.len() * (std::mem::size_of::() + 8) + std::mem::size_of::() + + self.acc.sums.len() * (std::mem::size_of::() + 16) } } // --------------------------------------------------------------------------- -// MultipleMinAccumulatorUpdater / MultipleMaxAccumulatorUpdater +// KeyedMinStateUpdater / KeyedMaxStateUpdater // --------------------------------------------------------------------------- macro_rules! multiple_extremum_updater { @@ -475,32 +468,32 @@ macro_rules! multiple_extremum_updater { }; } -multiple_extremum_updater!(MultipleMinAccumulatorUpdater, MultipleMinAccumulator); -multiple_extremum_updater!(MultipleMaxAccumulatorUpdater, MultipleMaxAccumulator); +multiple_extremum_updater!(KeyedMinStateUpdater, KeyedMinState); +multiple_extremum_updater!(KeyedMaxStateUpdater, KeyedMaxState); // --------------------------------------------------------------------------- -// MultipleIncreaseAccumulatorUpdater +// KeyedCounterStateUpdater // --------------------------------------------------------------------------- -pub struct MultipleIncreaseAccumulatorUpdater { - acc: MultipleIncreaseAccumulator, +pub struct KeyedCounterStateUpdater { + acc: KeyedCounterState, } -impl MultipleIncreaseAccumulatorUpdater { +impl KeyedCounterStateUpdater { pub fn new() -> Self { Self { - acc: MultipleIncreaseAccumulator::new(), + acc: KeyedCounterState::new(), } } } -impl Default for MultipleIncreaseAccumulatorUpdater { +impl Default for KeyedCounterStateUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { +impl AccumulatorUpdater for KeyedCounterStateUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -528,7 +521,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleIncreaseAccumulator::new(); + self.acc = KeyedCounterState::new(); } fn is_keyed(&self) -> bool { @@ -536,7 +529,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() + std::mem::size_of::() + self.acc.increases.len() * (std::mem::size_of::() + std::mem::size_of::()) @@ -899,27 +892,20 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &AggregationConfig) -> bool { - matches!( - config.aggregation_type, - AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax - | AggregationType::CountMinSketch - | AggregationType::CountMinSketchWithHeap - | AggregationType::CountSketch - | AggregationType::CountSketchWithHeap - | AggregationType::HydraKLL - ) +pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { + config + .accumulator_spec() + .expect("valid fixture") + .grouping + .is_some() } /// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. /// /// * `value` / `sum`: sum values per key (default). /// * `count` / `frequency` / `freq`: count occurrences per key. -fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { +#[cfg(test)] +fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { match config.sample_update_rule() { asap_types::SampleUpdateRule::Count => TopkWeight::Count, asap_types::SampleUpdateRule::Value { .. } @@ -927,7 +913,8 @@ fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { } } -fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { +#[cfg(test)] +fn topk_weight_scale_param(config: &PrecomputeMaterialization) -> f64 { match config.sample_update_rule() { asap_types::SampleUpdateRule::Value { scale } => scale, asap_types::SampleUpdateRule::CounterDelta { scale } => scale, @@ -943,6 +930,7 @@ fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { /// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with /// `SketchParams::Kll`, so the /// other arm is unreachable from a `spec` this module builds itself. +#[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { // Lossless: `accumulator_spec()` only ever stores a value that @@ -955,12 +943,12 @@ fn kll_k(params: &SketchParams) -> u16 { } } -/// Read `(width, depth)` out of `SketchParams::Cms` or `::CountSketch` +/// Read `(rows = depth, columns = width)` out of `SketchParams::Cms` or `::CountSketch` /// — same shape, different variant per bare-sketch identity. fn cms_dims(params: &SketchParams) -> (usize, usize) { match params { SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { - (*width as usize, *depth as usize) + (*depth as usize, *width as usize) } other => unreachable!( "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" @@ -968,7 +956,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { } } -/// Read `(width, depth, heap_size)` out of `SketchParams::CmsWithHeap` +/// Read `(rows = depth, columns = width, heap_size)` out of `SketchParams::CmsWithHeap` /// or `::CountSketchWithHeap`. fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { match params { @@ -981,7 +969,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { width, depth, heap_size, - } => (*width as usize, *depth as usize, *heap_size as usize), + } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), @@ -989,6 +977,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { } /// Read the DDSketch relative-accuracy `alpha` out of `SketchParams::DDSketch`. +#[cfg(test)] fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, @@ -998,54 +987,28 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { } } -/// Create an appropriate `AccumulatorUpdater` from an `AggregationConfig`. -/// -/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryFamilyType` identity -/// plus the keyed/unkeyed `grouping` axis — instead of the pre-Step-5 -/// `AggregationType` + `aggregation_sub_type` string combo. See -/// `asap_types::accumulator_spec`'s module doc for why min/max direction, -/// HydraKLL's `(row, col)` tiling, and top-k `weight_mode` still read -/// `config` directly rather than going through Planner family parameters — -/// none of those three have a field in the Planner-owned types. -pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let spec = match config.accumulator_spec() { - Ok(spec) => spec, - // Three fallback paths, preserved verbatim from the pre-Step-5 - // dispatch: same warning text, same default updater per case - // (Single- and MultipleSubpopulation default to *different* - // updaters — see `AccumulatorSpecError`'s doc). - Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown SingleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(SumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown MultipleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(MultipleSumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnmappedAggregationType(other)) => { - tracing::warn!( - "Unknown aggregation_type '{:?}', defaulting to SingleSubpopulation Sum", - other - ); - return Box::new(SumAccumulatorUpdater::new()); - } - }; +/// Construct isolated payload fixtures for kernel/storage unit tests. +/// Production execution requires a validated Planner DAG program. +#[cfg(test)] +pub fn create_fixture_accumulator( + config: &PrecomputeMaterialization, +) -> Box { + let spec = config + .accumulator_spec() + .expect("invalid isolated kernel fixture"); let keyed = spec.grouping.is_some(); match (&spec.family, keyed) { - (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), false) => { Box::new(SumAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { - Box::new(MultipleSumAccumulatorUpdater::new()) + Box::new(KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Sum)) } + (SummaryFamilyType::ExactAggregate(ExactKind::Count, _), true) => Box::new( + KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Count), + ), // Direction comes off the family itself now. It used to be read // back out of `aggregation_sub_type` because Planner had one @@ -1056,20 +1019,20 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - Box::new(MultipleMinAccumulatorUpdater::new()) + Box::new(KeyedMinStateUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), false) => { Box::new(MaxAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), true) => { - Box::new(MultipleMaxAccumulatorUpdater::new()) + Box::new(KeyedMaxStateUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), false) => { Box::new(IncreaseAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { - Box::new(MultipleIncreaseAccumulatorUpdater::new()) + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), true) => { + Box::new(KeyedCounterStateUpdater::new()) } (SummaryFamilyType::Sketch(kind, _), false) @@ -1187,14 +1150,8 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - tracing::warn!( - "SummaryFamilyType {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", - other_family, - keyed - ); - Box::new(SumAccumulatorUpdater::new()) + panic!("unsupported isolated kernel fixture {other_family:?}, keyed={keyed}") } } } @@ -1271,7 +1228,7 @@ mod tests { #[test] fn hll_and_univmon_raw_updates_share_value_identity() { for family in [AggregationType::HLL, AggregationType::UnivMon] { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( family, String::new(), Default::default(), @@ -1288,7 +1245,7 @@ mod tests { None, None, ); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); for value in [0.0, -0.0, 2.0, 2.0, f64::NAN] { updater.update_single(value, 1000); } @@ -1362,7 +1319,7 @@ mod tests { #[test] fn test_multiple_sum_updater() { - let mut updater = MultipleSumAccumulatorUpdater::new(); + let mut updater = KeyedSumCountAccumulatorUpdater::new(); assert!(updater.is_keyed()); let key_a = KeyByLabelValues::new_with_labels(vec!["a".to_string()]); @@ -1372,7 +1329,7 @@ mod tests { updater.update_keyed(&key_b, 2.0, 2000); let acc = updater.take_accumulator(); - assert_eq!(acc.type_name(), "MultipleSumAccumulator"); + assert_eq!(acc.type_name(), "KeyedSumCountAccumulator"); } #[test] @@ -1424,7 +1381,7 @@ mod tests { use std::collections::HashMap; let make_config = |agg_type: AggregationType, sub_type: &str| { - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), HashMap::new(), @@ -1463,18 +1420,15 @@ mod tests { AggregationType::MultipleSubpopulation, "Sum" ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleSum, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleIncrease, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleMax, - "" - ))); + let mut keyed = make_config(AggregationType::Sum, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Increase, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Max, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); assert!(config_is_keyed(&make_config( AggregationType::CountMinSketch, "" @@ -1497,12 +1451,12 @@ mod tests { for (agg_type, sub_type) in &[ (AggregationType::SingleSubpopulation, "Sum"), (AggregationType::MultipleSubpopulation, "Sum"), - (AggregationType::MultipleSum, ""), + (AggregationType::Sum, ""), (AggregationType::DatasketchesKLL, ""), (AggregationType::CountMinSketch, ""), ] { let config = make_config(*agg_type, sub_type); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); assert_eq!( config_is_keyed(&config), updater.is_keyed(), @@ -1518,7 +1472,7 @@ mod tests { use std::collections::HashMap; let mut params = HashMap::new(); params.insert("K".to_string(), serde_json::Value::from(50_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "DatasketchesKLL".to_string(), params, @@ -1535,7 +1489,7 @@ mod tests { None, None, ); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); let acc = updater.snapshot_accumulator(); let kll = acc .as_any() @@ -1555,7 +1509,7 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), serde_json::Value::from(7_u64)); params.insert("w".to_string(), serde_json::Value::from(2048_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), params, @@ -1575,7 +1529,7 @@ mod tests { assert_eq!(super::cms_params(&config), (7, 2048)); // Empty params — defaults `(4, 1000)`. - let empty_config = AggregationConfig::new( + let empty_config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), HashMap::new(), @@ -1601,7 +1555,10 @@ mod tests { /// Build a `*WithHeap` config keyed by group-by label `host`, with the /// given `weight_mode` param (None → default = value-weighted). - fn topk_config(agg_type: AggregationType, weight_mode: Option<&str>) -> AggregationConfig { + fn topk_config( + agg_type: AggregationType, + weight_mode: Option<&str>, + ) -> PrecomputeMaterialization { use std::collections::HashMap; let mut params = HashMap::new(); // Small, deterministic geometry; heap big enough to hold all hosts. @@ -1611,7 +1568,7 @@ mod tests { if let Some(m) = weight_mode { params.insert("weight_mode".to_string(), serde_json::Value::from(m)); } - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -1699,7 +1656,7 @@ mod tests { fn value_weighted_topk_ranks_hosts_by_sum_of_value() { // DEFAULT mode (no weight_mode param) must be value-weighted. let config = topk_config(AggregationType::CountMinSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); assert!(updater.is_keyed()); feed_stream(&mut *updater); @@ -1732,7 +1689,7 @@ mod tests { config .parameters .insert("weight_scale".into(), serde_json::json!(1_000_000)); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); updater.update_keyed(&host_key("payment"), 0.004, 1_000); updater.update_keyed(&host_key("order"), 0.002, 1_000); let ranked = ranked_topk(&*updater.take_accumulator()); @@ -1744,7 +1701,7 @@ mod tests { fn count_weighted_topk_still_ranks_by_occurrence_frequency() { // Opt-in frequency-top-k: weight_mode=count must rank by event count. let config = topk_config(AggregationType::CountMinSketchWithHeap, Some("count")); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); @@ -1764,7 +1721,7 @@ mod tests { // (real median-of-signed-rows math) — same value-weighted default // as the CMS-family heap path, but no longer conflated with it. let config = topk_config(AggregationType::CountSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); assert_eq!(acc.type_name(), "CountSketchWithHeapAccumulator"); @@ -1800,3 +1757,269 @@ mod tests { } } } + +#[cfg(test)] +mod planner_family_regression { + use super::*; + use asap_types::{enums::WindowKind, KeyByLabelNames}; + + // Every installed exact producer must retain its family in runtime state. + #[test] + fn exact_state_identity_survives_factory_and_reset() { + for kind in [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Rate, + AggregationType::Increase, + AggregationType::Min, + AggregationType::Max, + ] { + let config = PrecomputeMaterialization::new( + kind, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "metric".into(), + None, + None, + None, + ); + let mut updater = create_planner_accumulator( + &config.accumulator_spec().unwrap().family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &Default::default(), + ) + .unwrap(); + updater.update_single(4.0, 1000); + updater.update_single(7.0, 2000); + assert_eq!(updater.take_accumulator().get_accumulator_type(), kind); + assert_eq!(updater.snapshot_accumulator().get_accumulator_type(), kind); + } + } +} + +/// Construct the kernel declared by a Planner SummaryAgg. No backend config +/// tags participate in this dispatch and unsupported payloads are errors. +pub fn create_planner_accumulator( + family: &SummaryFamilyType, + input: &planner_types::post_asap::SummaryUpdate, + grouping: &planner_types::post_asap::GroupingStrategy, +) -> Result, String> { + use planner_types::post_asap::GroupingStrategy; + if grouping != &GroupingStrategy::PerSubpopulationInstance { + return Err("shared summary grouping requires a supported Planner Hydra kernel".into()); + } + if matches!(family, SummaryFamilyType::ExactAggregate(..)) { + return Ok(Box::new(PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator::new( + family.clone(), + input.item.is_some(), + )?, + })); + } + let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + return Err(format!("unsupported Planner summary family {family:?}")); + }; + if family_grouping != grouping { + return Err("Planner family and operator grouping disagree".into()); + } + let weight_scale = 1.0; + let updater: Box = match (kind.algorithm(), kind.params()) { + (SketchAlgorithm::Kll, SketchParams::Kll { k }) => Box::new(KllAccumulatorUpdater::new( + u16::try_from(*k).map_err(|_| "KLL k exceeds runtime bound")?, + )), + (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Box::new(DDSketchAccumulatorUpdater::new(*alpha)) + } + (SketchAlgorithm::Cms, params @ SketchParams::Cms { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CmsAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CountSketch, params @ SketchParams::CountSketch { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CountSketchAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CmsWithHeap, params @ SketchParams::CmsWithHeap { .. }) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CmsHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + ( + SketchAlgorithm::CountSketchWithHeap, + params @ SketchParams::CountSketchWithHeap { .. }, + ) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CountSketchWithHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => Box::new(HllUpdater { + acc: HllSketchAccumulator::new( + asap_sketchlib::HllVariant::Regular, + u32::from(*precision), + ), + }), + ( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + }, + ) => Box::new(UnivMonUpdater { + acc: UnivMonAccumulator::new( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + *layers as usize, + ) + .map_err(|e| e.to_string())?, + }), + _ => { + return Err(format!( + "unsupported Planner algorithm/parameters: {kind:?}" + )) + } + }; + if updater.is_keyed() != input.item.is_some() + && !asap_types::accumulator_spec::is_unit_sample_frequency(input) + { + return Err("Planner item expression does not match the selected kernel layout".into()); + } + Ok(updater) +} + +struct PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator, +} +impl AccumulatorUpdater for PlannerExactUpdater { + fn update_single(&mut self, value: f64, timestamp: i64) { + self.acc.update(None, value, timestamp); + } + fn update_keyed(&mut self, key: &KeyByLabelValues, value: f64, timestamp: i64) { + self.acc.update(Some(key), value, timestamp); + } + impl_clone_accumulator_methods!(acc); + fn reset(&mut self) { + self.acc = super::operators::exact_accumulator::ExactAccumulator::new( + self.acc.family().clone(), + self.acc.is_keyed(), + ) + .expect("installed exact family"); + } + fn is_keyed(&self) -> bool { + self.acc.is_keyed() + } + fn memory_usage_bytes(&self) -> usize { + self.acc.approx_memory_bytes() + } +} + +#[cfg(test)] +mod planner_parameter_regression { + use super::*; + use planner_types::post_asap::{SketchKind, SummaryInputExpr, SummaryUpdate}; + + // Planner width is the bucket count; depth is the independent hash-row count. + #[test] + fn planner_sketch_dimensions_are_not_transposed() { + for (algorithm, params) in [ + ( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ] { + let family = SummaryFamilyType::Sketch( + SketchKind::new(algorithm.clone(), params), + Default::default(), + ); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + }; + let state = create_planner_accumulator(&family, &update, &Default::default()) + .unwrap() + .snapshot_accumulator(); + let dims = match algorithm { + SketchAlgorithm::Cms => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketch => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows, s.inner.cols) + } + SketchAlgorithm::CmsWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketchWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + _ => unreachable!(), + }; + assert_eq!(dims, (3, 128), "{algorithm:?}"); + } + } +} diff --git a/data_plane/src/precompute_engine/coordination_checkpoint.rs b/data_plane/src/precompute_engine/coordination_checkpoint.rs index 249fad538..873725505 100644 --- a/data_plane/src/precompute_engine/coordination_checkpoint.rs +++ b/data_plane/src/precompute_engine/coordination_checkpoint.rs @@ -377,7 +377,7 @@ fn invalid(message: impl Into) -> io::Error { #[cfg(test)] mod tests { use super::*; - use asap_types::{sds::HalfOpenTimeRange, sds::SummaryDefinitionId, PolicyFingerprint}; + use asap_types::{sds::HalfOpenTimeRange, sds::StoredOutputId, PolicyFingerprint}; use std::collections::BTreeMap; fn generation() -> CatalogGeneration { @@ -399,7 +399,7 @@ mod tests { fn coordinates() -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 10, diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 47593276b..e97045fe5 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -21,7 +21,7 @@ struct Population { struct Observations { generation: Option, populations: BTreeMap, - extent: BTreeMap, + extent: BTreeMap, total_keys: usize, metadata_bytes: usize, invalid: Option, @@ -56,7 +56,7 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::AggregationConfig, + config: &asap_types::PrecomputeMaterialization, timestamp_ms: i64, value: f64, ) { @@ -97,7 +97,7 @@ impl RuntimeErpObserver { } let extent = state .extent - .entry(coordinates.summary_definition_id) + .entry(coordinates.stored_output_id) .or_insert((timestamp_ms, timestamp_ms)); extent.0 = extent.0.min(timestamp_ms); extent.1 = extent.1.max(timestamp_ms); @@ -133,7 +133,7 @@ impl RuntimeErpObserver { let population = state.populations.entry(id).or_insert_with(|| Population { source: format!( "summary-definition:{}", - coordinates.summary_definition_id.as_u64() + coordinates.stored_output_id.as_u64() ), coordinates, semantics, @@ -178,7 +178,7 @@ impl RuntimeErpObserver { return Ok(()); } let mut groups: BTreeMap< - (SummaryDefinitionId, i64, i64), + (StoredOutputId, i64, i64), ( String, String, @@ -188,14 +188,14 @@ impl RuntimeErpObserver { > = BTreeMap::new(); for (id, population) in &state.populations { let c = &population.coordinates; - let Some((first, last)) = state.extent.get(&c.summary_definition_id) else { + let Some((first, last)) = state.extent.get(&c.stored_output_id) else { continue; }; if c.time_range.start_ms < *first || c.time_range.end_ms > *last { continue; } let key = ( - c.summary_definition_id, + c.stored_output_id, c.time_range.start_ms, c.time_range.end_ms, ); @@ -207,7 +207,7 @@ impl RuntimeErpObserver { ErpPopulationObservations { schema_version: 1, catalog_generation: generation.clone(), - summary_definition_id: c.summary_definition_id, + stored_output_id: c.stored_output_id, observed_at_unix_ms: now_ms, window_start_ms: c.time_range.start_ms, window_end_ms: c.time_range.end_ms, @@ -264,8 +264,8 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { - let config = asap_types::AggregationConfig::new( + fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { + let config = asap_types::PrecomputeMaterialization::new( asap_types::AggregationType::HLL, String::new(), Default::default(), @@ -294,7 +294,7 @@ mod tests { } fn coordinate(group: usize) -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: asap_types::PolicyFingerprint(1).into(), + stored_output_id: asap_types::PolicyFingerprint(1).into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index 7f5e0a847..da79aad47 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -45,7 +45,7 @@ pub enum FrameLineageError { struct FrameLineageKey { plan_id: u64, plan_version: u64, - materialization: asap_types::sds::SummaryDefinitionId, + materialization: asap_types::sds::StoredOutputId, series_identity: String, producer_id: String, producer_epoch: String, diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index d975eb6a9..67940478b 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,7 +1,7 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::StreamingConfigHandle; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,11 +28,11 @@ pub struct IngestObservability { /// A full / delta frame failed to decode (or a delta failed to /// apply) and was dropped. pub dropped_decode_fail: AtomicU64, - /// A decoded sketch matched no `AggregationConfig` in the running + /// A decoded sketch matched no `PrecomputeMaterialization` in the running /// streaming config (legacy routing-side bucketing miss). pub dropped_unconfigured: AtomicU64, /// The output sink could not resolve a `policy_fp` to an - /// `AggregationConfig` (registry miss) and skipped the write. + /// `PrecomputeMaterialization` (registry miss) and skipped the write. pub dropped_policy_miss: AtomicU64, /// RES-1 — max number of distinct tumbling windows a per-series /// snapshot base may lag behind the newest observed `window_start` @@ -120,7 +120,7 @@ pub struct IngestState { pub samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64, /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. - /// This replaces the old frozen `Vec>`. + /// This replaces the old frozen `Vec>`. pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, @@ -157,7 +157,7 @@ impl IngestState { /// visible immediately without restart. /// /// Returns the shared `Arc` — no cloning of - /// individual AggregationConfig objects, just an atomic refcount + /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() @@ -247,7 +247,7 @@ impl IngestState { /// ingest sources (e.g. OTLP) can reuse it. pub fn extract_group_key_for( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { extract_group_key(series_key, config) } @@ -261,7 +261,7 @@ impl IngestState { /// [`Self::extract_group_key_for`] does after the round-trip. pub fn extract_group_key_from_labels( labels: &std::collections::HashMap, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map( |name| { @@ -278,7 +278,7 @@ impl IngestState { /// for a given series key and aggregation config. fn extract_group_key( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { let labels = parse_labels_from_series_key(series_key); crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map(|name| { @@ -294,18 +294,18 @@ mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Kept as a parameter to // avoid churning the call sites below. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), std::collections::HashMap::new(), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index e8f416e55..683ed5c06 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -70,7 +70,7 @@ type PendingOutput = ( enum MaintenanceInputs<'a> { Live { - definition: asap_types::sds::SummaryDefinitionId, + definition: asap_types::sds::StoredOutputId, state: SummaryState, }, Frozen(&'a [crate::storage_engines::sketch_db::index::FrozenExactWindows]), @@ -91,7 +91,7 @@ impl MaintenanceInputs<'_> { fn frozen_population_value( inputs: &[crate::storage_engines::sketch_db::index::FrozenExactWindows], - definition: asap_types::sds::SummaryDefinitionId, + definition: asap_types::sds::StoredOutputId, family: Option, complete: bool, ) -> Result, String> { @@ -122,7 +122,7 @@ fn frozen_population_value( struct OperatorAdapter<'a> { binding: &'a BackendExecutableBinding, inputs: MaintenanceInputs<'a>, - configs: &'a [asap_types::aggregation_config::AggregationConfig], + configs: &'a [asap_types::aggregation_config::PrecomputeMaterialization], } impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { @@ -133,7 +133,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { node: &ExecutableDagNode, ) -> Result, String> { let definition = match self.binding.node(node.id) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Ok(None), }; let family = node.output_schema.fields.iter().find_map(|field| { @@ -166,15 +166,15 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { node: &ExecutableDagNode, inputs: &[Arc], ) -> Result { + if node.output_state.timing != planner_types::post_asap::ExecutionTiming::IngestionTime { + return Err("maintenance runtime requires ingestion-time nodes".into()); + } match &node.payload { ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs), - ExecutableOperatorPayload::Binary { - operator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, - } => { + ExecutableOperatorPayload::Binary { operator } => { if !self.inputs.frozen_inputs().is_some() || node.output_state - != planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS + != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS { return Err("maintenance binary requires immutable completed row inputs".into()); } @@ -183,7 +183,6 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { ExecutableOperatorPayload::Value { operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, } => { if !self.inputs.frozen_inputs().is_some() { return Err( @@ -193,7 +192,12 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } finalize_exact(node, inputs) } - ExecutableOperatorPayload::SummaryAgg { family, input, .. } => { + ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } => { let [value] = inputs else { return Err("maintenance SummaryAgg requires exactly one row input".into()); }; @@ -206,9 +210,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { ); } let target = match self.binding.node(node.id) { - Some(BackendNodeBinding::Materialization { summary_definition }) => { - summary_definition - } + Some(BackendNodeBinding::Materialization { stored_output }) => stored_output, _ => { return Err( "maintenance SummaryAgg lacks installed materialization binding".into(), @@ -251,7 +253,9 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = super::accumulator_factory::create_accumulator_updater(config); + let mut updater = super::accumulator_factory::create_planner_accumulator( + family, input, grouping, + )?; if updater.is_keyed() { return Err("keyed maintenance accumulator requires an item expression".into()); } @@ -304,8 +308,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { }) } payload => Err(format!( - "maintenance operator {:?} has no summary-state implementation", - payload + "maintenance operator {payload:?} has no summary-state implementation" )), } } @@ -706,7 +709,7 @@ fn prepare_frozen_maintenance_sink( > { installed.validate()?; let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("immutable sink lacks a materialization binding".into()), }; let config = configs @@ -745,10 +748,10 @@ fn prepare_frozen_maintenance_sink( .nodes .iter() .filter_map(|(node, binding)| match binding { - BackendNodeBinding::Materialization { summary_definition } - if expected_input.inputs.contains(summary_definition) => + BackendNodeBinding::Materialization { stored_output } + if expected_input.inputs.contains(stored_output) => { - Some((*node, *summary_definition)) + Some((*node, *stored_output)) } _ => None, }) @@ -769,7 +772,7 @@ fn prepare_frozen_maintenance_sink( let key = MaterializationCommitKey { plan_id: generation.plan_id, plan_version: generation.plan_version, - summary_definition: target, + stored_output: target, window_start_ms: i64::try_from(output_window.0) .map_err(|_| "output window exceeds timestamp range")?, window_end_ms: i64::try_from(output_window.1) @@ -860,7 +863,7 @@ pub fn execute_completed_maintenance( group: &BTreeMap, ) -> Result { let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("maintenance sink lacks an installed output identity".into()), }; let derived = configs @@ -896,14 +899,14 @@ pub(crate) fn execute_completed_maintenance_cohort( installed: &asap_types::executable_plan::InstalledPostAsapDag, configs: &[asap_types::PrecomputeMaterialization], sink: PostAsapNodeId, - source_sids: &BTreeMap, + source_sids: &BTreeMap, target_sid: u64, window: (u64, u64), group: &BTreeMap, ) -> Result { use asap_types::executable_plan::BackendNodeBinding; let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("maintenance sink lacks an installed output identity".into()), }; let target_config = configs @@ -1079,7 +1082,7 @@ fn execute_finite_source_cohort( .as_ref() .ok_or("finite maintenance requires a catalog generation")?; let Some(BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }) = installed.binding.node(sink) else { return Err("finite maintenance sink has no installed definition".into()); @@ -1239,7 +1242,7 @@ fn execute_finite_complete_populations( generation: &Arc, ) -> Result<(), String> { let target = match installed.binding.node(sink) { - Some(BackendNodeBinding::Materialization { summary_definition }) => *summary_definition, + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("complete maintenance target is not bound".into()), }; let config = plan @@ -1426,7 +1429,7 @@ pub(crate) fn execute_finite_maintenance( for installed in plan.executable_dags.values() { for sink in &installed.binding.precompute_sinks { let Some(BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }) = installed.binding.node(*sink) else { continue; @@ -1574,7 +1577,7 @@ struct CommittedState { struct CommitRegistryState { generation: Option<(u64, u64)>, entries: BTreeMap, - frontiers: BTreeMap, + frontiers: BTreeMap, pending_batch: Option<[u8; 32]>, batch_has_published: bool, admitted_keys: BTreeSet, @@ -1591,7 +1594,7 @@ impl CommitRegistryState { if !self.admitted_keys.contains(key) && self .frontiers - .get(&key.summary_definition) + .get(&key.stored_output) .is_some_and(|(latest, horizon)| { key.window_end_ms <= latest.saturating_sub(i64::try_from(*horizon).unwrap_or(i64::MAX)) @@ -1681,7 +1684,7 @@ impl CommitRegistry { } let frontier = state .frontiers - .entry(key.summary_definition) + .entry(key.stored_output) .or_insert((key.window_end_ms, *horizon)); frontier.0 = frontier.0.max(key.window_end_ms); frontier.1 = frontier.1.max(*horizon); @@ -1689,7 +1692,7 @@ impl CommitRegistry { let frontiers = state.frontiers.clone(); state.entries.retain(|key, _| { frontiers - .get(&key.summary_definition) + .get(&key.stored_output) .is_none_or(|(latest, horizon)| { key.window_end_ms > latest.saturating_sub(i64::try_from(*horizon).unwrap_or(i64::MAX)) @@ -1800,13 +1803,13 @@ impl MaintenanceDagSink { output: PrecomputedOutput, state: Box, ) -> Result, String> { - let source_definition: asap_types::sds::SummaryDefinitionId = output.policy_fp.into(); + let source_definition: asap_types::sds::StoredOutputId = output.policy_fp.into(); let source: SummaryState = Arc::from(state); let mut derived = Vec::new(); let mut matched = false; let mut lineage = Sha256::new(); lineage.update(b"asap-maintenance-lineage-v1"); - let definition_bytes = source_definition.0 .0.to_be_bytes(); + let definition_bytes = source_definition.0.to_be_bytes(); lineage.update(definition_bytes); if let Some(input) = &output.input_revision { if input.generation.plan_id != plan.plan_id() @@ -1835,7 +1838,7 @@ impl MaintenanceDagSink { .binding .nodes .iter() - .filter_map(|(id, binding)| matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source_definition).then_some(*id)) + .filter_map(|(id, binding)| matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == source_definition).then_some(*id)) .collect::>(); if source_nodes.is_empty() { continue; @@ -1852,9 +1855,9 @@ impl MaintenanceDagSink { // Derived summaries consume complete immutable windows at the // completion barrier, never additive worker fragments. if matches!(installed.binding.node(*sink_node), - Some(BackendNodeBinding::Materialization { summary_definition }) + Some(BackendNodeBinding::Materialization { stored_output }) if plan.precompute_plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() + config.policy_fingerprint() == stored_output.fingerprint() && config.derived_input.is_some())) { continue; @@ -1868,8 +1871,8 @@ impl MaintenanceDagSink { !has_input && matches!( installed.binding.node(*node), - Some(BackendNodeBinding::Materialization { summary_definition }) - if *summary_definition != source_definition + Some(BackendNodeBinding::Materialization { stored_output }) + if *stored_output != source_definition ) }); if foreign_source { @@ -1894,15 +1897,13 @@ impl MaintenanceDagSink { } matched = true; let target = match installed.binding.node(*sink_node) { - Some(BackendNodeBinding::Materialization { summary_definition }) => { - *summary_definition - } + Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("precompute sink lacks materialization binding".into()), }; let key = MaterializationCommitKey { plan_id: plan.plan_id(), plan_version: plan.plan_version(), - summary_definition: target, + stored_output: target, window_start_ms: output.start_timestamp as i64, window_end_ms: output.end_timestamp as i64, input_lineage: lineage.clone(), @@ -2110,21 +2111,21 @@ impl OutputSink for MaintenanceDagSink { /// semantic dependencies rather than assuming source and output identities match. pub(crate) fn affected_materializations( plan: &asap_types::precompute_plan::PrecomputePlan, - source: asap_types::sds::SummaryDefinitionId, -) -> BTreeSet { + source: asap_types::sds::StoredOutputId, +) -> BTreeSet { use asap_types::executable_plan::BackendNodeBinding; let mut affected = BTreeSet::from([source]); for installed in plan.executable_dags.values() { let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { - matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) + matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == source).then_some(*node) }).collect::>(); let mut frontier = reachable.iter().copied().collect::>(); while let Some(producer) = frontier.pop() { for edge in &installed.document.edges { let immutable = matches!(installed.binding.node(edge.consumer), - Some(BackendNodeBinding::Materialization { summary_definition }) + Some(BackendNodeBinding::Materialization { stored_output }) if plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() + config.policy_fingerprint() == stored_output.fingerprint() && config.derived_input.is_some())); if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { frontier.push(edge.consumer); @@ -2133,10 +2134,10 @@ pub(crate) fn affected_materializations( } for sink in &installed.binding.precompute_sinks { if reachable.contains(sink) { - if let Some(BackendNodeBinding::Materialization { summary_definition }) = + if let Some(BackendNodeBinding::Materialization { stored_output }) = installed.binding.nodes.get(sink) { - affected.insert(*summary_definition); + affected.insert(*stored_output); } } } @@ -2153,10 +2154,30 @@ mod tests { WindowEdgeCompatibility, }; - fn definition(value: u64) -> asap_types::sds::SummaryDefinitionId { + fn definition(value: u64) -> asap_types::sds::StoredOutputId { asap_types::PolicyFingerprint(value).into() } + fn maintenance_only( + mut dag: ExecutableDag, + mut binding: BackendExecutableBinding, + ) -> (ExecutableDag, BackendExecutableBinding) { + let retained = dag + .nodes + .iter() + .filter(|node| { + node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime + }) + .map(|node| node.id) + .collect::>(); + dag.nodes.retain(|node| retained.contains(&node.id)); + dag.edges + .retain(|edge| retained.contains(&edge.producer) && retained.contains(&edge.consumer)); + binding.nodes.retain(|id, _| retained.contains(id)); + dag.root = binding.precompute_sinks[0]; + (dag, binding) + } + #[test] fn cohort_lineage_is_order_independent_and_binds_every_input() { use crate::storage_engines::sketch_db::index::FrozenExactWindows; @@ -2223,7 +2244,7 @@ mod tests { ExecutableDagNode { id: PostAsapNodeId(id), payload: ExecutableOperatorPayload::SummaryMerge, - output_state: planner_types::post_asap::ExecutionDataState::MAINTENANCE_SUMMARY, + output_state: planner_types::post_asap::ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: vec![], time_index: None, @@ -2241,7 +2262,7 @@ mod tests { fields: vec![], time_index: None, }, - data_state: planner_types::post_asap::ExecutionDataState::MAINTENANCE_SUMMARY, + data_state: planner_types::post_asap::ExecutionDataState::INGESTION_SUMMARY, grouping: GroupingEdgeCompatibility::Identical, window: WindowEdgeCompatibility::NotApplicable, } @@ -2273,10 +2294,10 @@ mod tests { let binding = BackendExecutableBinding { nodes: [(1, definition(1)), (2, definition(2)), (3, definition(3))] .into_iter() - .map(|(id, summary_definition)| { + .map(|(id, stored_output)| { ( PostAsapNodeId(id), - BackendNodeBinding::Materialization { summary_definition }, + BackendNodeBinding::Materialization { stored_output }, ) }) .collect(), @@ -2379,14 +2400,14 @@ mod tests { ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: source_definition, + stored_output: source_definition, }, ), (PostAsapNodeId(2), BackendNodeBinding::MaintenanceInput), ( PostAsapNodeId(3), BackendNodeBinding::Materialization { - summary_definition: target, + stored_output: target, }, ), ]), @@ -2417,7 +2438,6 @@ mod tests { let mut read = node(2); read.payload = ExecutableOperatorPayload::Value { operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, }; read.output_schema.fields = vec![SummaryField { name: "value".into(), @@ -2462,14 +2482,14 @@ mod tests { dtype: SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), nullable: false, }]; - read.output_state = planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS; + read.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; aggregate.output_schema.fields = vec![SummaryField { name: "state".into(), dtype: configs[1].accumulator_spec().unwrap().family, nullable: false, }]; let mut query = node(4); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let mut first_edge = edge(1, 2); first_edge.intermediate_schema = source_node.output_schema.clone(); let mut second_edge = edge(2, 3); @@ -2486,7 +2506,7 @@ mod tests { scheduled_binding.nodes.insert( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: source_definition, + stored_output: source_definition, }, ); scheduled_binding @@ -2500,6 +2520,7 @@ mod tests { ); scheduled_binding.query_sink = PostAsapNodeId(4); scheduled_binding.query_plan_sink = control_plane::query_plan::QueryNodeId(4); + let (dag, scheduled_binding) = maintenance_only(dag, scheduled_binding); let scheduled_adapter = OperatorAdapter { binding: &scheduled_binding, ..adapter @@ -2507,7 +2528,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 1, plan_version: 1, - summary_definition: target, + stored_output: target, window_start_ms: 0, window_end_ms: 1000, input_lineage: vec![1], @@ -2535,7 +2556,8 @@ mod tests { persistence::config::SketchStorePersistenceConfig, SketchStore, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; - let document = OwnedPostAsapDag::from_executable("immutable-chain".into(), &dag).unwrap(); + let mut document = + OwnedPostAsapDag::from_executable("immutable-chain".into(), &dag).unwrap(); let mut durable_configs = configs.to_vec(); durable_configs[1].derived_input = Some( asap_types::derived_input::DerivedInputIdentity::from_dag( @@ -2545,11 +2567,12 @@ mod tests { ) .unwrap(), ); + document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; let mut durable_binding = scheduled_binding.clone(); durable_binding.nodes.insert( PostAsapNodeId(3), BackendNodeBinding::Materialization { - summary_definition: durable_configs[1].policy_fingerprint().into(), + stored_output: durable_configs[1].policy_fingerprint().into(), }, ); let installed = InstalledPostAsapDag { @@ -2582,7 +2605,7 @@ mod tests { let generation = store.active_catalog_generation().unwrap(); for ((start, end), value) in [((0, 1000), 2.0), ((1000, 2000), 7.0)] { let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source_definition, + stored_output_id: source_definition, time_range: asap_types::sds::HalfOpenTimeRange { start_ms: start, end_ms: end, @@ -2905,7 +2928,7 @@ mod tests { planner_types::pre_asap::ColumnRef::SampleValue, ); } - let document = + let mut document = OwnedPostAsapDag::from_executable("two-source-fixture".into(), &dag).unwrap(); let mut target = configs[1].clone(); if complete_groups { @@ -2922,15 +2945,16 @@ mod tests { ) .unwrap(), ); + document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; let mut binding = binding.clone(); - for (node, summary_definition) in [ + for (node, stored_output) in [ (1, first_id), (5, second_id), (3, target.policy_fingerprint().into()), ] { binding.nodes.insert( PostAsapNodeId(node), - BackendNodeBinding::Materialization { summary_definition }, + BackendNodeBinding::Materialization { stored_output }, ); } binding @@ -3004,7 +3028,7 @@ mod tests { BTreeMap::new() }; let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: config.policy_fingerprint().into(), + stored_output_id: config.policy_fingerprint().into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: start, end_ms: start + 2000, @@ -3364,9 +3388,8 @@ mod tests { }; operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, }; - operation.output_state = planner_types::post_asap::ExecutionDataState::MAINTENANCE_ROWS; + operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; assert!(frozen .execute(&operation, &[left.clone(), right.clone()]) .is_ok()); @@ -3383,8 +3406,8 @@ mod tests { .is_err()); operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), - timing: planner_types::post_asap::ExecutionTiming::ReadTime, }; + operation.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; assert!(frozen .execute(&operation, &[left.clone(), right.clone()]) .is_err()); @@ -3589,7 +3612,7 @@ mod tests { nodes: BTreeMap::from([( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: config.policy_fingerprint().into(), + stored_output: config.policy_fingerprint().into(), }, )]), query_sink: PostAsapNodeId(1), @@ -3654,7 +3677,7 @@ mod tests { nodes: BTreeMap::from([( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: config.policy_fingerprint().into(), + stored_output: config.policy_fingerprint().into(), }, )]), query_sink: PostAsapNodeId(1), @@ -3700,7 +3723,7 @@ mod tests { let key = |end| MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: end - 10, window_end_ms: end, input_lineage: vec![0; 32], @@ -3735,7 +3758,7 @@ mod tests { let key = |end| MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: end - 10, window_end_ms: end, input_lineage: vec![0; 32], @@ -3774,7 +3797,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: definition(target), + stored_output: definition(target), window_start_ms: 0, window_end_ms: 10, input_lineage: vec![0; 32], @@ -3839,7 +3862,7 @@ mod tests { .policy_fingerprint() .into(); let mut query = node(2); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![node(0), node(1), query], edges: vec![edge(0, 1), edge(1, 2)], @@ -3850,13 +3873,13 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Materialization { - summary_definition: definition(1), + stored_output: definition(1), }, ), ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: target_definition, + stored_output: target_definition, }, ), ( @@ -3870,10 +3893,17 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; + let (dag, binding) = maintenance_only(dag, binding); bundle.precompute_plan.executable_dags = BTreeMap::from([( "retry".into(), InstalledPostAsapDag { - document: OwnedPostAsapDag::from_executable("retry".into(), &dag).unwrap(), + document: { + let mut document = + OwnedPostAsapDag::from_executable("retry".into(), &dag).unwrap(); + document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; + document + }, binding, }, )]); @@ -3972,7 +4002,7 @@ mod tests { // source 0 is shared by both branches; root therefore contains two // copies of its value while node 0 itself is evaluated once. let mut query = node(4); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: (0..4).map(node).chain([query]).collect(), edges: vec![edge(0, 1), edge(0, 2), edge(1, 3), edge(2, 3), edge(3, 4)], @@ -3984,7 +4014,7 @@ mod tests { ( PostAsapNodeId(id), BackendNodeBinding::Materialization { - summary_definition: definition(if id == 0 { 1 } else { id as u64 + 1 }), + stored_output: definition(if id == 0 { 1 } else { id as u64 + 1 }), }, ) }) @@ -3999,6 +4029,7 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], }; + let (dag, binding) = maintenance_only(dag, binding); let source = sum(2.0); let adapter = OperatorAdapter { binding: &binding, @@ -4012,7 +4043,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 2, - summary_definition: definition(4), + stored_output: definition(4), window_start_ms: 0, window_end_ms: 10, input_lineage: b"batch:1".to_vec(), @@ -4043,7 +4074,7 @@ mod tests { let mut unsupported = node(1); unsupported.payload = ExecutableOperatorPayload::SummarySubtract; let mut query = node(2); - query.output_state = planner_types::post_asap::ExecutionDataState::READ_ROWS; + query.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![node(0), unsupported, query], edges: vec![edge(0, 1), edge(1, 2)], @@ -4054,13 +4085,13 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Materialization { - summary_definition: definition(1), + stored_output: definition(1), }, ), ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: definition(2), + stored_output: definition(2), }, ), ( @@ -4074,6 +4105,7 @@ mod tests { query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; + let (dag, binding) = maintenance_only(dag, binding); let adapter = OperatorAdapter { binding: &binding, inputs: MaintenanceInputs::Live { @@ -4086,7 +4118,7 @@ mod tests { let key = MaterializationCommitKey { plan_id: 7, plan_version: 2, - summary_definition: definition(2), + stored_output: definition(2), window_start_ms: 0, window_end_ms: 10, input_lineage: b"batch:1".to_vec(), diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 068ac13b7..3f744870a 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod metrics; pub mod multisource_coordinator; pub mod operators; pub mod output_sink; +pub mod raw_dag; pub mod series_buffer; pub mod series_router; pub mod subdag_scheduler; diff --git a/data_plane/src/precompute_engine/multisource_coordinator.rs b/data_plane/src/precompute_engine/multisource_coordinator.rs index d429d774a..1afe3a407 100644 --- a/data_plane/src/precompute_engine/multisource_coordinator.rs +++ b/data_plane/src/precompute_engine/multisource_coordinator.rs @@ -24,7 +24,7 @@ pub struct LogicalSourcePartition { #[serde(deny_unknown_fields)] pub struct CoordinatedInput { pub input_node_id: String, - pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub stored_output_id: asap_types::sds::StoredOutputId, pub partitions: BTreeSet, } @@ -36,7 +36,7 @@ pub struct MultiSourceNodeSpec { pub consumer_node_id: String, /// The content-addressed installed output binds its source/window contract. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_definition: Option, + pub output_definition: Option, pub inputs: Vec, /// Named output grouping. An empty projection represents one global group. pub output_grouping: Vec, @@ -122,7 +122,7 @@ impl MultiSourceCoordinator { let supplied: BTreeSet<_> = spec .inputs .iter() - .map(|input| input.summary_definition_id) + .map(|input| input.stored_output_id) .collect(); if &supplied != expected || supplied.len() != spec.inputs.len() @@ -143,15 +143,13 @@ impl MultiSourceCoordinator { let source = plan .materializations .iter() - .find(|config| { - config.policy_fingerprint() == input.summary_definition_id.fingerprint() - }) + .find(|config| config.policy_fingerprint() == input.stored_output_id.fingerprint()) .ok_or_else(|| invalid("coordinator input is not installed"))?; sources.push(source); let producers: Vec<_> = plan .producers .iter() - .filter(|producer| producer.materialization == input.summary_definition_id) + .filter(|producer| producer.materialization == input.stored_output_id) .collect(); if producers.is_empty() || producers @@ -201,7 +199,7 @@ impl MultiSourceCoordinator { .installed_plan .as_ref() .unwrap() - .validate_watermark_scope(input.summary_definition_id, &barrier) + .validate_watermark_scope(input.stored_output_id, &barrier) .map_err(|error| invalid(error.to_string()))?; } } @@ -265,7 +263,7 @@ impl MultiSourceCoordinator { .iter() .filter(|input| input.partitions.contains(&logical(&barrier.source))) { - plan.validate_watermark_scope(input.summary_definition_id, &barrier) + plan.validate_watermark_scope(input.stored_output_id, &barrier) .map_err(|error| invalid(error.to_string()))?; } } @@ -347,7 +345,7 @@ impl MultiSourceCoordinator { .iter() .find(|requirement| requirement.input_node_id == input.input_node_id) .ok_or_else(|| invalid("staged input node is not required"))?; - if input.coordinates.summary_definition_id != requirement.summary_definition_id + if input.coordinates.stored_output_id != requirement.stored_output_id || !requirement.partitions.contains(&logical(&input.source)) { return Err(invalid( @@ -359,7 +357,7 @@ impl MultiSourceCoordinator { .materializations .iter() .find(|config| { - config.policy_fingerprint() == requirement.summary_definition_id.fingerprint() + config.policy_fingerprint() == requirement.stored_output_id.fingerprint() }) .ok_or_else(|| invalid("staged input definition is not installed"))?; let window = input.coordinates.time_range; @@ -544,12 +542,12 @@ mod tests { inputs: vec![ CoordinatedInput { input_node_id: "left".into(), - summary_definition_id: PolicyFingerprint(1).into(), + stored_output_id: PolicyFingerprint(1).into(), partitions: BTreeSet::from([partition("0")]), }, CoordinatedInput { input_node_id: "right".into(), - summary_definition_id: PolicyFingerprint(2).into(), + stored_output_id: PolicyFingerprint(2).into(), partitions: BTreeSet::from([partition("1")]), }, ], @@ -569,7 +567,7 @@ mod tests { }, instance_id: SummaryInstanceId::new(format!("{node}-{epoch}")).unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: PolicyFingerprint(definition).into(), + stored_output_id: PolicyFingerprint(definition).into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 10, @@ -656,7 +654,7 @@ mod tests { }); spec.inputs.push(CoordinatedInput { input_node_id: format!("input-{ordinal}"), - summary_definition_id: definition, + stored_output_id: definition, partitions: partition_ids .into_iter() .map(|partition_id| LogicalSourcePartition { @@ -695,7 +693,7 @@ mod tests { }, instance_id: SummaryInstanceId::new(key.clone()).unwrap(), coordinates: SummaryInstanceCoordinates { - summary_definition_id: requirement.summary_definition_id, + stored_output_id: requirement.stored_output_id, time_range: HalfOpenTimeRange { start_ms: start, end_ms: start + 60000, diff --git a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs index 86b95a5ba..2874f5102 100644 --- a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs @@ -56,45 +56,11 @@ impl DatasketchesKLLAccumulator { /// DataCollector's `kllprocessor` emits when /// `encoding = KLL_SKETCH_ENCODING_PROTO`. /// - /// ⚠ This is a **lossy statistical reconstruction**, not a - /// bit-identical round-trip: the `KllState` proto carries the - /// retained items in level order plus an explicit `levels[]` - /// boundary array, but sketch-core's `KllSketch` backend types - /// keep their level structure private. Rather than touch - /// upstream `asap_sketchlib` to add a typed-state constructor, - /// we build a fresh `DatasketchesKLLAccumulator` with the same - /// `k` and replay every retained item through `update()`. - /// Quantile estimates on the reconstructed sketch are - /// approximately equivalent to the source's — within KLL's - /// own rank-error bound, which is the same bound the source - /// already inherited — so Phase 1 hot-path queries that hit - /// the reconstructed sketch return answers the user would - /// already have accepted from the source. Bit-identical - /// reconstruction is tracked as a sketchlib upstream follow-up. + /// The neutral codec decodes the sketchlib envelope. + /// The level-aware constructor below preserves the supplied retained + /// sample layout without replaying updates. pub fn from_sketchlib_proto_bytes(buffer: &[u8]) -> Result> { - use asap_sketchlib::proto::sketchlib::{sketch_envelope, KllState, SketchEnvelope}; - use prost::Message; - - // DataCollector's kllprocessor wraps the state in a - // `SketchEnvelope{kll: KllState}` via sketchlib-go's - // `SerializePortableFO` + `proto.Marshal`. Try envelope first, - // fall back to bare `KllState` for callers (e.g. unit tests) - // that encode the state directly. Mirrors the PR #14 fix on - // `CountMinSketchAccumulator::from_sketchlib_proto_bytes`. - let state = match SketchEnvelope::decode(buffer) { - Ok(env) => match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(st)) => st, - Some(other) => { - return Err(format!( - "SketchEnvelope contains non-KLL sketch: {:?}", - std::mem::discriminant(&other) - ) - .into()); - } - None => KllState::decode(buffer).map_err(|e| format!("decode KllState: {e}"))?, - }, - Err(_) => KllState::decode(buffer).map_err(|e| format!("decode KllState: {e}"))?, - }; + let state = asap_sketch_codec::kll_state(buffer)?; if state.k < 8 { return Err(format!("KllState.k must be >= 8 (got {})", state.k).into()); } @@ -394,6 +360,16 @@ impl MergeableAccumulator for DatasketchesKLLAccumul mod tests { use super::*; + fn encode_state(state: asap_sketchlib::proto::sketchlib::KllState) -> Vec { + use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; + use prost::Message; + SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec() + } + #[test] fn test_datasketches_kll_creation() { let kll = DatasketchesKLLAccumulator::new(200); @@ -590,7 +566,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let acc = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes).expect("decode ok"); @@ -635,7 +611,7 @@ mod tests { residuals: vec![], }; let decoded = - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&state.encode_to_vec()).unwrap(); + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&encode_state(state)).unwrap(); assert_eq!(decoded.inner.count(), source.count() as u64); for q in [0.0, 0.1, 0.5, 0.9, 1.0] { assert_eq!(decoded.inner.quantile(q), source.quantile(q), "q={q}"); @@ -705,7 +681,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let result = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("k must be >= 8")); @@ -727,7 +703,7 @@ mod tests { value_scale: 0, residuals: Vec::new(), }; - let bytes = state.encode_to_vec(); + let bytes = encode_state(state); let result = DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&bytes); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("levels length")); diff --git a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs index 926a1e883..0f63348b1 100644 --- a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs @@ -91,42 +91,7 @@ impl DDSketchAccumulator { /// DataCollector's `ddsketchprocessor` emits when /// `encoding = DD_SKETCH_ENCODING_PROTO`. pub fn from_sketchlib_proto_bytes(buffer: &[u8]) -> Result> { - use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; - use prost::Message; - - // DataCollector's ddsketchprocessor wraps the state in a - // `SketchEnvelope{ddsketch: DdSketchState}` via sketchlib-go's - // `SerializePortableFO` + `proto.Marshal`. Try envelope first, - // fall back to bare `DdSketchState` for callers (e.g. unit - // tests) that encode the state directly. Mirrors the PR #14 - // fix on `CountMinSketchAccumulator::from_sketchlib_proto_bytes`. - // Capture the envelope's `sample_p` alongside the state so a `Count` - // query can rescale by `1/p`. Bare `DdSketchState` bytes (no envelope) - // carry no sampling info → `sample_p` 1.0 (no rescale). - let (state, sample_p) = match SketchEnvelope::decode(buffer) { - Ok(env) => { - let sp = env.sample_p; - match env.sketch_state { - Some(sketch_envelope::SketchState::Ddsketch(st)) => (st, sp), - Some(other) => { - return Err(format!( - "SketchEnvelope contains non-DDSketch sketch: {:?}", - std::mem::discriminant(&other) - ) - .into()); - } - None => ( - DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, - 1.0, - ), - } - } - Err(_) => ( - DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))?, - 1.0, - ), - }; + let (state, sample_p) = asap_sketch_codec::ddsketch_state(buffer)?; if !(state.alpha > 0.0 && state.alpha < 1.0) { return Err(format!( "DDSketchState alpha {} out of range (expected 0 < alpha < 1)", @@ -340,14 +305,18 @@ mod tests { // asap_sketchlib#57); the proto now carries only // `alpha`/`store_counts`/`store_offset`. fn encode_state(alpha: f64, store_counts: Vec, store_offset: i32) -> Vec { - use asap_sketchlib::proto::sketchlib::DdSketchState; + use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; let state = DdSketchState { alpha, store_counts, store_offset, }; - state.encode_to_vec() + SketchEnvelope { + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec() } #[test] diff --git a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs b/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs deleted file mode 100644 index fba53304a..000000000 --- a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs +++ /dev/null @@ -1,391 +0,0 @@ -//! Edge-runtime adapter — Phase 3 step 3 of the ASAP edge-framework -//! migration (`docs/design-asap-edge-framework.md`, ADR-0002). -//! -//! Routes the **shared** ingest work (envelope wire-format parsing, -//! per-sketch state extraction, sketch reconstruction, sketch merge) -//! through the [`asap_precompute_rs`] crate so the same code runs in -//! agents (Rust edge runtime) and the backend (this repo). What stays -//! in this repo: the QUERY-side engine — PromQL aggregation, storage, -//! and query planning. See README §"Ingest path consumes -//! asap-precompute-rs" for the contract. -//! -//! # What's shared -//! -//! - **Envelope wire-format parsing.** The runtime view -//! [`SketchEnvelope`] (renamed from a backend-internal struct to -//! asap-precompute-rs's canonical type) decodes the on-the-wire -//! `asap_sketchlib::proto::sketchlib::SketchEnvelope` proto plus the -//! surrounding metadata (window bounds, `agg_id`, encoding tag, -//! sketch type, host-neutral labels, metric name, count, -//! temporality). -//! - **Per-sketch state extraction.** [`unwrap_envelope_state`] dispatches -//! the [`asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState`] -//! oneof into a typed `*State` proto for the requested sketch type, -//! producing the same diagnostic shape every accumulator's -//! `from_sketchlib_proto_bytes` used to repeat by hand. -//! - **Sketch reconstruction (DDSketch + KLL today).** -//! [`reconstruct_via_runtime`] constructs an asap-precompute-rs -//! `*Wrapper`, runs `Sketch::apply_delta(envelope_bytes)` (which is -//! the Layer-3 runtime's reconstruct-from-bytes path), then extracts -//! the underlying `asap_sketchlib::sketches::*` state via -//! `wrapper.inner().clone()`. DDSketch and KLL byte parity holds in -//! `asap_sketchlib::main` (PRs #40, #41); HLL / CountSketch / -//! CountMinSketch are tracked under -//! ProjectASAP/ASAPCollector#243 — until those land we keep the -//! backend's per-accumulator decoders for the three sketches and -//! only delegate envelope-parsing. -//! - **Cross-runtime sketch merge.** [`merge_via_runtime`] uses -//! asap-precompute-rs's `Sketch::merge` + `Sketch::snapshot` round- -//! trip so the merge logic lives in one place. Identical results to -//! `asap_sketchlib::sketches::*::merge_refs` because both call into -//! the same underlying merge implementation. -//! -//! # What stays put -//! -//! - The backend's **per-accumulator query-side surface** (`AggregateCore`, -//! `query_statistic`, `MergeableAccumulator`, ...) — query-side and -//! not what asap-precompute-rs is for. -//! - **Sparse delta application** (`apply_proto_delta_bytes` on the -//! backend's accumulators) — `asap_sketchlib` doesn't yet expose the -//! `compute_delta` family (Go's `sketchlib-go` has it; tracked -//! upstream), so asap-precompute-rs's wrappers fall back to "always -//! full" delta encoding. Backend's typed-delta apply is independent -//! and stays. - -use asap_precompute_rs::{ - envelope::ProtoSketchEnvelope, sketches::DDSketchWrapper, sketches::KLLWrapper, Sketch, -}; -use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; -use prost::Message; - -/// Re-export of asap-precompute-rs's runtime view of the -/// `SketchEnvelope` proto (the prost-generated wire-format type plus -/// surrounding host-neutral metadata: window bounds, `agg_id`, -/// encoding tag, sketch-type tag, labels, metric name, count, -/// temporality). -/// -/// Kept as a re-export so all backend code that touches the runtime -/// envelope reaches for the canonical type from the edge-framework -/// runtime — preventing a "backend-flavored" runtime view from -/// drifting alongside the agent-flavored one. -pub use asap_precompute_rs::envelope::{Encoding, SketchEnvelope, SketchType}; - -/// Re-export the [`asap_precompute_rs::Sketch`] trait family so backend -/// code that wants to operate on envelope bytes via the host-neutral -/// runtime trait (`snapshot`, `apply_delta`, `merge`, `reset`) imports -/// from one place. -pub use asap_precompute_rs::{CardinalitySketch, FrequencySketch, QuantileSketch}; - -/// Decode the wire-format `asap_sketchlib` `SketchEnvelope` proto from -/// `bytes` and return the inner [`SketchState`] oneof variant. -/// -/// Mirrors the per-accumulator `match SketchEnvelope::decode(buffer) -/// → Some(SketchState::X(st)) → st` ladder that every -/// `from_sketchlib_proto_bytes` used to inline. The fall-back to -/// decoding `bytes` as the bare typed-state proto is preserved at the -/// caller so the existing PR #14 contract continues to work for unit -/// tests that encode the state directly (without an envelope wrapper). -/// -/// This is the **single shared envelope-unwrap path** between -/// asap-precompute-rs and backend ingest. asap-precompute-rs's -/// per-wrapper `decode_envelope` does the same prost-decode + -/// oneof-extraction work; the difference is that asap-precompute-rs -/// then constructs a typed `asap_sketchlib::sketches::*` from the -/// state, which the backend may or may not want depending on the -/// downstream caller. -pub fn unwrap_envelope_state( - bytes: &[u8], -) -> Result, Box> { - let env = - ProtoSketchEnvelope::decode(bytes).map_err(|e| format!("decode SketchEnvelope: {e}"))?; - Ok(env.sketch_state) -} - -/// Result of [`reconstruct_via_runtime`] — backend uses the inner -/// `asap_sketchlib::sketches::*` to construct its own accumulator. -pub enum ReconstructedSketch { - /// Reconstructed [`asap_sketchlib::DdSketch`] state. - DdSketch(asap_sketchlib::DdSketch), - /// Reconstructed KLL — asap-precompute-rs's [`KLLWrapper`] owns - /// the high-throughput `asap_sketchlib::sketches::kll::KLL` - /// internally; backend's KLL accumulator wraps the wire-format- - /// aligned `KllSketch` instead. We surface the wrapper's - /// re-snapshot bytes so the caller can route them through - /// backend's existing `KllSketch::deserialize_msgpack` / - /// proto-state path or replay items via `KllSketch::update()`. - Kll { - /// Bytes of the reconstructed sketch's snapshot — same shape - /// as the input envelope, validated round-trip. - snapshot_bytes: Vec, - }, -} - -/// Reconstruct an `asap_sketchlib` sketch from envelope bytes by -/// delegating envelope parsing + sketch construction to -/// asap-precompute-rs's `Sketch` trait family. -/// -/// The function is sketch-type-aware because the wrappers' constructor -/// parameters (alpha for DDSketch, k+seed for KLL, ...) live partly in -/// the envelope state proto. We peek the state, construct a wrapper -/// with matching parameters, then call [`Sketch::apply_delta`] which -/// runs asap-precompute-rs's canonical decode + reconstruct pathway. -/// -/// Today wires DDSketch (byte parity per asap_sketchlib#40) and KLL -/// (byte parity per asap_sketchlib#41). HLL / CountSketch / -/// CountMinSketch are tracked under ProjectASAP/ASAPCollector#243 — -/// callers fall back to backend's per-accumulator decoder for those. -pub fn reconstruct_via_runtime( - sketch_type: SketchType, - envelope_bytes: &[u8], -) -> Result> { - match sketch_type { - SketchType::DDSketch => { - // Peek the state to learn alpha, then construct the - // wrapper with matching alpha so `apply_delta`'s merge - // step doesn't trip on `DdSketch::merge`'s alpha-equality - // guard. - let state = unwrap_envelope_state(envelope_bytes)?; - let alpha = match &state { - Some(SketchState::Ddsketch(s)) => s.alpha, - Some(_) => { - return Err("envelope is not a DDSketch".into()); - } - None => return Err("envelope has no sketch_state".into()), - }; - if !(alpha > 0.0 && alpha < 1.0) { - return Err(format!("DDSketch alpha {alpha} out of (0,1)").into()); - } - let mut wrapper = DDSketchWrapper::new(alpha); - wrapper - .apply_delta(envelope_bytes) - .map_err(|e| format!("DDSketchWrapper apply_delta: {e}"))?; - Ok(ReconstructedSketch::DdSketch(wrapper.inner().clone())) - } - SketchType::KLLSketch => { - // KLL: peek `k`, construct an empty wrapper, apply. - let state = unwrap_envelope_state(envelope_bytes)?; - let k = match state { - Some(SketchState::Kll(s)) => { - if s.k > i32::MAX as u32 { - return Err(format!("KllState.k too large: {}", s.k).into()); - } - s.k as i32 - } - Some(_) => return Err("envelope is not a KLL".into()), - None => return Err("envelope has no sketch_state".into()), - }; - let mut wrapper = KLLWrapper::new(k, None); - wrapper - .apply_delta(envelope_bytes) - .map_err(|e| format!("KLLWrapper apply_delta: {e}"))?; - // Re-snapshot via the wrapper's `Sketch::snapshot` — - // canonical asap-precompute-rs encode of the reconstructed - // state. Backend's KLL accumulator can then re-decode via - // its existing `from_sketchlib_proto_bytes` path; the - // edge-runtime adapter has done the envelope + state - // unwrap, the wire-format reshape, and the reconstruction - // round-trip. - let snapshot_bytes = wrapper - .snapshot() - .map_err(|e| format!("KLLWrapper snapshot: {e}"))?; - Ok(ReconstructedSketch::Kll { snapshot_bytes }) - } - SketchType::HLLSketch | SketchType::CountSketch | SketchType::CountMinSketch => { - Err(format!( - "reconstruct_via_runtime({sketch_type:?}): byte parity for \ - HLL / CountSketch / CountMinSketch not yet in upstream \ - asap_sketchlib — tracked at ProjectASAP/ASAPCollector#243. \ - Caller must fall back to backend's per-accumulator decoder." - ) - .into()) - } - SketchType::Unspecified => Err("reconstruct_via_runtime: SketchType::Unspecified".into()), - } -} - -/// Snapshot a backend-side `asap_sketchlib::DdSketch` through -/// asap-precompute-rs's `Sketch` trait — the canonical encode path -/// shared with the agent runtime. Used by the round-trip test -/// (`tests/edge_runtime_adapter.rs`). -pub fn snapshot_ddsketch_via_runtime( - sk: &asap_sketchlib::DdSketch, -) -> Result, Box> { - let mut wrapper = DDSketchWrapper::new(sk.alpha); - // Bridge into the wrapper by merging in the existing sketch. - // We can't move-construct the wrapper from a non-empty `DdSketch`, - // but `Sketch::apply_delta` against the existing snapshot bytes - // is equivalent. - if sk.total_count() > 0 { - // Re-encode the source's state into the canonical envelope - // shape that asap-precompute-rs's wrapper recognizes, then - // round-trip through `apply_delta`. Mirrors the agent runtime's - // own merge path. - let bridge_bytes = encode_ddsketch_envelope(sk); - wrapper - .apply_delta(&bridge_bytes) - .map_err(|e| format!("DDSketchWrapper apply_delta (bridge): {e}"))?; - } - wrapper - .snapshot() - .map_err(|e| format!("DDSketchWrapper snapshot: {e}").into()) -} - -/// Encode a backend-side `DdSketch` as the same `SketchEnvelope` proto -/// shape that asap-precompute-rs's `DDSketchWrapper::snapshot` emits. -/// -/// Matches asap-precompute-rs's `DDSketchWrapper::build_state` + -/// `encode_envelope` byte-for-byte — they call into the same -/// `asap_sketchlib::proto::sketchlib::*` types. Lives here so the -/// backend's existing accumulators don't need to import the wrapper -/// internals. -pub fn encode_ddsketch_envelope(sk: &asap_sketchlib::DdSketch) -> Vec { - use asap_sketchlib::proto::sketchlib::{ - sketch_envelope, DdSketchState, SketchEnvelope as ProtoEnvelope, - }; - let state = DdSketchState { - // Use `wire_alpha` so the bytes match Go's - // `sketchlib-go::DDSketch.SerializePortable` (PR - // asap_sketchlib#40 closes this). - alpha: sk.wire_alpha(), - store_counts: sk.store_counts.clone(), - store_offset: sk.store_offset, - // The DataPoint-level scalars (count/sum/min/max) were dropped from - // `DDSketchState` (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); - // the bucket counts carry all reconstructable state. - }; - let env = ProtoEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - // No edge sampling on this path: 0.0 is the proto3 default (dual-read as - // 1.0) so the encoded envelope stays byte-identical. - sample_p: 0.0, - sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), - }; - let mut buf = Vec::with_capacity(env.encoded_len()); - env.encode(&mut buf).expect("prost encode"); - buf -} - -#[cfg(test)] -/// Merge two `DdSketch` instances by routing through asap-precompute-rs's -/// runtime `Sketch::merge`. The result is byte-identical to -/// `asap_sketchlib::DdSketch::merge_refs(&[a, b])` because -/// both paths call the same underlying merge logic. -/// -/// Used by the cross-runtime parity test -/// (`tests/edge_runtime_adapter.rs::ddsketch_merge_via_runtime_matches_native`). -pub fn merge_ddsketches_via_runtime( - a: &asap_sketchlib::DdSketch, - b: &asap_sketchlib::DdSketch, -) -> Result> { - if (a.alpha - b.alpha).abs() > f64::EPSILON { - return Err(format!( - "merge_ddsketches_via_runtime: alpha mismatch ({} vs {})", - a.alpha, b.alpha - ) - .into()); - } - let mut wrapper_a = DDSketchWrapper::new(a.alpha); - if a.total_count() > 0 { - let bridge = encode_ddsketch_envelope(a); - wrapper_a - .apply_delta(&bridge) - .map_err(|e| format!("merge_ddsketches_via_runtime/a: {e}"))?; - } - let mut wrapper_b = DDSketchWrapper::new(b.alpha); - if b.total_count() > 0 { - let bridge = encode_ddsketch_envelope(b); - wrapper_b - .apply_delta(&bridge) - .map_err(|e| format!("merge_ddsketches_via_runtime/b: {e}"))?; - } - // `Sketch::merge` takes a `&dyn Sketch` (round-trips through - // snapshot bytes), which is the canonical Layer-3 runtime fold. - wrapper_a - .merge(&wrapper_b) - .map_err(|e| format!("DDSketchWrapper merge: {e}"))?; - Ok(wrapper_a.inner().clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// asap-precompute-rs's wrapper produces an envelope; backend's - /// shared envelope-unwrap path returns the matching oneof variant. - /// The dedup target. - #[test] - fn unwrap_envelope_state_matches_wrapper_output() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=10 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot ok"); - let state = unwrap_envelope_state(&bytes).expect("decode ok"); - match state { - Some(SketchState::Ddsketch(s)) => { - // `count` was dropped from `DdSketchState` - // (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); - // a non-empty sketch carries it in the bucket store. - assert!(s.store_counts.iter().sum::() > 0); - assert!(s.alpha > 0.0 && s.alpha < 1.0); - } - other => panic!("expected DDSketch state, got {other:?}"), - } - } - - /// asap-precompute-rs's wrapper produces an envelope; the runtime - /// adapter's reconstruction returns a backend-shaped - /// `asap_sketchlib::DdSketch` whose serialized bytes - /// (re-encoded through the same envelope shape) match the - /// original. - #[test] - fn ddsketch_round_trip_through_runtime_adapter() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=100 { - w.update(i as f64); - } - let original_bytes = w.snapshot().expect("snapshot ok"); - let reconstructed = - reconstruct_via_runtime(SketchType::DDSketch, &original_bytes).expect("reconstruct ok"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - ReconstructedSketch::Kll { .. } => panic!("got KLL, expected DDSketch"), - }; - assert_eq!(dd.total_count(), 100); - let re_encoded = encode_ddsketch_envelope(&dd); - assert_eq!( - re_encoded, original_bytes, - "round-trip via runtime adapter must be byte-identical" - ); - } - - /// Construct two non-overlapping DDSketches, merge via the runtime - /// adapter, and verify counts add up. Uses asap-precompute-rs's - /// `Sketch::merge` + `Sketch::snapshot` round-trip. - #[test] - fn ddsketch_merge_via_runtime_combines_counts() { - let mut a = DDSketchWrapper::new(0.01); - for i in 1..=10 { - a.update(i as f64); - } - let mut b = DDSketchWrapper::new(0.01); - for i in 11..=20 { - b.update(i as f64); - } - // Reach into the wrapper's inner via snapshot/decode. - let a_inner = - match reconstruct_via_runtime(SketchType::DDSketch, &a.snapshot().unwrap()).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let b_inner = - match reconstruct_via_runtime(SketchType::DDSketch, &b.snapshot().unwrap()).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let merged = merge_ddsketches_via_runtime(&a_inner, &b_inner).expect("merge ok"); - assert_eq!(merged.total_count(), 20); - } -} diff --git a/data_plane/src/precompute_engine/operators/exact_accumulator.rs b/data_plane/src/precompute_engine/operators/exact_accumulator.rs new file mode 100644 index 000000000..b7fcead12 --- /dev/null +++ b/data_plane/src/precompute_engine/operators/exact_accumulator.rs @@ -0,0 +1,327 @@ +//! Exact summary state identified by Planner family, independent of keyed layout. +use super::increase_accumulator::IncreaseAccumulator; +use crate::storage_engines::types::{ + AggregateCore, AggregationType, AuxStats, KeyByLabelValues, Measurement, SerializableToSink, +}; +use asap_types::Statistic; +use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +type Error = Box; + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum ScalarState { + Sum(f64), + Count(u64), + Min(Option), + Max(Option), + Counter(Option), +} + +/// Both the family and population layout survive persistence. Sharing counter +/// arithmetic never authorizes a Rate state to answer an Increase readout. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExactAccumulator { + family: SummaryFamilyType, + scalar: ScalarState, + keyed: Option>, +} + +impl ExactAccumulator { + pub fn new(family: SummaryFamilyType, keyed: bool) -> Result { + use ExactKind as K; + use ExactParams as P; + let scalar = match &family { + SummaryFamilyType::ExactAggregate(K::Sum, P::Sum) => ScalarState::Sum(0.0), + SummaryFamilyType::ExactAggregate(K::Count, P::Count) => ScalarState::Count(0), + SummaryFamilyType::ExactAggregate(K::Min, P::Min) => ScalarState::Min(None), + SummaryFamilyType::ExactAggregate(K::Max, P::Max) => ScalarState::Max(None), + SummaryFamilyType::ExactAggregate(K::Rate, P::Rate) + | SummaryFamilyType::ExactAggregate(K::Increase, P::Increase) => { + ScalarState::Counter(None) + } + _ => return Err(format!("unsupported exact Planner family: {family:?}")), + }; + Ok(Self { + family, + scalar, + keyed: keyed.then(HashMap::new), + }) + } + + pub fn family(&self) -> &SummaryFamilyType { + &self.family + } + pub fn is_keyed(&self) -> bool { + self.keyed.is_some() + } + + pub fn update(&mut self, key: Option<&KeyByLabelValues>, value: f64, timestamp: i64) { + let state = match (&mut self.keyed, key) { + (Some(states), Some(key)) => states + .entry(key.clone()) + .or_insert_with(|| self.scalar.clone()), + (None, None) => &mut self.scalar, + _ => panic!("exact update population layout differs from installed DAG"), + }; + match state { + ScalarState::Sum(sum) => *sum += value, + ScalarState::Count(count) => { + *count = count.checked_add(1).expect("exact count overflow") + } + ScalarState::Min(current) => { + *current = Some(current.map_or(value, |old| old.min(value))) + } + ScalarState::Max(current) => { + *current = Some(current.map_or(value, |old| old.max(value))) + } + ScalarState::Counter(current) => match current { + Some(counter) => counter.update(Measurement::new(value), timestamp), + None => { + *current = Some(IncreaseAccumulator::new( + Measurement::new(value), + timestamp, + Measurement::new(value), + timestamp, + )) + } + }, + } + } + + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let state: Self = rmp_serde::from_slice(bytes)?; + let expected = Self::new(state.family.clone(), state.is_keyed())?; + let same_variant = |value: &ScalarState| { + std::mem::discriminant(value) == std::mem::discriminant(&expected.scalar) + }; + if !same_variant(&state.scalar) + || state + .keyed + .as_ref() + .is_some_and(|states| states.values().any(|s| !same_variant(s))) + { + return Err("exact payload differs from declared Planner family".into()); + } + Ok(state) + } + + fn statistic(&self) -> Statistic { + match self.family { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Min, _) => Statistic::Min, + SummaryFamilyType::ExactAggregate(ExactKind::Max, _) => Statistic::Max, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => Statistic::Rate, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => Statistic::Increase, + _ => unreachable!("validated exact family"), + } + } +} + +fn merge_scalar(left: &ScalarState, right: &ScalarState) -> Result { + Ok(match (left, right) { + (ScalarState::Sum(a), ScalarState::Sum(b)) => ScalarState::Sum(a + b), + (ScalarState::Count(a), ScalarState::Count(b)) => { + ScalarState::Count(a.checked_add(*b).ok_or("exact count overflow")?) + } + (ScalarState::Min(a), ScalarState::Min(b)) => { + ScalarState::Min(a.iter().chain(b).copied().reduce(f64::min)) + } + (ScalarState::Max(a), ScalarState::Max(b)) => { + ScalarState::Max(a.iter().chain(b).copied().reduce(f64::max)) + } + (ScalarState::Counter(a), ScalarState::Counter(b)) => { + ScalarState::Counter(match (a, b) { + (Some(a), Some(b)) => Some( + >::merge_accumulators(vec![a.clone(), b.clone()])?, + ), + (a, b) => a.clone().or_else(|| b.clone()), + }) + } + _ => return Err("exact scalar state families differ".into()), + }) +} + +impl SerializableToSink for ExactAccumulator { + fn serialize_to_json(&self) -> serde_json::Value { + serde_json::json!({"family": self.family, "scalar": self.scalar, "keyed": self.keyed.as_ref().map(|m|m.iter().collect::>())}) + } + fn serialize_to_bytes(&self) -> Vec { + rmp_serde::to_vec_named(self).expect("exact state encoding") + } +} + +impl AggregateCore for ExactAccumulator { + fn clone_boxed_core(&self) -> Box { + Box::new(self.clone()) + } + fn type_name(&self) -> &'static str { + "PlannerExactAccumulatorV1" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn merge_with(&self, other: &dyn AggregateCore) -> Result, Error> { + let other = other + .as_any() + .downcast_ref::() + .ok_or("merge requires Planner exact state")?; + if self.family != other.family || self.is_keyed() != other.is_keyed() { + return Err("cannot merge different Planner families or layouts".into()); + } + let mut merged = self.clone(); + if let (Some(target), Some(source)) = (&mut merged.keyed, &other.keyed) { + for (key, state) in source { + let combined = match target.get(key) { + Some(old) => merge_scalar(old, state)?, + None => state.clone(), + }; + target.insert(key.clone(), combined); + } + } else { + merged.scalar = merge_scalar(&self.scalar, &other.scalar)?; + } + Ok(Box::new(merged)) + } + fn get_accumulator_type(&self) -> AggregationType { + match self.statistic() { + Statistic::Sum => AggregationType::Sum, + Statistic::Count => AggregationType::Count, + Statistic::Min => AggregationType::Min, + Statistic::Max => AggregationType::Max, + Statistic::Rate => AggregationType::Rate, + Statistic::Increase => AggregationType::Increase, + _ => unreachable!(), + } + } + fn approx_memory_bytes(&self) -> usize { + std::mem::size_of::() + + self.keyed.as_ref().map_or(0, |m| { + m.keys() + .map(|k| { + std::mem::size_of::() + + k.labels.iter().map(String::len).sum::() + }) + .sum::() + }) + } + fn aux_stats(&self) -> AuxStats { + if self.is_keyed() { + return AuxStats::empty(); + } + match self.scalar { + ScalarState::Sum(value) => AuxStats { + sum: Some(value), + ..AuxStats::empty() + }, + ScalarState::Count(value) => AuxStats { + count: Some(value), + ..AuxStats::empty() + }, + ScalarState::Min(value) => AuxStats { + min: value, + ..AuxStats::empty() + }, + ScalarState::Max(value) => AuxStats { + max: value, + ..AuxStats::empty() + }, + ScalarState::Counter(_) => AuxStats::empty(), + } + } + fn get_keys(&self) -> Option> { + self.keyed.as_ref().map(|m| m.keys().cloned().collect()) + } + fn query_statistic( + &self, + statistic: Statistic, + key: &Option, + kwargs: &HashMap, + ) -> Result { + if statistic != self.statistic() { + return Err("readout differs from Planner exact family".into()); + } + let state = match (&self.keyed, key) { + (Some(states), Some(key)) => states.get(key).ok_or("unknown exact population")?, + (None, None) => &self.scalar, + _ => return Err("readout population differs from installed layout".into()), + }; + match state { + ScalarState::Sum(sum) => Ok(*sum), + ScalarState::Count(count) => Ok(*count as f64), + ScalarState::Min(value) | ScalarState::Max(value) => { + value.ok_or_else(|| "empty exact population".into()) + } + ScalarState::Counter(Some(counter)) => { + counter.query_statistic(statistic, &None, kwargs) + } + ScalarState::Counter(None) => Err("empty counter population".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Identity, population isolation, and readout survive the persisted format. + #[test] + fn exact_families_roundtrip_and_reject_cross_family_operations() { + let families = [ + (ExactKind::Sum, ExactParams::Sum, Statistic::Sum, 16.0), + (ExactKind::Count, ExactParams::Count, Statistic::Count, 3.0), + (ExactKind::Min, ExactParams::Min, Statistic::Min, 2.0), + (ExactKind::Max, ExactParams::Max, Statistic::Max, 8.0), + (ExactKind::Rate, ExactParams::Rate, Statistic::Rate, 3.0), + ( + ExactKind::Increase, + ExactParams::Increase, + Statistic::Increase, + 6.0, + ), + ]; + for keyed in [false, true] { + let key = keyed.then(|| KeyByLabelValues::new_with_labels(vec!["a".into()])); + let mut states = Vec::new(); + for (kind, params, stat, value) in &families { + let mut state = ExactAccumulator::new( + SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), + keyed, + ) + .unwrap(); + for (ts, v) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(key.as_ref(), v, ts); + } + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()).unwrap(); + assert_eq!(restored.family(), state.family()); + assert_eq!( + restored + .query_statistic(*stat, &key, &HashMap::new()) + .unwrap(), + *value + ); + for (_, _, wrong, _) in &families { + if wrong != stat { + assert!(restored + .query_statistic(*wrong, &key, &HashMap::new()) + .is_err()); + } + } + states.push(restored); + } + for (i, a) in states.iter().enumerate() { + for (j, b) in states.iter().enumerate() { + assert_eq!(a.merge_with(b).is_ok(), i == j); + } + } + } + } +} diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs similarity index 86% rename from data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_counter_state.rs index c1d59aa1a..b94d2faba 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs @@ -12,11 +12,11 @@ use asap_types::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys /// Allows tracking rate/increase for different label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleIncreaseAccumulator { +pub struct KeyedCounterState { pub increases: HashMap, } -impl MultipleIncreaseAccumulator { +impl KeyedCounterState { pub fn new() -> Self { Self { increases: HashMap::new(), @@ -91,13 +91,13 @@ impl MultipleIncreaseAccumulator { } } -impl Default for MultipleIncreaseAccumulator { +impl Default for KeyedCounterState { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleIncreaseAccumulator { +impl SerializableToSink for KeyedCounterState { fn serialize_to_json(&self) -> Value { let entries: Vec = self .increases @@ -135,13 +135,13 @@ impl SerializableToSink for MultipleIncreaseAccumulator { } } -impl AggregateCore for MultipleIncreaseAccumulator { +impl AggregateCore for KeyedCounterState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleIncreaseAccumulator" + "KeyedCounterState" } fn as_any(&self) -> &dyn std::any::Any { @@ -156,20 +156,20 @@ impl AggregateCore for MultipleIncreaseAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleIncreaseAccumulator + // Check if other is also a KeyedCounterState if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleIncreaseAccumulator with {}", + "Cannot merge KeyedCounterState with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleIncreaseAccumulator + // Downcast to KeyedCounterState let other_multiple_increase = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleIncreaseAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedCounterState")?; // Clone self once, then merge each matching counter with the same // reset-aware, boundary-aware implementation used by the unkeyed path. @@ -189,7 +189,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleIncrease + AggregationType::Increase } fn approx_memory_bytes(&self) -> usize { @@ -210,14 +210,12 @@ impl AggregateCore for MultipleIncreaseAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleIncreaseAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedCounterState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { +impl MultipleSubpopulationAggregate for KeyedCounterState { fn query( &self, statistic: Statistic, @@ -227,7 +225,7 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { let data = self .increases .get(key) - .ok_or_else(|| format!("Key {key} not found in MultipleIncreaseAccumulator"))?; + .ok_or_else(|| format!("Key {key} not found in KeyedCounterState"))?; data.query(statistic, query_kwargs) } @@ -237,15 +235,15 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { } } -impl MergeableAccumulator for MultipleIncreaseAccumulator { +impl MergeableAccumulator for KeyedCounterState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleIncreaseAccumulator::new(); + let mut result = KeyedCounterState::new(); for accumulator in accumulators { for (key, data) in accumulator.increases { @@ -291,14 +289,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_creation() { - let acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_creation() { + let acc = KeyedCounterState::new(); assert!(acc.increases.is_empty()); } #[test] - fn test_multiple_increase_accumulator_update() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_update() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -316,8 +314,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_query() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_query() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -344,14 +342,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_sum_per_key() { - // `sum by (zone) (counter)` reaches MultipleIncreaseAccumulator + fn test_keyed_counter_state_sum_per_key() { + // `sum by (zone) (counter)` reaches KeyedCounterState // only when the ASAP-tier ingest groups multiple series under // a single accumulator (the `Multiple*` variant). In that case // each per-key Sum should be the series' latest cumulative // value; the engine's outer `by` aggregation does the cross-key // grouping. (Issue ProjectASAP/ASAPCollector#46.) - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let east = KeyByLabelValues::new_with_labels(vec!["us-east-1".to_string()]); let west = KeyByLabelValues::new_with_labels(vec!["us-west-2".to_string()]); @@ -369,9 +367,9 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_merge() { - let mut acc1 = MultipleIncreaseAccumulator::new(); - let mut acc2 = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_merge() { + let mut acc1 = KeyedCounterState::new(); + let mut acc2 = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -387,7 +385,7 @@ mod tests { create_test_increase_accumulator_with_time(15.0, 2000, 30.0, 3000), ); // Later time range - let merged = MultipleIncreaseAccumulator::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = KeyedCounterState::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.increases.len(), 2); assert!(merged.increases.contains_key(&key1)); @@ -400,8 +398,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_serialization() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_serialization() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let second_key = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -415,7 +413,7 @@ mod tests { // Test JSON serialization let json_value = acc.serialize_to_json(); - let deserialized = MultipleIncreaseAccumulator::deserialize_from_json(&json_value).unwrap(); + let deserialized = KeyedCounterState::deserialize_from_json(&json_value).unwrap(); assert_eq!(deserialized.increases.len(), 2); let deserialized_acc = deserialized.increases.get(&key).unwrap(); @@ -425,8 +423,7 @@ mod tests { // Test binary serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = - MultipleIncreaseAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedCounterState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.increases.len(), 2); let deserialized_acc_bytes = deserialized_bytes.increases.get(&key).unwrap(); @@ -445,8 +442,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_get_keys() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_get_keys() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let key2 = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -462,7 +459,7 @@ mod tests { #[test] fn test_trait_object() { - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new(); acc.update(key.clone(), create_test_increase_accumulator(10.0, 25.0)); @@ -477,7 +474,7 @@ mod tests { } // #[test] - // fn test_multiple_increase_accumulator_arroyo_deserialization() { + // fn test_keyed_counter_state_arroyo_deserialization() { // // Create test data in Arroyo MessagePack format // // Format: {key: [starting_value, starting_timestamp, last_seen_value, last_seen_timestamp]} // let mut test_data = std::collections::HashMap::new(); @@ -489,7 +486,7 @@ mod tests { // // Test Arroyo deserialization // let deserialized_acc = - // MultipleIncreaseAccumulator::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); + // KeyedCounterState::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); // // Verify the deserialized accumulator has the correct data // assert_eq!(deserialized_acc.increases.len(), 2); diff --git a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_max_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_max_state.rs index 1865d2686..4309c04a0 100644 --- a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_max_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key maximum over many populations, mergeable by comparison. /// /// The minimum direction is -/// [`MultipleMinAccumulator`](super::multiple_min_accumulator::MultipleMinAccumulator), +/// [`KeyedMinState`](super::keyed_min_state::KeyedMinState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMaxAccumulator { +pub struct KeyedMaxState { pub values: HashMap, } -impl MultipleMaxAccumulator { +impl KeyedMaxState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMaxAccumulator { } } -impl SerializableToSink for MultipleMaxAccumulator { +impl SerializableToSink for KeyedMaxState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMaxAccumulator { } } -impl AggregateCore for MultipleMaxAccumulator { +impl AggregateCore for KeyedMaxState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMaxAccumulator" + "KeyedMaxState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMaxAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMaxAccumulator with {}", + "Cannot merge KeyedMaxState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMaxAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMaxAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMaxState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMaxAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMax + AggregationType::Max } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMaxAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMaxAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMaxState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { +impl MultipleSubpopulationAggregate for KeyedMaxState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMaxAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMaxAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMaxState").into()), + other => Err(format!("Unsupported statistic in KeyedMaxState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { } } -impl MergeableAccumulator for MultipleMaxAccumulator { +impl MergeableAccumulator for KeyedMaxState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMaxAccumulator::new(); + let mut result = KeyedMaxState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_largest_per_key() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Min, &key("a"), None).is_err()); assert!(acc.query(Statistic::Max, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMaxAccumulator::new(); + let mut left = KeyedMaxState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMaxAccumulator::new(); + let mut right = KeyedMaxState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Max, &key("a"), None).unwrap(), 10.0); assert_eq!(merged.query(Statistic::Max, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_min_accumulator::MultipleMinAccumulator; - let mine = MultipleMaxAccumulator::new(); - let theirs = MultipleMinAccumulator::new(); + use super::super::keyed_min_state::KeyedMinState; + let mine = KeyedMaxState::new(); + let theirs = KeyedMinState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMaxAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMaxState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMaxAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMaxState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_min_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_min_state.rs index 00c250402..5be698f50 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_min_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key minimum over many populations, mergeable by comparison. /// /// The maximum direction is -/// [`MultipleMaxAccumulator`](super::multiple_max_accumulator::MultipleMaxAccumulator), +/// [`KeyedMaxState`](super::keyed_max_state::KeyedMaxState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMinAccumulator { +pub struct KeyedMinState { pub values: HashMap, } -impl MultipleMinAccumulator { +impl KeyedMinState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMinAccumulator { } } -impl SerializableToSink for MultipleMinAccumulator { +impl SerializableToSink for KeyedMinState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMinAccumulator { } } -impl AggregateCore for MultipleMinAccumulator { +impl AggregateCore for KeyedMinState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMinAccumulator" + "KeyedMinState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMinAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMinAccumulator with {}", + "Cannot merge KeyedMinState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMinAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMinAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMinState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMinAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMin + AggregationType::Min } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMinAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMinAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMinState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMinAccumulator { +impl MultipleSubpopulationAggregate for KeyedMinState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMinAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMinAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMinState").into()), + other => Err(format!("Unsupported statistic in KeyedMinState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { } } -impl MergeableAccumulator for MultipleMinAccumulator { +impl MergeableAccumulator for KeyedMinState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMinAccumulator::new(); + let mut result = KeyedMinState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_smallest_per_key() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Max, &key("a"), None).is_err()); assert!(acc.query(Statistic::Min, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMinAccumulator::new(); + let mut left = KeyedMinState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMinAccumulator::new(); + let mut right = KeyedMinState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Min, &key("a"), None).unwrap(), 5.0); assert_eq!(merged.query(Statistic::Min, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_max_accumulator::MultipleMaxAccumulator; - let mine = MultipleMinAccumulator::new(); - let theirs = MultipleMaxAccumulator::new(); + use super::super::keyed_max_state::KeyedMaxState; + let mine = KeyedMinState::new(); + let theirs = KeyedMaxState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMinAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMinState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMinAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMinState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs similarity index 50% rename from data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs index 85a9982d8..c39d55831 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs @@ -7,26 +7,53 @@ use serde_json::Value; use std::collections::HashMap; use asap_types::Statistic; +use planner_types::post_asap::ExactKind; + +fn sum_family() -> ExactKind { + ExactKind::Sum +} /// Accumulator that maintains separate sum values for multiple keys /// Allows querying sums for specific label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleSumAccumulator { +pub struct KeyedSumCountAccumulator { + #[serde(default = "sum_family")] + pub family: ExactKind, pub sums: HashMap, + #[serde(default)] + pub counts: HashMap, } -impl MultipleSumAccumulator { +impl KeyedSumCountAccumulator { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { + assert!(matches!(family, ExactKind::Sum | ExactKind::Count)); Self { + family, sums: HashMap::new(), + counts: HashMap::new(), } } pub fn update(&mut self, key: KeyByLabelValues, value: f64) { - *self.sums.entry(key).or_insert(0.0) += value; + let is_new = !self.sums.contains_key(&key); + *self.sums.entry(key.clone()).or_insert(0.0) += value; + if let Some(count) = self.counts.get(&key).copied() { + if let Some(next) = count.checked_add(1).filter(|next| *next != u64::MAX) { + self.counts.insert(key, next); + } else { + self.counts.remove(&key); + } + } else if is_new { + self.counts.insert(key, 1); + } } pub fn add_sum(&mut self, key: KeyByLabelValues, sum: f64) { + self.counts.remove(&key); self.sums.insert(key, sum); } @@ -43,7 +70,28 @@ impl MultipleSumAccumulator { sums.insert(key, sum); } - Ok(Self { sums }) + let mut counts = HashMap::new(); + if let Some(counts_data) = data.get("counts").and_then(Value::as_object) { + for (key_str, value) in counts_data { + let key_json: Value = serde_json::from_str(key_str)?; + let key = KeyByLabelValues::deserialize_from_json(&key_json)?; + let count = value.as_u64().ok_or("Invalid count value")?; + if !sums.contains_key(&key) { + return Err("Count key missing from sums".into()); + } + counts.insert(key, count); + } + } + let family = match data.get("family").and_then(Value::as_str) { + None | Some("Sum") => ExactKind::Sum, + Some("Count") => ExactKind::Count, + _ => return Err("Invalid keyed additive family".into()), + }; + Ok(Self { + family, + sums, + counts, + }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -62,6 +110,7 @@ impl MultipleSumAccumulator { offset += 4; let mut sums = HashMap::new(); + let mut keys = Vec::new(); for _ in 0..num_entries { // Read key length and data @@ -99,20 +148,50 @@ impl MultipleSumAccumulator { ]); offset += 8; + keys.push(key.clone()); sums.insert(key, sum); } - - Ok(Self { sums }) + let remaining = buffer.len() - offset; + let count_bytes = num_entries + .checked_mul(8) + .ok_or("Count section too large")?; + if remaining != 0 && remaining != count_bytes && remaining != count_bytes + 1 { + return Err("Invalid count section length".into()); + } + let mut counts = HashMap::new(); + if count_bytes != 0 && remaining >= count_bytes { + for key in keys { + let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); + offset += 8; + if count != u64::MAX { + counts.insert(key, count); + } + } + } + let family = if remaining == count_bytes + 1 { + match buffer[offset] { + 0 => ExactKind::Sum, + 1 => ExactKind::Count, + _ => return Err("Invalid keyed additive family tag".into()), + } + } else { + ExactKind::Sum + }; + Ok(Self { + family, + sums, + counts, + }) } } -impl Default for MultipleSumAccumulator { +impl Default for KeyedSumCountAccumulator { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleSumAccumulator { +impl SerializableToSink for KeyedSumCountAccumulator { fn serialize_to_json(&self) -> Value { let mut sums_obj = serde_json::Map::new(); for (key, sum) in &self.sums { @@ -124,8 +203,16 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + let mut counts_obj = serde_json::Map::new(); + for (key, count) in &self.counts { + let key_str = serde_json::to_string(&key.serialize_to_json()).unwrap(); + counts_obj.insert(key_str, Value::from(*count)); + } + serde_json::json!({ - "sums": sums_obj + "family": if self.family == ExactKind::Count { "Count" } else { "Sum" }, + "sums": sums_obj, + "counts": counts_obj }) } @@ -136,7 +223,9 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&(self.sums.len() as u32).to_le_bytes()); // Write each key-value pair + let mut ordered_keys = Vec::with_capacity(self.sums.len()); for (key, sum) in &self.sums { + ordered_keys.push(key); let key_bytes = key.serialize_to_bytes(); // Write key length and data @@ -147,17 +236,34 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&sum.to_le_bytes()); } + for key in ordered_keys { + buffer.extend_from_slice( + &self + .counts + .get(key) + .copied() + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + } + + buffer.push(if self.family == ExactKind::Count { + 1 + } else { + 0 + }); + buffer } } -impl AggregateCore for MultipleSumAccumulator { +impl AggregateCore for KeyedSumCountAccumulator { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleSumAccumulator" + "KeyedSumCountAccumulator" } fn as_any(&self) -> &dyn std::any::Any { @@ -172,20 +278,20 @@ impl AggregateCore for MultipleSumAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleSumAccumulator + // Check if other is also a KeyedSumCountAccumulator if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleSumAccumulator with {}", + "Cannot merge KeyedSumCountAccumulator with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleSumAccumulator + // Downcast to KeyedSumCountAccumulator let other_multiple_sum = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleSumAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedSumCountAccumulator")?; // Use the existing merge_accumulators method let merged = Self::merge_accumulators(vec![self.clone(), other_multiple_sum.clone()])?; @@ -194,13 +300,17 @@ impl AggregateCore for MultipleSumAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleSum + if self.family == ExactKind::Count { + AggregationType::Count + } else { + AggregationType::Sum + } } fn approx_memory_bytes(&self) -> usize { // HashMap. Label strings dominate; use a // conservative per-entry estimate plus HashMap overhead. - const BYTES_PER_ENTRY: usize = 96; + const BYTES_PER_ENTRY: usize = 112; std::mem::size_of::() + self.sums.len() * BYTES_PER_ENTRY } @@ -217,26 +327,35 @@ impl AggregateCore for MultipleSumAccumulator { use crate::storage_engines::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() - .ok_or("Key required for MultipleSumAccumulator")?; + .ok_or("Key required for KeyedSumCountAccumulator")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleSumAccumulator { +impl MultipleSubpopulationAggregate for KeyedSumCountAccumulator { fn query( &self, statistic: Statistic, key: &KeyByLabelValues, _query_kwargs: Option<&HashMap>, ) -> Result> { - match statistic { - Statistic::Sum | Statistic::Count => self - .sums + match (&self.family, statistic) { + (ExactKind::Sum, Statistic::Sum) => self.sums.get(key).copied().ok_or_else(|| { + "Key not found in KeyedSumCountAccumulator" + .to_string() + .into() + }), + (ExactKind::Count, Statistic::Count) => self + .counts .get(key) - .copied() - .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), + .map(|count| *count as f64) + .ok_or_else(|| { + "Sample count unavailable in KeyedSumCountAccumulator" + .to_string() + .into() + }), _ => Err( - format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), + format!("Unsupported statistic in KeyedSumCountAccumulator: {statistic:?}").into(), ), } } @@ -246,17 +365,41 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { } } -impl MergeableAccumulator for MultipleSumAccumulator { +impl MergeableAccumulator for KeyedSumCountAccumulator { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleSumAccumulator::new(); + let family = accumulators[0].family.clone(); + if accumulators.iter().any(|acc| acc.family != family) { + return Err("Cannot merge different keyed additive families".into()); + } + let mut result = KeyedSumCountAccumulator::for_family(family); for acc in accumulators { + for key in acc.sums.keys() { + match ( + result.counts.get(key).copied(), + acc.counts.get(key).copied(), + ) { + (None, Some(count)) if !result.sums.contains_key(key) => { + result.counts.insert(key.clone(), count); + } + (Some(existing), Some(count)) => { + if let Some(total) = existing.checked_add(count) { + result.counts.insert(key.clone(), total); + } else { + result.counts.remove(key); + } + } + _ => { + result.counts.remove(key); + } + } + } for (key, sum) in acc.sums { *result.sums.entry(key).or_insert(0.0) += sum; } @@ -273,14 +416,14 @@ mod tests { use super::*; #[test] - fn test_multiple_sum_accumulator_creation() { - let acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_creation() { + let acc = KeyedSumCountAccumulator::new(); assert!(acc.sums.is_empty()); } #[test] - fn test_multiple_sum_accumulator_update() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_update() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -295,8 +438,37 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_query() { - let mut acc = MultipleSumAccumulator::new(); + fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut first = KeyedSumCountAccumulator::for_family(ExactKind::Count); + first.update(key.clone(), 10.0); + first.update(key.clone(), 20.0); + let mut second = KeyedSumCountAccumulator::for_family(ExactKind::Count); + second.update(key.clone(), 7.0); + let merged = KeyedSumCountAccumulator::merge_accumulators(vec![first, second]).unwrap(); + for acc in [ + merged.clone(), + KeyedSumCountAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + ] { + assert_eq!(acc.family, ExactKind::Count); + assert!(acc.query(Statistic::Sum, &key, None).is_err()); + assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); + } + } + + #[test] + fn keyed_additive_merge_rejects_different_planner_families() { + assert!(KeyedSumCountAccumulator::merge_accumulators(vec![ + KeyedSumCountAccumulator::for_family(ExactKind::Sum), + KeyedSumCountAccumulator::for_family(ExactKind::Count), + ]) + .is_err()); + } + + #[test] + fn test_keyed_sum_count_accumulator_query() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -315,8 +487,8 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_get_keys() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_get_keys() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -332,9 +504,9 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_merge() { - let mut acc1 = MultipleSumAccumulator::new(); - let mut acc2 = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_merge() { + let mut acc1 = KeyedSumCountAccumulator::new(); + let mut acc2 = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -345,15 +517,15 @@ mod tests { acc2.add_sum(key1.clone(), 5.0); // Same key, different accumulator - let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.sums.get(&key1), Some(&15.0)); // Should be merged assert_eq!(merged.sums.get(&key2), Some(&20.0)); // Should be preserved } #[test] - fn test_multiple_sum_accumulator_serialization() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_serialization() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -361,18 +533,18 @@ mod tests { // Test JSON serialization let json = acc.serialize_to_json(); - let deserialized = MultipleSumAccumulator::deserialize_from_json(&json).unwrap(); + let deserialized = KeyedSumCountAccumulator::deserialize_from_json(&json).unwrap(); assert_eq!(deserialized.sums.get(&key), Some(&42.5)); // Test byte serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = MultipleSumAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedSumCountAccumulator::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.sums.get(&key), Some(&42.5)); } #[test] fn test_trait_object() { - let mut acc = MultipleSumAccumulator::new(); + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -381,6 +553,6 @@ mod tests { let trait_obj: Box = Box::new(acc); // Test type name through trait object - assert_eq!(trait_obj.type_name(), "MultipleSumAccumulator"); + assert_eq!(trait_obj.type_name(), "KeyedSumCountAccumulator"); } } diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index af284459e..073db6e82 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -4,16 +4,16 @@ pub mod count_sketch_accumulator; pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; -pub mod edge_runtime_adapter; +pub mod exact_accumulator; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; +pub mod keyed_counter_state; +pub mod keyed_max_state; +pub mod keyed_min_state; +pub mod keyed_sum_count_accumulator; pub mod max_accumulator; pub mod min_accumulator; -pub mod multiple_increase_accumulator; -pub mod multiple_max_accumulator; -pub mod multiple_min_accumulator; -pub mod multiple_sum_accumulator; pub mod sketch_envelope_accumulator; pub mod sum_accumulator; pub mod univmon_accumulator; @@ -27,11 +27,11 @@ pub use dd_sketch_accumulator::*; pub use hll_sketch_accumulator::*; pub use hydra_kll_accumulator::*; pub use increase_accumulator::*; +pub use keyed_counter_state::*; +pub use keyed_max_state::*; +pub use keyed_min_state::*; +pub use keyed_sum_count_accumulator::*; pub use max_accumulator::*; pub use min_accumulator::*; -pub use multiple_increase_accumulator::*; -pub use multiple_max_accumulator::*; -pub use multiple_min_accumulator::*; -pub use multiple_sum_accumulator::*; pub use sketch_envelope_accumulator::*; pub use sum_accumulator::*; diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 2cbe66fe5..d5ff3b02c 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -197,7 +197,11 @@ impl SingleSubpopulationAggregate for SumAccumulator { } match statistic { - Statistic::Sum | Statistic::Count => Ok(self.sum), + Statistic::Sum => Ok(self.sum), + Statistic::Count => self + .observation_count + .map(|count| count as f64) + .ok_or_else(|| "sample count is unavailable for this Sum payload".into()), _ => Err(format!("Unsupported statistic in SumAccumulator: {statistic:?}").into()), } } @@ -308,10 +312,7 @@ mod tests { crate::SingleSubpopulationAggregate::query(&acc, Statistic::Sum, None).unwrap(), 42.0 ); - assert_eq!( - crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), - 42.0 - ); + assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).is_err()); assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Min, None).is_err()); // SumAccumulator is a single subpopulation accumulator, doesn't need key-based queries @@ -321,6 +322,17 @@ mod tests { ); } + #[test] + fn count_readout_uses_observation_count_not_sum() { + let mut acc = SumAccumulator::new(); + acc.update(10.0); + acc.update(20.0); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), + 2.0 + ); + } + #[test] fn test_sum_accumulator_merge() { let acc1 = SumAccumulator::with_sum(10.0); diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index a710e57a9..940df03e0 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -112,7 +112,7 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `AggregationConfig` via + /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via /// `PolicyRegistry::get(output.policy_fp)`. The legacy /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` /// is the only identity handle on `PrecomputedOutput`. Outputs @@ -208,7 +208,7 @@ impl SketchStoreSink { return false; }; let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: output.policy_fp.into(), + stored_output_id: output.policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values, }; @@ -338,7 +338,7 @@ mod tests { use crate::precompute_engine::operators::{DDSketchAccumulator, SumAccumulator}; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -384,11 +384,13 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> AggregationConfig { + fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers obtain the id // via `config.policy_fp_u64()`. - AggregationConfig { + PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs new file mode 100644 index 000000000..75bdad529 --- /dev/null +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -0,0 +1,284 @@ +//! Bind raw ingestion to a selected Planner producer and its raw dependency edge. +use super::accumulator_factory::{create_planner_accumulator, AccumulatorUpdater}; +use crate::storage_engines::types::KeyByLabelValues; +use asap_types::{executable_plan::BackendNodeBinding, PrecomputeMaterialization}; +use planner_types::post_asap::{ + EdgeRole, ExecutableOperatorPayload, GroupingStrategy, PostAsapNodeId, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; +use planner_types::pre_asap::{ColumnRef, QueryExpr, Source}; +use std::collections::HashMap; + +/// A validated executable projection; semantics come from the installed node. +/// The retained node ID makes failures attributable to the selected DAG. +#[derive(Debug, Clone)] +pub struct RawDagProgram { + pub node: PostAsapNodeId, + pub family: SummaryFamilyType, + pub input: SummaryUpdate, + pub grouping: GroupingStrategy, + pub reduction: planner_types::pre_asap::Reduction, + projected_column: Option, +} + +impl RawDagProgram { + pub fn from_plan( + plan: &asap_types::precompute_plan::PrecomputePlan, + config: &PrecomputeMaterialization, + ) -> Result { + let mut selected: Option = None; + for installed in plan.executable_dags.values() { + installed.validate()?; + let dag = installed.document.decode()?; + for node in &dag.nodes { + if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { stored_output }) if stored_output.fingerprint() == config.policy_fingerprint()) + { + continue; + } + let ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + reduction, + } = &node.payload + else { + return Err( + "raw materialization binding must identify a Planner SummaryAgg".into(), + ); + }; + if config.derived_input.is_some() { + return Err("derived producer must execute through maintenance DAG".into()); + } + let incoming: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node.id).collect(); + let [edge] = incoming.as_slice() else { + return Err("raw SummaryAgg must have exactly one DAG input".into()); + }; + if edge.role != EdgeRole::Input { + return Err("raw SummaryAgg input edge has wrong role".into()); + } + let source = dag + .nodes + .iter() + .find(|n| n.id == edge.producer) + .ok_or("missing raw DAG input")?; + let ExecutableOperatorPayload::Fallback { expression } = &source.payload else { + return Err("raw producer requires an executable source input; maintenance edges cannot be bypassed".into()); + }; + let scan = match expression { + QueryExpr::TimeRange { child, .. } => child.as_ref(), + source => source, + }; + match scan { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + .. + } if metric == &config.metric => { + let (metric, window, filter) = + control_plane::physical::compiler::raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + )?; + if metric != config.metric + || window.is_some_and(|seconds| seconds != config.window_size) + || asap_types::utils::normalize_spatial_filter(&filter) + != config.spatial_filter_normalized + { + return Err( + "raw DAG source filter/window differs from physical binding".into(), + ); + } + } + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if config.table_name.as_ref() == Some(table_ref) => { + return Err( + "raw table execution requires a validated table scan executor".into(), + ); + } + _ => return Err("raw DAG input does not match installed source routing".into()), + } + if let planner_types::pre_asap::Reduction::Reduce(keys) = reduction { + if keys.is_without() { + return Err( + "raw without reduction requires explicit dynamic population routing" + .into(), + ); + } + let names = keys + .keys() + .iter() + .map(|id| { + source + .output_schema + .fields + .get(*id) + .map(|f| f.name.clone()) + .ok_or("missing reduction column") + }) + .collect::, _>>()?; + if names != config.grouping_labels.names() { + return Err("DAG reduction differs from physical population binding".into()); + } + } + if let SummaryFamilyType::ExactAggregate(kind, _) = family { + if input.item.is_some() { + return Err( + "raw exact populations must follow Planner reduction, not an item map" + .into(), + ); + } + if !matches!(input.weight, SummaryInputExpr::Column(_)) + && !(matches!(kind, planner_types::post_asap::ExactKind::Count) + && input.weight == SummaryInputExpr::Constant(1.0)) + { + return Err("raw exact update differs from stored source projection".into()); + } + } + if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { + return Err( + "materialization storage family differs from selected Planner node".into(), + ); + } + // The stored descriptor must name the same update semantics; its + // content identity cannot be reused for an unrelated DAG program. + let update_matches = match (&input.weight, config.sample_update_rule()) { + ( + SummaryInputExpr::Column(_), + asap_types::SampleUpdateRule::Value { scale }, + ) => scale == 1.0, + (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { + *value == 1.0 + } + _ => { + asap_types::accumulator_spec::is_unit_sample_frequency(input) + || (matches!( + family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ) + ) && input.weight == SummaryInputExpr::Constant(1.0)) + } + }; + if !update_matches { + return Err("DAG update differs from stored summary identity".into()); + } + let program = Self { + node: node.id, + family: family.clone(), + input: input.clone(), + grouping: grouping.clone(), + reduction: reduction.clone(), + projected_column: config + .effective_value_projection() + .column() + .map(str::to_owned), + }; + program.validate()?; + if let Some(old) = &selected { + if old.family != program.family + || old.input != program.input + || old.grouping != program.grouping + || old.reduction != program.reduction + { + return Err( + "one stored definition is bound to incompatible Planner producers" + .into(), + ); + } + } else { + selected = Some(program); + } + } + } + selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) + } + + pub fn updater(&self) -> Result, String> { + create_planner_accumulator(&self.family, &self.input, &self.grouping) + } + + fn validate(&self) -> Result<(), String> { + match &self.input.weight { + SummaryInputExpr::Column(ColumnRef::SampleValue) | SummaryInputExpr::Constant(_) => {} + SummaryInputExpr::Column( + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }, + ) if self.projected_column.as_ref() == Some(name) => {} + _ => return Err("raw DAG weight expression is unsupported".into()), + } + fn item(expr: &SummaryInputExpr) -> bool { + match expr { + SummaryInputExpr::Column(ColumnRef::Named(_) | ColumnRef::SampleValue) => true, + SummaryInputExpr::Tuple(items) => items.iter().all(item), + SummaryInputExpr::EntityIdentity( + planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + ) => excluding.is_empty(), + _ => false, + } + } + if self.input.item.as_ref().is_some_and(|e| !item(e)) { + return Err("raw DAG item expression is unsupported".into()); + } + self.updater().map(|_| ()) + } + + pub fn uses_counter_delta(&self) -> bool { + false + } + + pub fn apply( + &self, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + ) -> Result<(), String> { + let weight = match &self.input.weight { + SummaryInputExpr::Constant(c) => *c, + // The worker retains one previous value per series across pane rotation. + SummaryInputExpr::Column(_) => value, + _ => return Err("unsupported raw weight expression".into()), + }; + let scalar_frequency = asap_types::accumulator_spec::is_unit_sample_frequency(&self.input) + && !updater.is_keyed(); + let weight = if scalar_frequency { value } else { weight }; + updater.validate_single_input(weight)?; + if updater.is_keyed() { + let labels = super::worker::parse_labels_from_series_key(series); + fn eval( + expr: &SummaryInputExpr, + series: &str, + value: f64, + labels: &HashMap<&str, &str>, + ) -> Result, String> { + Ok(match expr { + SummaryInputExpr::EntityIdentity(_) => vec![series.to_owned()], + SummaryInputExpr::Column(ColumnRef::SampleValue) => vec![value.to_string()], + SummaryInputExpr::Column(ColumnRef::Named(name)) => vec![labels + .get(name.as_str()) + .map(|s| super::worker::decode_label_value(s).into_owned()) + .ok_or_else(|| format!("missing DAG item column {name}"))?], + SummaryInputExpr::Tuple(items) => items + .iter() + .map(|i| eval(i, series, value, labels)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + _ => return Err("unsupported raw item expression".into()), + }) + } + let item = self + .input + .item + .as_ref() + .ok_or("keyed DAG kernel requires an explicit item")?; + let key = KeyByLabelValues::new_with_labels(eval(item, series, value, &labels)?); + updater.update_keyed(&key, weight, timestamp); + } else { + updater.update_single(weight, timestamp); + } + Ok(()) + } +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 751f47e5f..01af62314 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -20,7 +20,7 @@ use xxhash_rust::xxh64::xxh64; /// hashing or pane lookup. `group_key` and `policy_fp` still travel /// alongside the sid: `group_key` is consumed at emit-time to render the /// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `AggregationConfig` from the hot-reload snapshot (window +/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window /// shape, late-data policy, etc.). Together they let the worker key state /// by sid without losing the data the legacy `(agg_id, group_key)` shape /// carried. @@ -50,8 +50,8 @@ pub enum WorkerMessage { /// `(metric, attrs_fingerprint, agg_kind_canonical)` — see /// `SeriesIdResolver::resolve`. Worker keys `group_states` on this. sid: u64, - /// Source `AggregationConfig` fingerprint. Worker looks up its - /// `AggregationConfig` (window size, sketch kind/config, late + /// Source `PrecomputeMaterialization` fingerprint. Worker looks up its + /// `PrecomputeMaterialization` (window size, sketch kind/config, late /// data policy, etc.) via `snap.get_aggregation_config(policy_fp.as_u64())`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons (e.g. "constant"). @@ -78,7 +78,7 @@ pub enum WorkerMessage { AccumulatorInput { /// Registry-allocated bucket identity; see `GroupSamples::sid`. sid: u64, - /// Source `AggregationConfig` fingerprint; see + /// Source `PrecomputeMaterialization` fingerprint; see /// `GroupSamples::policy_fp`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons, matching the diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 4cbd57a66..660ef8906 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -12,7 +12,7 @@ use std::{ pub struct MaterializationCommitKey { pub plan_id: u64, pub plan_version: u64, - pub summary_definition: asap_types::sds::SummaryDefinitionId, + pub stored_output: asap_types::sds::StoredOutputId, pub window_start_ms: i64, pub window_end_ms: i64, /// Producer lineage identity, including source and immutable input payload. @@ -67,14 +67,16 @@ where R: PrecomputeOperatorRegistry, S: IdempotentCommitSink, { - if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { summary_definition }) if *summary_definition == key.summary_definition) + if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { stored_output }) if *stored_output == key.stored_output) { return Err(ScheduleError::Invalid(format!( "commit key materialization {:?} does not match sink {}", - key.summary_definition, sink_node.0 + key.stored_output, sink_node.0 ))); } - binding.validate(dag).map_err(ScheduleError::Invalid)?; + binding + .validate_maintenance(dag) + .map_err(ScheduleError::Invalid)?; if !binding.precompute_sinks.contains(&sink_node) || !matches!( binding.node(sink_node), @@ -151,7 +153,7 @@ where let node = nodes .get(&id) .ok_or_else(|| ScheduleError::Invalid(format!("missing node {id}")))?; - if node.output_state == ExecutionDataState::READ_ROWS { + if node.output_state == ExecutionDataState::QUERY_ROWS { return Err(ScheduleError::Invalid(format!( "query-time node {id} in precompute dependency path" ))); @@ -208,8 +210,7 @@ mod tests { ( PostAsapNodeId(id), BackendNodeBinding::Materialization { - summary_definition: asap_types::PolicyFingerprint(u64::from(id) + 1) - .into(), + stored_output: asap_types::PolicyFingerprint(u64::from(id) + 1).into(), }, ) }) @@ -226,11 +227,32 @@ mod tests { } } + fn maintenance_only( + mut dag: ExecutableDag, + mut binding: BackendExecutableBinding, + sink: PostAsapNodeId, + ) -> (ExecutableDag, BackendExecutableBinding) { + let retained = dag + .nodes + .iter() + .filter(|node| { + node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime + }) + .map(|node| node.id) + .collect::>(); + dag.nodes.retain(|node| retained.contains(&node.id)); + dag.edges + .retain(|edge| retained.contains(&edge.producer) && retained.contains(&edge.consumer)); + binding.nodes.retain(|id, _| retained.contains(id)); + dag.root = sink; + (dag, binding) + } + fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), payload: ExecutableOperatorPayload::SummarySubtract, - output_state: ExecutionDataState::MAINTENANCE_SUMMARY, + output_state: ExecutionDataState::INGESTION_SUMMARY, output_schema: SummarySchema { fields: Vec::new(), time_index: None, @@ -248,7 +270,7 @@ mod tests { fields: Vec::new(), time_index: None, }, - data_state: ExecutionDataState::MAINTENANCE_SUMMARY, + data_state: ExecutionDataState::INGESTION_SUMMARY, grouping: GroupingEdgeCompatibility::Identical, window: WindowEdgeCompatibility::NotApplicable, } @@ -295,7 +317,7 @@ mod tests { MaterializationCommitKey { plan_id: 7, plan_version: 1, - summary_definition: asap_types::PolicyFingerprint(u64::from(node_id) + 1).into(), + stored_output: asap_types::PolicyFingerprint(u64::from(node_id) + 1).into(), window_start_ms: 10, window_end_ms: 20, input_lineage: b"checkpoint:3".to_vec(), @@ -324,7 +346,6 @@ mod tests { } let mut binary = node(3); binary.payload = ExecutableOperatorPayload::Binary { - timing: planner_types::post_asap::ExecutionTiming::MaintenanceTime, operator: BinaryOperator { checked_relative_division: false, checked_finite_division: false, @@ -339,16 +360,17 @@ mod tests { let mut dag = ExecutableDag { nodes: vec![node(0), node(1), node(2), binary, { let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; query }], edges: vec![right, left], root: PostAsapNodeId(3), }; let execute = |dag: &ExecutableDag| { + let (dag, binding) = maintenance_only(dag.clone(), binding(), PostAsapNodeId(3)); execute_precompute_sink( - dag, - &binding(), + &dag, + &binding, PostAsapNodeId(3), key(3), &Subtract, @@ -374,7 +396,7 @@ mod tests { .map(node) .chain([{ let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; query }]) .collect(), @@ -383,27 +405,16 @@ mod tests { }; let registry = Registry::default(); let sink = Sink::default(); - let first = execute_precompute_sink( - &dag, - &binding(), - PostAsapNodeId(3), - key(3), - ®istry, - &sink, - ) - .unwrap(); + let (dag, binding) = maintenance_only(dag, binding(), PostAsapNodeId(3)); + let first = + execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) + .unwrap(); assert_eq!(*first, 6); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); - let replay = execute_precompute_sink( - &dag, - &binding(), - PostAsapNodeId(3), - key(3), - ®istry, - &sink, - ) - .unwrap(); + let replay = + execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) + .unwrap(); assert!(Arc::ptr_eq(&first, &replay)); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); } @@ -430,9 +441,9 @@ mod tests { } } let mut raw = node(0); - raw.output_state = ExecutionDataState::READ_ROWS; + raw.output_state = ExecutionDataState::QUERY_ROWS; let mut query = node(4); - query.output_state = ExecutionDataState::READ_ROWS; + query.output_state = ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![raw, node(1), node(2), node(3), query], edges: vec![edge(0, 1), edge(1, 2), edge(1, 3), edge(2, 3), edge(3, 4)], @@ -442,6 +453,7 @@ mod tests { bindings .nodes .insert(PostAsapNodeId(0), BackendNodeBinding::QueryInput); + let (dag, bindings) = maintenance_only(dag, bindings, PostAsapNodeId(3)); let registry = FrontierRegistry(Registry::default()); let sink = Sink::default(); let result = @@ -457,7 +469,7 @@ mod tests { #[test] fn rejects_query_node_in_precompute_path_and_mismatched_lineage_key() { let mut query_child = node(0); - query_child.output_state = ExecutionDataState::READ_ROWS; + query_child.output_state = ExecutionDataState::QUERY_ROWS; let dag = ExecutableDag { nodes: vec![query_child, node(1)], edges: vec![edge(0, 1)], @@ -476,7 +488,7 @@ mod tests { ( PostAsapNodeId(1), BackendNodeBinding::Materialization { - summary_definition: asap_types::PolicyFingerprint(2).into(), + stored_output: asap_types::PolicyFingerprint(2).into(), }, ), ] @@ -488,7 +500,7 @@ mod tests { }; assert!(matches!( execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink), - Err(ScheduleError::Invalid(message)) if message.contains("query-time node") + Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") )); assert!(matches!( execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(0), ®istry, &sink), diff --git a/data_plane/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs index afccd0169..e1214fd9b 100644 --- a/data_plane/src/precompute_engine/window_manager.rs +++ b/data_plane/src/precompute_engine/window_manager.rs @@ -18,7 +18,7 @@ pub struct WindowManager { impl WindowManager { /// Create a new WindowManager. /// - /// `window_size_secs` and `slide_interval_secs` come from `AggregationConfig` + /// `window_size_secs` and `slide_interval_secs` come from `PrecomputeMaterialization` /// (which stores them in seconds). They are converted to milliseconds internally. pub fn new(window_size_secs: u64, slide_interval_secs: u64) -> Self { Self::with_origin(window_size_secs, slide_interval_secs, None) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 2c48006f4..847245e13 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,6 +1,6 @@ -use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, -}; +#[cfg(test)] +use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; +use crate::precompute_engine::accumulator_factory::AccumulatorUpdater; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::group_key::GroupKey; use crate::precompute_engine::metrics::record_late_input; @@ -11,7 +11,7 @@ use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; @@ -37,10 +37,11 @@ use tracing::{debug, debug_span, info, warn}; /// producing one output per (sid, window) — exactly like Arroyo's /// `GROUP BY window, key`. struct GroupState { + program: Option>, series_id: u64, catalog_generation: Option>, input_revisions: BTreeMap>, - config: Arc, + config: Arc, /// Source policy fingerprint that minted this sid. Held so /// `evict_orphaned_groups` can check liveness against the streaming /// config snapshot (a sid stays alive only while its source policy is @@ -411,7 +412,7 @@ impl Worker { /// /// B7.6 — buckets are now keyed by `sid` (a single u64) rather than /// `(agg_id, group_key)`. `policy_fp` is the source config's - /// fingerprint, used to fetch the `AggregationConfig` from the + /// fingerprint, used to fetch the `PrecomputeMaterialization` from the /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// @@ -425,12 +426,16 @@ impl Worker { sid: u64, policy_fp: PolicyFingerprint, group_key: &Arc, - ) -> Option<&mut GroupState> { + ) -> Result, String> { if !self.group_states.contains_key(&sid) { let snap = self.hot_reload.snapshot(); - let cfg = snap.get_aggregation_config(policy_fp.as_u64())?; + let Some(cfg) = snap.get_aggregation_config(policy_fp.as_u64()) else { + return Ok(None); + }; + let program = snap.raw_programs.get(&policy_fp.as_u64()).cloned(); let config = Arc::new(cfg.clone()); let gs = GroupState { + program, series_id: sid, catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), @@ -454,7 +459,7 @@ impl Worker { self.group_count .store(self.group_states.len(), Ordering::Relaxed); } - self.group_states.get_mut(&sid) + Ok(self.group_states.get_mut(&sid)) } /// Process a batch of samples for a specific sid bucket. @@ -462,7 +467,7 @@ impl Worker { /// /// This is the core of the Arroyo-equivalent GROUP BY logic. /// B7.6 — buckets are keyed by `sid`; `policy_fp` is the source - /// `AggregationConfig` fingerprint used to resolve the bucket's + /// `PrecomputeMaterialization` fingerprint used to resolve the bucket's /// config on first sight; `group_key` is held on the resulting /// `GroupState` for emit-time label rendering. pub fn process_group_samples( @@ -479,7 +484,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -489,6 +494,10 @@ impl Worker { return Ok(()); } let state = self.group_states.get_mut(&sid).unwrap(); + #[cfg(not(test))] + if state.program.is_none() { + return Err("raw precompute requires an installed post-ASAP DAG producer".into()); + } // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. @@ -537,12 +546,19 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = - if let SampleUpdateRule::CounterDelta { .. } = state.config.sample_update_rule() { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), + ) { + reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) + } else { + Some(*val) + }; for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -589,9 +605,14 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } + if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), ) { if let Some(input) = state.input_revisions.get_mut(&bucket_start) { Arc::make_mut(input).first_revision = 0; @@ -600,15 +621,23 @@ impl Worker { continue; } record_late_input("append_correction", "raw_sample"); - let mut updater = create_accumulator_updater(&state.config); - apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + let mut updater = + installed_updater(state.program.as_deref(), &state.config)?; + apply_installed_sample( + state.program.as_deref(), + &mut *updater, + series_key, + *val, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { observer.observe( &revision.generation, asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: policy_fp.into(), + stored_output_id: policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: bucket_start, end_ms: bucket_end, @@ -646,18 +675,27 @@ impl Worker { // only closes an idle pane, not a long-running bulk ingest whose // records share one event timestamp. state.touch_pane(bucket_start, now_ms); - let updater = state - .active_panes - .entry(bucket_start) - .or_insert_with(|| create_accumulator_updater(&state.config)); + if let std::collections::btree_map::Entry::Vacant(entry) = + state.active_panes.entry(bucket_start) + { + entry.insert(installed_updater(state.program.as_deref(), &state.config)?); + } + let updater = state.active_panes.get_mut(&bucket_start).unwrap(); if let Some(value) = value { - apply_sample(&mut **updater, series_key, value, *ts, &state.config); + apply_installed_sample( + state.program.as_deref(), + &mut **updater, + series_key, + value, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { observer.observe( &revision.generation, asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: policy_fp.into(), + stored_output_id: policy_fp.into(), time_range: asap_types::sds::HalfOpenTimeRange { start_ms: bucket_start, end_ms: bucket_end, @@ -767,7 +805,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -1349,7 +1387,10 @@ pub fn extract_metric_name(series_key: &str) -> &str { /// aggregation config's `grouping_labels`. /// /// The series key format is: `metric_name{label1="val1",label2="val2",...}` -pub fn extract_key_from_series(series_key: &str, config: &AggregationConfig) -> KeyByLabelValues { +pub fn extract_key_from_series( + series_key: &str, + config: &PrecomputeMaterialization, +) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1492,19 +1533,61 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { std::borrow::Cow::Owned(out) } +fn installed_updater( + program: Option<&super::raw_dag::RawDagProgram>, + config: &PrecomputeMaterialization, +) -> Result, String> { + if let Some(program) = program { + return program.updater(); + } + #[cfg(test)] + { + Ok(create_fixture_accumulator(config)) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + +fn apply_installed_sample( + program: Option<&super::raw_dag::RawDagProgram>, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + config: &PrecomputeMaterialization, +) -> Result<(), String> { + if let Some(program) = program { + return program.apply(updater, series, value, timestamp); + } + #[cfg(test)] + { + apply_sample(updater, series, value, timestamp, config); + Ok(()) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + /// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. /// /// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted /// from the series' **aggregated_labels** — these are the labels that become /// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a MultipleSumAccumulator's HashMap). This matches the Arroyo SQL +/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL /// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +#[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) { if updater.is_keyed() { // Planner's PromQL Top-K item is the series identity. When no @@ -1529,7 +1612,7 @@ pub(crate) fn apply_sample( /// Convert a cumulative counter sample into a non-negative, reset-aware /// increment. Only the immediately preceding sample per series is retained; /// pane rotation therefore cannot lose the boundary increment. -fn reset_aware_counter_delta( +pub(crate) fn reset_aware_counter_delta( previous: &mut HashMap, series_key: &str, value: f64, @@ -1560,7 +1643,7 @@ fn reset_aware_counter_delta( /// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. fn extract_aggregated_key_from_series( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1792,7 +1875,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::storage_engines::types::StreamingConfig; @@ -1808,7 +1891,7 @@ mod tests { window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { make_agg_config_full( id, metric, @@ -1831,7 +1914,7 @@ mod tests { slide_secs: u64, grouping: Vec<&str>, aggregated: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers below build the // streaming-config map by reading `config.policy_fp_u64()` @@ -1841,7 +1924,7 @@ mod tests { } else { WindowKind::Sliding }; - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, agg_sub_type.to_string(), HashMap::new(), @@ -1861,7 +1944,7 @@ mod tests { } fn make_worker( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1871,7 +1954,7 @@ mod tests { } fn make_worker_with_lateness( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1900,12 +1983,12 @@ mod tests { } /// Build a fresh `StreamingConfigHandle` from a map of agg_id - /// → AggregationConfig. Worker::new takes this handle instead of - /// the old `HashMap>`. Tests use this + /// → PrecomputeMaterialization. Worker::new takes this handle instead of + /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( - configs: HashMap, + configs: HashMap, ) -> crate::storage_engines::types::StreamingConfigHandle { crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), @@ -1991,7 +2074,7 @@ mod tests { assert_eq!(output.start_timestamp as i64, *ts); assert_eq!(output.end_timestamp as i64, *ts); // Raw mode emits PolicyFingerprint::UNSET (no source - // AggregationConfig in the raw-mode fast path). The sink + // PrecomputeMaterialization in the raw-mode fast path). The sink // drops UNSET outputs with a warn — verified separately // via integration tests. assert!(output.policy_fp.is_unset()); @@ -2452,7 +2535,7 @@ mod tests { #[test] fn test_keyed_accumulator_aggregated_labels() { // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside MultipleSumAccumulator) + // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) let config = make_agg_config_full( 3, "cpu", @@ -2504,10 +2587,10 @@ mod tests { let (_output, acc) = &captured[0]; let ms_acc = acc .as_any() - .downcast_ref::() - .expect("should be MultipleSumAccumulator"); + .downcast_ref::() + .expect("should be KeyedSumCountAccumulator"); - // The MultipleSumAccumulator should have two internal keys: "A" and "B" + // The KeyedSumCountAccumulator should have two internal keys: "A" and "B" assert_eq!(ms_acc.sums.len(), 2, "two host keys inside one accumulator"); let mut found_a = false; @@ -2680,100 +2763,15 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn test_worker_from_streaming_config_yaml() { - let yaml = r#" -aggregations: -- aggregationType: SingleSubpopulation - aggregationSubType: Sum - labels: - grouping: [] - rollup: [] - aggregated: [] - metric: requests_total - parameters: {} - tumblingWindowSize: 10 - windowSize: 10 - windowType: tumbling - slideInterval: 0 - spatialFilter: '' -"#; - - let data: serde_yaml::Value = serde_yaml::from_str(yaml).expect("valid YAML"); - let streaming_config = - StreamingConfig::from_yaml_data(&data).expect("valid streaming config"); - - // PR 5: the streaming-config key is the policy fingerprint. - let agg_id = *streaming_config - .materializations() - .keys() - .next() - .expect("one agg"); - assert!(streaming_config.contains(agg_id)); - - let agg_configs = streaming_config.materializations().clone(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - let pf = PolicyFingerprint(agg_id); - let sid = 1_u64; - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(1_000, 3.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(5_000, 4.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(9_000, 5.0)]), - ) - .unwrap(); - assert_eq!(sink.len(), 0); - - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(10_000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - - let (output, acc) = &captured[0]; - let _ = agg_id; - assert!(!output.policy_fp.is_unset()); - assert_eq!(output.start_timestamp, 0); - assert_eq!(output.end_timestamp, 10_000); - - let sum_acc = acc - .as_any() - .downcast_ref::() - .expect("should be SumAccumulator"); - assert!( - (sum_acc.sum - 12.0).abs() < 1e-10, - "sum should be 3+4+5=12, got {}", - sum_acc.sum - ); + fn test_worker_rejects_flat_streaming_config_yaml() { + let data = + serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); + assert!(StreamingConfig::from_yaml_data(&data).is_err()); } #[test] fn test_extract_key_from_series() { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), @@ -3413,7 +3411,7 @@ aggregations: /// Build a worker with explicit wall-clock closure grace values. fn make_worker_with_wall_clock_policy( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, late_data_policy: LateDataPolicy, idle_grace_period_ms: i64, @@ -4278,3 +4276,184 @@ aggregations: ); } } + +#[cfg(test)] +mod dag_execution_tests { + use super::*; + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::precompute_engine::output_sink::CapturingOutputSink; + use crate::storage_engines::types::StreamingConfig; + use asap_types::query_plan::ExactReadout; + + fn plan(query: &str) -> control_plane::physical::compiler::CompiledPhysicalPlan { + let mut json: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut item = json["query_workload"]["repeating_queries"][0].clone(); + item["query"] = query.into(); + json["query_workload"]["repeating_queries"] = serde_json::json!([item]); + let snapshot = serde_json::from_value(json).unwrap(); + crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap() + } + + // A selected producer must govern updates, persisted family, and query readout. + #[test] + fn installed_dag_ingestion_persistence_and_readout() { + for (query, readout, answer) in [ + ( + "sum_over_time(asap_demo_gauge[5s])", + ExactReadout::Sum, + 54.0, + ), + ( + "count_over_time(asap_demo_gauge[5s])", + ExactReadout::Count, + 5.0, + ), + ("min_over_time(asap_demo_gauge[5s])", ExactReadout::Min, 3.0), + ( + "max_over_time(asap_demo_gauge[5s])", + ExactReadout::Max, + 20.0, + ), + ("rate(asap_demo_counter_total[5s])", ExactReadout::Rate, 5.5), + ( + "increase(asap_demo_counter_total[5s])", + ExactReadout::Increase, + 27.5, + ), + ] { + let plan = plan(query); + let config = plan + .precompute_plan + .materializations + .first() + .expect("ASAP producer required") + .clone(); + let fp = config.policy_fingerprint(); + let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); + let doc = serde_json::to_value(&streaming).unwrap(); + assert!(doc.get("aggregation_configs").is_none()); + let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let sink = Arc::new(CapturingOutputSink::new()); + let (_tx, rx) = mpsc::channel(8); + let mut worker = Worker::new( + 0, + rx, + sink.clone(), + StreamingConfigHandle::new(streaming), + WorkerRuntimeConfig { + max_buffer_per_series: 100, + allowed_lateness_ms: 10_000, + pass_raw_samples: false, + raw_mode_aggregation_id: 0, + late_data_policy: LateDataPolicy::Drop, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + }, + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicI64::new(0)), + ); + worker + .process_group_samples( + 1, + fp, + &Arc::new(GroupKey::new([])), + [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, v)| (config.metric.clone(), t, v)) + .collect(), + ) + .unwrap(); + worker.force_close_all().unwrap(); + let mut states = BTreeMap::new(); + for (output, state) in sink.drain() { + let select = if matches!( + config.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) { + output.start_timestamp == 0 && output.end_timestamp == 5000 + } else { + output.end_timestamp <= 5000 + }; + if select { + assert_eq!( + state.get_accumulator_type().planner_exact_family(), + Some(readout.planner_family()) + ); + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()) + .unwrap(); + states.insert( + output.end_timestamp as i64, + Arc::new(restored) as Arc, + ); + } + } + assert!(!states.is_empty(), "{query}: no stored states"); + let group = + crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { + entries: vec![std::rc::Rc::new(states)], + agg_type: config.aggregation_type, + }; + assert_eq!( + group.exact_value_for(readout, &None, 0, 5000), + Some(answer), + "{query}" + ); + } + } + + // A flat config and a DAG whose producer no longer matches its binding cannot install. + #[test] + fn execution_requires_matching_dag_producer() { + assert!(serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err()); + let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; + plan.executable_dags.clear(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("DAG producer")); + } + // Changing a raw update must not silently reuse the original summary identity. + #[test] + fn altered_dag_update_cannot_reuse_a_stored_definition() { + let mut plan = plan("sum_over_time(asap_demo_gauge[5s])").precompute_plan; + let installed = plan.executable_dags.values_mut().next().unwrap(); + let mut dag = installed.document.decode().unwrap(); + for node in &mut dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + input, .. + } = &mut node.payload + { + input.weight = planner_types::post_asap::SummaryInputExpr::Constant(99.0); + } + } + installed.document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( + installed.document.query_id.clone(), + &dag, + ) + .unwrap(); + installed.document.schema_version = + asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; + let error = StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string(); + assert!( + error.contains("update") || error.contains("semantic definition differs"), + "{error}" + ); + } +} diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index bc4d4741d..0ef02672d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -558,7 +558,7 @@ mod tests { config.pane_origin_ms = Some(0); config.table_timestamp_column = Some("timestamp_ms".into()); let sds = SummaryCatalog::from_materializations(41, 1, &[config.clone()]).unwrap(); - let materialization = *sds.materializations.keys().next().unwrap(); + let materialization = *sds.outputs.keys().next().unwrap(); let read = QueryNodeId(0); let readout = QueryNodeId(1); let input_schema = relation_schema(&[ @@ -590,6 +590,7 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization, + stored_output_reference: sds.output_reference(materialization).unwrap(), output_grouping: PhysicalGrouping::Reduce(Vec::new()), item_labels: Vec::new(), window_ms: 1_000, @@ -667,7 +668,7 @@ mod tests { root, QueryPlanNode::Relational { input: sort, - operation: serde_json::to_value(ValueOperation::Limit { n: 1, offset: 0 }) + operation: serde_json::to_value(ValueOperation::Limit { n: 1, offset: 0, partition_by: planner_types::pre_asap::GroupKeys::by(vec![]) }) .unwrap(), input_schema: projected_schema.clone(), output_schema: projected_schema, @@ -728,6 +729,7 @@ mod tests { tables, accuracy: planner_types::types::AccuracyTarget::Exact, }), + selected_dags: Default::default(), entries: BTreeMap::from([( QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &canonical_sql), entry, @@ -782,7 +784,7 @@ mod tests { &["fixture".into()], ) .unwrap(); - precompute.summary_catalog = Some(sds.reference().unwrap()); + precompute.bind_catalog(&sds).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope.clone(), &precompute, diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index c47b7b48e..8d673fff3 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -484,7 +484,16 @@ impl ClickHouseRelationalAdapter { .rows .sort_by(|left, right| compare_sort_keys(left, right, keys, &schema)); } - ValueOperation::Limit { n, offset } => { + ValueOperation::Limit { + n, + offset, + partition_by, + } => { + if !partition_by.keys().is_empty() || partition_by.is_without() { + return Err(ClickHouseRelationalError::Unsupported( + "partitioned Limit".into(), + )); + } input.rows = input.rows.into_iter().skip(*offset).take(*n).collect(); } other => return Err(ClickHouseRelationalError::Unsupported(format!("{other:?}"))), @@ -1604,7 +1613,11 @@ mod tests { }], partition_by: GroupKeys::none(), }, - ValueOperation::Limit { n: 1, offset: 0 }, + ValueOperation::Limit { + n: 1, + offset: 0, + partition_by: planner_types::pre_asap::GroupKeys::by(vec![]), + }, ] { relation = adapter .apply_operation(&operation, &projected_schema, relation) diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index ba07c0b9b..c227e5eb0 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -4,9 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; -use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; +use asap_types::sds::{StoredOutputId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; -use asap_types::AggregationType; +use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; use crate::query_engines::EngineError; @@ -21,10 +21,10 @@ fn miss(reason: impl Into) -> EngineError { /// Borrow descriptors; never reconstruct them from bindings or store payloads. pub(crate) fn resolve( catalog: &SummaryCatalog, - id: SummaryDefinitionId, + id: StoredOutputId, ) -> Result, EngineError> { let identity = catalog - .materializations + .outputs .get(&id) .ok_or_else(|| miss(format!("unknown materialization {}", id.fingerprint().0)))?; let summary = catalog @@ -51,64 +51,40 @@ impl ResolvedMaterialization<'_> { matches!( &self.summary.operator, SummaryOperator::Configured { - aggregation_type: AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease - | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleMin - | AggregationType::MultipleMax, + family: SummaryFamilyType::ExactAggregate(..), .. } ) } fn supports(&self, node: &QueryPlanNode) -> bool { - let SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - .. - } = &self.summary.operator - else { + let SummaryOperator::Configured { family, .. } = &self.summary.operator else { // Partial legacy descriptors cannot attest a configured capability. return false; }; - use AggregationType::*; match node { - QueryPlanNode::ExactReadout { readout, .. } => match readout { - ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Sum, - ExactReadout::Increase | ExactReadout::Rate => { - matches!(aggregation_type, Increase | MultipleIncrease) - } - // Direction is the family now -- no `aggregation_sub_type` - // cross-check, and a minimum summary can no longer be - // offered up for a maximum readout. - ExactReadout::Min => matches!(aggregation_type, Min | MultipleMin), - ExactReadout::Max => matches!(aggregation_type, Max | MultipleMax), - }, + QueryPlanNode::ExactReadout { readout, .. } => family == &readout.planner_family(), QueryPlanNode::SummaryEstimate { query, .. } => match query { QueryReadout::Quantile { q } => { q.is_finite() && (0.0..=1.0).contains(q) - && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) + } + QueryReadout::Cardinality => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon)) } - QueryReadout::Cardinality => matches!(aggregation_type, HLL | UnivMon), QueryReadout::FrequencyL2 | QueryReadout::FrequencyEntropy => { - *aggregation_type == UnivMon + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) } - QueryReadout::PointCount { value: None, .. } if *aggregation_type == UnivMon => { + QueryReadout::PointCount { value: None, .. } if matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) => { true } - QueryReadout::PointCount { .. } => matches!( - aggregation_type, - CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap - ), - QueryReadout::TopK { .. } => matches!( - aggregation_type, - CountMinSketchWithHeap | CountSketchWithHeap - ), + QueryReadout::PointCount { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap)) + } + QueryReadout::TopK { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + } }, _ => false, } @@ -155,7 +131,7 @@ pub(crate) fn validate_payload( QueryPlanNode::SummaryMerge { inputs } => { let mut ids = BTreeSet::new(); for input in inputs { - let children: &BTreeSet = states + let children: &BTreeSet = states .get(input) .ok_or_else(|| miss("summary merge has no state input"))?; if children.is_empty() { @@ -167,7 +143,7 @@ pub(crate) fn validate_payload( } QueryPlanNode::SummaryEstimate { input, .. } | QueryPlanNode::ExactReadout { input, .. } => { - let ids: &BTreeSet = states + let ids: &BTreeSet = states .get(input) .ok_or_else(|| miss("readout has no state input"))?; if ids.is_empty() || ids.iter().any(|id| !resolved[id].supports(node)) { @@ -256,11 +232,11 @@ mod tests { fn resolves_without_descriptor_copies() { let bundle = fixture(); let catalog = &bundle.summary_catalog; - let id = *catalog.materializations.keys().next().unwrap(); + let id = *catalog.outputs.keys().next().unwrap(); let result = resolve(catalog, id).unwrap(); assert!(std::ptr::eq( result.summary, - &catalog.summary_descriptors[&catalog.materializations[&id].summary_descriptor_id] + &catalog.summary_descriptors[&catalog.outputs[&id].summary_descriptor_id] )); let mut broken = catalog.clone(); broken.data_descriptors.clear(); @@ -270,8 +246,8 @@ mod tests { #[test] fn rejects_operator_fidelity_mismatch_during_resolution() { let mut catalog = catalog_fixture(); - let id = *catalog.materializations.keys().next().unwrap(); - let descriptor_id = catalog.materializations[&id].summary_descriptor_id.clone(); + let id = *catalog.outputs.keys().next().unwrap(); + let descriptor_id = catalog.outputs[&id].summary_descriptor_id.clone(); catalog .summary_descriptors .get_mut(&descriptor_id) diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index cae8ff229..cc6407ee8 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -286,7 +286,7 @@ impl ASAPQueryEngine { // Candidate-filtered exact cuts have a data dependency: read the // installed membership subtree once, then use that vector to build the // Prometheus selector. Keeping the result as a prepared leaf also means - // CandidateTopK reuses the same membership readout during composition. + // MembershipFilter reuses the same membership readout during composition. let dependencies = super::exact_subqueries::external_dependencies(entry, times)?; let mut prepared = super::logical_dag::PreparedLeaves::new(); let unique_inputs = dependencies diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index b5b822641..b24908b45 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -85,9 +85,9 @@ fn leaves( } _ => pending.extend(inputs.iter().map(|input| (*input, at))), }, - // CandidateTopK is a typed composition node rather than a Logical + // MembershipFilter is a typed composition node rather than a Logical // wrapper, but its value input can still be a Prometheus leaf. - QueryPlanNode::CandidateTopK { inputs, .. } => { + QueryPlanNode::MembershipFilter { inputs, .. } => { pending.extend(inputs.iter().map(|input| (*input, at))); } QueryPlanNode::ExternalExact { request, inputs } => { @@ -650,22 +650,30 @@ mod tests { } #[tokio::test] - async fn candidate_exact_is_discovered_and_prepared_behind_candidate_topk_root() { - use asap_types::query_plan::{residual::Grouping, CandidateCompleteness}; + async fn candidate_exact_is_discovered_and_prepared_behind_membership_filter_root() { + use asap_types::query_plan::CandidateCompleteness; let mut entry = candidate_entry("sum by (job) (rate(m[5m]))"); entry.nodes.insert( QueryNodeId(2), - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs: [QueryNodeId(1), QueryNodeId(0)], - k: 2, - grouping: Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::BestEffort { guarantee: None }, }, ); - entry.root = QueryNodeId(2); + entry.nodes.insert( + QueryNodeId(3), + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { + k: 2, + grouping: asap_types::query_plan::residual::Grouping { + labels: vec![], + without: false, + }, + }, + inputs: vec![QueryNodeId(2)], + }, + ); + entry.root = QueryNodeId(3); let dependencies = external_dependencies(&entry, &[1_000]).unwrap(); assert_eq!(dependencies, vec![(QueryNodeId(0), QueryNodeId(1), 1_000)]); let prepared = prepare_external( @@ -902,9 +910,9 @@ mod tests { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, @@ -960,6 +968,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: MATERIALIZATION.into(), + stored_output_reference: super::super::test_plan::bound_reference( + &store, + MATERIALIZATION.into(), + ), output_grouping: PhysicalGrouping::Reduce(vec!["job".into()]), window_ms: AT, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 577409c86..103247fdf 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -194,6 +194,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 4b77a5a8f..31ef9f7c8 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -205,16 +205,13 @@ impl Result> Evaluator<' } self.logical(operator, &inputs, at)? } - QueryPlanNode::CandidateTopK { + QueryPlanNode::MembershipFilter { inputs, - k, - grouping, completeness, } => { let candidates = vector(self.eval(inputs[0], at)?)?; let values = vector(self.eval(inputs[1], at)?)?; - let (selected, warning) = - candidate_topk(k, &grouping, candidates, values, &completeness)?; + let (selected, warning) = membership_filter(candidates, values, &completeness)?; if let Some(warning) = warning { self.warnings.push(warning); } @@ -423,9 +420,7 @@ impl Result> Evaluator<' } } -fn candidate_topk( - k: u64, - grouping: &Grouping, +fn membership_filter( candidates: Vector, values: Vector, completeness: &CandidateCompleteness, @@ -440,25 +435,22 @@ fn candidate_topk( .map(|(labels, _)| identity(labels)) .collect(); let value_ids: BTreeSet<_> = values.iter().map(|(labels, _)| identity(labels)).collect(); - let dangling = candidate_ids - .iter() - .any(|candidate| !value_ids.contains(candidate)); - if dangling && matches!(completeness, CandidateCompleteness::Certified { .. }) { - return Err(miss("certified TopK candidate has no exact counter value")); - } - let matched = values + let missing: BTreeSet<_> = candidate_ids.difference(&value_ids).collect(); + let selected = values .into_iter() .filter(|(labels, _)| candidate_ids.contains(&identity(labels))) .collect(); - let selected = topk_selection(k, grouping, matched); + if !missing.is_empty() && matches!(completeness, CandidateCompleteness::Certified { .. }) { + return Err(miss("certified membership key has no authoritative value")); + } let warning = match completeness { CandidateCompleteness::Certified { .. } => None, CandidateCompleteness::BestEffort { guarantee } => Some(match guarantee { Some(guarantee) => format!( - "ASAP TopK candidate membership is approximate: {:?}", + "ASAP membership pruning is approximate: {:?}", guarantee.metric ), - None => "ASAP TopK candidate membership is approximate and uncertified".into(), + None => "ASAP membership pruning is approximate and uncertified".into(), }), }; Ok((selected, warning)) @@ -908,6 +900,14 @@ mod topk_tests { (labels(&[("series", "high")]), 3.0), ], ); + let selected = topk_selection( + 2, + &Grouping { + labels: vec![], + without: false, + }, + selected, + ); assert_eq!( selected .iter() @@ -1177,12 +1177,7 @@ mod topk_tests { (labels(&[("pod", "b")]), 8.0), (labels(&[("pod", "c")]), 9.0), ]; - let (selected, warning) = candidate_topk( - 2, - &Grouping { - labels: vec![], - without: false, - }, + let (selected, warning) = membership_filter( candidates, exact, &CandidateCompleteness::Certified { @@ -1190,6 +1185,14 @@ mod topk_tests { }, ) .unwrap(); + let selected = topk_selection( + 2, + &Grouping { + labels: vec![], + without: false, + }, + selected, + ); assert_eq!( selected .iter() @@ -1204,7 +1207,8 @@ mod topk_tests { fn installed_candidate_sidecar_reads_both_summary_inputs() { let candidate_id = QueryNodeId(0); let value_id = QueryNodeId(1); - let root = QueryNodeId(2); + let filter = QueryNodeId(2); + let root = QueryNodeId(3); let entry = QueryPlanEntry { language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), @@ -1225,19 +1229,27 @@ mod topk_tests { }, ), ( - root, - QueryPlanNode::CandidateTopK { + filter, + QueryPlanNode::MembershipFilter { inputs: [candidate_id, value_id], - k: 1, - grouping: Grouping { - labels: vec![], - without: false, - }, completeness: CandidateCompleteness::Certified { guarantee: topk_membership_guarantee(), }, }, ), + ( + root, + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { + k: 1, + grouping: Grouping { + labels: vec![], + without: false, + }, + }, + inputs: vec![filter], + }, + ), ]), instant: InstantExecution { lookback_ms: 300_000, @@ -1251,7 +1263,10 @@ mod topk_tests { ( (candidate_id, at), PreparedLeaf { - value: Value::Vector(vec![(labels(&[("pod", "b")]), 100.0)]), + value: Value::Vector(vec![ + (labels(&[("pod", "b")]), 100.0), + (labels(&[("pod", "c")]), 1.0), + ]), remote: false, remote_evaluations: 0, remote_rpcs: 0, @@ -1263,6 +1278,7 @@ mod topk_tests { value: Value::Vector(vec![ (labels(&[("pod", "a")]), 2.0), (labels(&[("pod", "b")]), 1.0), + (labels(&[("pod", "c")]), 3.0), ]), remote: false, remote_evaluations: 0, @@ -1278,8 +1294,8 @@ mod topk_tests { panic!("vector expected") }; assert_eq!(result.values.len(), 1); - assert_eq!(result.values[0].value, 1.0, "exact value is authoritative"); - assert_eq!(result.values[0].labels.labels, vec!["b"]); + assert_eq!(result.values[0].value, 3.0, "exact value is authoritative"); + assert_eq!(result.values[0].labels.labels, vec!["c"]); assert_eq!(stats.summary_readout_evaluations, 2); assert!(result.warnings.is_empty()); } @@ -1288,30 +1304,20 @@ mod topk_tests { fn uncertified_candidate_sidecar_warns_or_falls_back_explicitly() { let candidates = vec![(labels(&[("pod", "a")]), 1.0)]; let exact = vec![(labels(&[("pod", "a")]), 2.0)]; - let (_, warning) = candidate_topk( - 1, - &Grouping { - labels: vec![], - without: false, - }, + let (_, warning) = membership_filter( candidates.clone(), exact.clone(), &CandidateCompleteness::BestEffort { guarantee: None }, ) .unwrap(); assert!(warning.unwrap().contains("approximate")); - // Exact queries never lower an uncertified CandidateTopK. The Planner + // Exact queries never lower an uncertified MembershipFilter. The Planner // emits its ordinary exact fallback instead; this runtime node is only // valid for certified or explicitly approximate plans. let certified = CandidateCompleteness::Certified { guarantee: topk_membership_guarantee(), }; - assert!(candidate_topk( - 1, - &Grouping { - labels: vec![], - without: false - }, + assert!(membership_filter( vec![(labels(&[("pod", "missing")]), 1.0)], exact, &certified, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 41b41ab77..2ea8c6229 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -154,8 +154,7 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { .catalog .as_ref() .and_then(|catalog| { - let definition = - catalog.materializations.get(&binding.materialization)?; + let definition = catalog.outputs.get(&binding.materialization)?; catalog .data_descriptors .get(&definition.data_descriptor_id)? @@ -306,7 +305,7 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { }) } QueryPlanNode::Logical { .. } - | QueryPlanNode::CandidateTopK { .. } + | QueryPlanNode::MembershipFilter { .. } | QueryPlanNode::Relational { .. } | QueryPlanNode::ExternalExact { .. } | QueryPlanNode::RelationalJoin { .. } => Err(PhysicalNodeError::Fallback( @@ -883,6 +882,10 @@ mod tests { binding: MaterializationBinding { full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + config.policy_fingerprint().into(), + ), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, @@ -1074,7 +1077,9 @@ mod tests { #[test] fn compiled_window_schedules_execute_exact_ranges() { use crate::precompute_engine::window_manager::WindowManager; - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, + }; for evaluation_secs in [20, 45, 60, 120, 90] { for phase_ms in [0, 5_000] { for full in [false, true] { @@ -1102,7 +1107,7 @@ mod tests { asap_types::WindowMaterializationLayout::FullWindow ) == full }); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; let manager = WindowManager::with_layout( config.window_size, @@ -1121,16 +1126,16 @@ mod tests { } } let idx = SketchStore::new(); + idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) + .unwrap(); idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config( + config, + ), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, @@ -1173,7 +1178,9 @@ mod tests { // Compile the two readouts, store one pane series, and execute the actual ratio. #[test] fn compiled_shared_sum_panes_preserve_each_lookback() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, + }; let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -1184,21 +1191,19 @@ mod tests { entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let config = &plan.precompute_plan.materializations[0]; let policy = config.policy_fingerprint(); let idx = SketchStore::new(); + idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) + .unwrap(); idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config(config), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, @@ -1314,6 +1319,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), @@ -1366,9 +1375,9 @@ mod tests { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, @@ -1411,6 +1420,10 @@ mod tests { full_window_slide_ms: None, item_labels: Vec::new(), materialization: policy.into(), + stored_output_reference: super::super::test_plan::bound_reference( + &idx, + policy.into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs index fb75ae6bc..803653718 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs @@ -184,7 +184,7 @@ pub fn execute( Ok(ExecOutcome::Value(out)) } - SummaryExpr::SummaryMerge { children } => { + SummaryExpr::SummaryMerge { children, .. } => { if children.is_empty() { return Err(ExecError::EmptyMerge); } @@ -223,11 +223,10 @@ pub fn execute( SummaryExpr::SummaryJoin { .. } => Err(ExecError::NotYetSupported("SummaryJoin")), SummaryExpr::RelationalJoin { .. } => Err(ExecError::NotYetSupported("RelationalJoin")), - // CandidateTopK is lowered to the deployed QueryPlan DAG, where both + // MembershipFilter is lowered to the deployed QueryPlan DAG, where both // row inputs retain labels for intersection and exact reranking. This // legacy generic adapter exposes opaque GroupKey values and cannot // implement that contract without losing label identity. - SummaryExpr::CandidateTopK { .. } => Err(ExecError::NotYetSupported("CandidateTopK")), SummaryExpr::BinaryOp { .. } => Err(ExecError::NotYetSupported("BinaryOp")), SummaryExpr::ValueOperation { .. } => Err(ExecError::NotYetSupported("ValueOperation")), SummaryExpr::SummarySubtract { .. } => Err(ExecError::NotYetSupported("SummarySubtract")), @@ -350,7 +349,10 @@ mod tests { fn merge_node(children: Vec>) -> Rc { Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { children }, + expr: SummaryExpr::SummaryMerge { + children, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, + }, schema: lift(vec!["value"]), guarantee: None, }) @@ -522,7 +524,7 @@ mod tests { let child = logical_node(); let tree = SummaryNode { expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::ReadTime, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, lhs: child.clone(), rhs: child.clone(), operator: planner_types::post_asap::BinaryOperator { diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index f48ba405d..e25ea4fcd 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -65,8 +65,8 @@ use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryNode, + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, }; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; @@ -203,11 +203,13 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match agg_type { - AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease => asap_types::Statistic::Sum, + let stat = match agg_type.planner_exact_family()? { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => asap_types::Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => asap_types::Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { + asap_types::Statistic::Increase + } + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -224,9 +226,7 @@ impl GroupState { .ok() } - /// Finalize a compiler-declared exact readout. Rate and increase share - /// reset-aware Increase state physically, but remain distinct operations - /// in QueryPlan so serving never infers semantics from PromQL text. + /// Finalize the Planner-declared exact family with its matching readout. pub fn exact_value_for( &self, readout: asap_types::query_plan::ExactReadout, @@ -237,40 +237,32 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { - asap_types::Statistic::Count - } - ( - asap_types::query_plan::ExactReadout::Sum, - AggregationType::Sum | AggregationType::MultipleSum, - ) => asap_types::Statistic::Sum, - ( - asap_types::query_plan::ExactReadout::Increase, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Increase, - ( - asap_types::query_plan::ExactReadout::Rate, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Rate, - ( - asap_types::query_plan::ExactReadout::Min, - AggregationType::Min | AggregationType::MultipleMin, - ) => asap_types::Statistic::Min, - ( - asap_types::query_plan::ExactReadout::Max, - AggregationType::Max | AggregationType::MultipleMax, - ) => asap_types::Statistic::Max, - _ => return None, + if agg_type.planner_exact_family().as_ref() != Some(&readout.planner_family()) { + return None; + } + let stat = match readout { + asap_types::query_plan::ExactReadout::Count => asap_types::Statistic::Count, + asap_types::query_plan::ExactReadout::Sum => asap_types::Statistic::Sum, + asap_types::query_plan::ExactReadout::Increase => asap_types::Statistic::Increase, + asap_types::query_plan::ExactReadout::Rate => asap_types::Statistic::Rate, + asap_types::query_plan::ExactReadout::Min => asap_types::Statistic::Min, + asap_types::query_plan::ExactReadout::Max => asap_types::Statistic::Max, }; + let planner_state = entries.iter().flat_map(|w| w.values()).any(|a| { + a.as_any() + .is::() + }); // Temporal exact summaries are the hot path for long-window // dashboards. Merge their concrete, fixed-size states in one batch // instead of allocating a boxed trait object for every pane. - if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease - ) { + if !planner_state + && matches!( + readout, + asap_types::query_plan::ExactReadout::Increase + | asap_types::query_plan::ExactReadout::Rate + ) + { let accumulators = entries .iter() .flat_map(|windows| windows.values()) @@ -286,11 +278,7 @@ impl GroupState { ]); return merged.query_statistic(stat, key, &query_kwargs).ok(); } - if matches!( - agg_type, - AggregationType::Min | AggregationType::MultipleMin - ) && readout == asap_types::query_plan::ExactReadout::Min - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Min { return entries .iter() .flat_map(|windows| windows.values()) @@ -303,11 +291,7 @@ impl GroupState { .into_iter() .reduce(f64::min); } - if matches!( - agg_type, - AggregationType::Max | AggregationType::MultipleMax - ) && readout == asap_types::query_plan::ExactReadout::Max - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() .flat_map(|windows| windows.values()) @@ -334,9 +318,6 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == asap_types::query_plan::ExactReadout::Count { - return merged.aux_stats().count.map(|count| count as f64); - } merged.query_statistic(stat, key, &query_kwargs).ok() } @@ -444,14 +425,37 @@ fn validate_binding_phase( impl QueryExecutionContext<'_> { /// Resolve exactly one compiler-bound materialization. This is the formal - /// QueryPlan path: fingerprint -> SID is the only lookup; metadata checks - /// are integrity checks and never broaden the candidate set. + /// QueryPlan path: the validated stored output resolves to its definition's + /// generation-scoped SID index. Metadata checks never broaden that set. pub fn read_bound_materialization( &self, binding: &asap_types::query_plan::MaterializationBinding, ) -> Result, GroupState)>, SummaryExecutorError> { use asap_types::query_plan::PhysicalGrouping; + if binding.stored_output_reference.validate().is_err() + || binding.stored_output_reference.stored_output_id != binding.materialization + { + return Err(SummaryExecutorError::Unsupported( + "read binding has invalid stored output", + )); + } + + let catalog = + self.index + .summary_catalog_snapshot() + .ok_or(SummaryExecutorError::Unsupported( + "bound read requires installed SDS definitions", + ))?; + let expected = catalog + .output_reference(binding.materialization) + .map_err(|_| SummaryExecutorError::Unsupported("stored output is not installed"))?; + if binding.stored_output_reference != expected { + return Err(SummaryExecutorError::Unsupported( + "bound read semantic identity differs from installed output", + )); + } + let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { start_ms: i64::try_from(self.t0_ms).map_err(|_| { @@ -590,9 +594,13 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + let exact_family = agg_type.planner_exact_family(); if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Increase | ExactKind::Rate, + _ + )) ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -608,12 +616,12 @@ impl QueryExecutionContext<'_> { )); } } - if let Some((reduction, is_min)) = match agg_type { - AggregationType::Min | AggregationType::MultipleMin => Some(( + if let Some((reduction, is_min)) = match exact_family.as_ref() { + Some(SummaryFamilyType::ExactAggregate(ExactKind::Min, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Min, true, )), - AggregationType::Max | AggregationType::MultipleMax => Some(( + Some(SummaryFamilyType::ExactAggregate(ExactKind::Max, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Max, false, )), @@ -665,8 +673,11 @@ impl QueryExecutionContext<'_> { // counter and extrema state. Additive pane summaries must // remain contiguous because a missing pane is not zero. if matches!( - agg_type, - AggregationType::Sum | AggregationType::MultipleSum + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count, + _ + )) ) { check_panes(windows.keys().copied().collect())?; } @@ -911,9 +922,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } ( - GroupState::ExactAgg { entries, .. }, - GroupState::ExactAgg { entries: more, .. }, + GroupState::ExactAgg { entries, agg_type }, + GroupState::ExactAgg { + entries: more, + agg_type: incoming, + }, ) => { + if agg_type.planner_exact_family() != incoming.planner_exact_family() { + return Err(SummaryExecutorError::UnsupportedFamily); + } entries.extend(more); } // `find_candidates`'s exact-match contract never produces a @@ -1255,29 +1272,19 @@ fn summary_family_matches_sketch( /// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> /// ExactKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease -/// -> ExactKind::Increase` — confirmed against that module's own -/// dispatch table rather than invented here). +/// side. Count and Rate remain distinct families even though their runtime +/// accumulators share implementations with Sum and Increase. /// -/// `ExactKind::Count`/`Rate`/`Min`/`Max` are not matched by this legacy -/// family-discovery path. For `Count`/`Rate` the final operation is ambiguous -/// from the stored accumulator alone. `Min`/`Max` were excluded for a reason -/// that no longer holds -- direction used to be unrecoverable once a summary -/// reached `AggKind::ExactAgg`, and is now the family itself -- but admitting -/// them here widens candidate discovery beyond the family split and is left -/// as follow-up. Installed QueryPlans carry an explicit `ExactReadout`, and +/// Installed QueryPlans carry an explicit `ExactReadout`, and /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (family, agg_type), - ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - AggregationType::Sum | AggregationType::MultipleSum, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - AggregationType::Increase | AggregationType::MultipleIncrease, + family, + SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count | ExactKind::Increase | ExactKind::Rate, + _ ) - ) + ) && agg_type.planner_exact_family().as_ref() == Some(family) } /// Project a full label-values map down to the requested `by` columns -- @@ -1397,7 +1404,7 @@ fn find_metric(node: &SummaryNode) -> Option { SummaryExpr::KeepPreAsap(qe) => find_metric_in_query_expr(qe), SummaryExpr::SummaryAgg { child, .. } => find_metric(child), SummaryExpr::SummaryEstimate { summary_input, .. } => find_metric(summary_input), - SummaryExpr::SummaryMerge { children } => children.first().and_then(|c| find_metric(c)), + SummaryExpr::SummaryMerge { children, .. } => children.first().and_then(|c| find_metric(c)), _ => None, } } @@ -1447,12 +1454,67 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + #[test] + fn keyed_count_state_follows_planner_family_and_query_readout() { + use crate::precompute_engine::operators::KeyedSumCountAccumulator; + use asap_types::query_plan::ExactReadout; + + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut payload = KeyedSumCountAccumulator::for_family(ExactKind::Count); + payload.update(key.clone(), 10.0); + payload.update(key.clone(), 20.0); + let state = GroupState::ExactAgg { + entries: vec![Rc::new(BTreeMap::from([( + 60_000, + Arc::new(payload) as Arc, + )]))], + agg_type: AggregationType::Count, + }; + assert_eq!( + state.exact_value_for(ExactReadout::Count, &Some(key.clone()), 0, 60_000), + Some(2.0) + ); + assert_eq!( + state.exact_value_for(ExactReadout::Sum, &Some(key), 0, 60_000), + None + ); + } + + #[test] + fn state_merge_rejects_different_planner_families() { + let index = SketchStore::new(); + let context = QueryExecutionContext { + index: &index, + t0_ms: 0, + t1_ms: 60_000, + is_cumulative: true, + allowed_materializations: None, + }; + let states = vec![ + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Rate, + }, + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Increase, + }, + ]; + assert!(matches!( + context.merge_states(states), + Err(SummaryExecutorError::UnsupportedFamily) + )); + } + #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { let binding = asap_types::query_plan::MaterializationBinding { full_window_slide_ms: None, item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + asap_types::PolicyFingerprint(7).into(), + ), output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(7_000), @@ -1888,6 +1950,7 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: Some(20_000), materialization: fp.into(), + stored_output_reference: super::super::test_plan::bound_reference(&index, fp.into()), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 60_000, @@ -1954,6 +2017,7 @@ mod tests { let binding = MaterializationBinding { full_window_slide_ms: None, materialization: fp.into(), + stored_output_reference: super::super::test_plan::bound_reference(&index, fp.into()), output_grouping: PhysicalGrouping::PerEntity, item_labels: vec![], window_ms: 1000, diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 8b9f5b57a..563d85a1b 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -60,6 +60,15 @@ pub(super) fn entry( ) .then_some(config.slide_interval * 1000), materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::summary_catalog::SummaryCatalog::from_materializations( + 1, + 1, + &[config.clone()], + ) + .unwrap() + .output_reference(config.policy_fingerprint().into()) + .unwrap(), output_grouping: grouping, item_labels: config.aggregated_labels.labels.clone(), window_ms: config.stored_window_ms(), @@ -100,7 +109,7 @@ pub(super) fn install( .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); - precompute.summary_catalog = Some(catalog.reference().unwrap()); + precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -128,6 +137,7 @@ pub(super) fn install( plan_id: 1, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: entries .into_iter() .map(|e| (e.canonical_query.clone(), e)) @@ -153,3 +163,31 @@ pub(super) fn engine( .with_sketch_index(index) .with_active_physical_plan(active) } + +/// Low-level operator fixtures install the descriptors of their explicit states. +/// Process tests use compiled publications instead of this fixture adapter. +pub(super) fn bound_reference( + index: &SketchStore, + output: asap_types::sds::StoredOutputId, +) -> asap_types::sds::StoredOutputReference { + if let Some(catalog) = index.summary_catalog_snapshot() { + return catalog.output_reference(output).unwrap(); + } + let metadata = index.snapshot_instances(); + let entries = metadata + .iter() + .filter(|m| !m.policy_fp.is_unset()) + .map(|m| { + let (summary, data) = index.descriptors_for_series_id(m.sid).unwrap(); + (m.policy_fp, (*summary).clone(), (*data).clone()) + }) + .collect::>(); + let catalog = asap_types::summary_catalog::SummaryCatalog::build(1, 1, entries).unwrap(); + index + .install_summary_catalog(Arc::new(catalog.clone())) + .unwrap(); + for metadata in metadata { + index.register((*metadata).clone()); + } + catalog.output_reference(output).unwrap() +} diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index 08eb75e00..0d46b2525 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -102,7 +102,7 @@ impl QueryResult { /// Attach an accuracy envelope. Chainable so engine paths /// can build the bare result first and decorate once the - /// `agg_id → AggregationConfig → AccuracyProfile` lookup + /// `agg_id → PrecomputeMaterialization → AccuracyProfile` lookup /// has resolved. pub fn with_accuracy(mut self, envelope: AccuracyEnvelope) -> Self { match &mut self { diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index f3780b044..084c87cc6 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -1,5 +1,5 @@ //! `AccuracyProfile` — derived error / confidence bound for each -//! `AggregationConfig`. +//! `PrecomputeMaterialization`. //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. @@ -12,7 +12,7 @@ //! //! ## Scope of this module //! -//! Pure derivation: `derive(&AggregationConfig)` +//! Pure derivation: `derive(&PrecomputeMaterialization)` //! looks at `aggregation_type` and the relevant entries in //! `config.parameters` and returns an `AccuracyProfile`. No //! runtime measurement, no sampling — just the textbook bound. @@ -31,14 +31,14 @@ use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::AggregationType; pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; use planner_types::post_asap::SketchParams as PlannerParams; /// Derive an [`AccuracyProfile`] from a pinned -/// [`AggregationConfig`]. Reads `aggregation_type` and any +/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any /// necessary entries in `parameters`; falls back to exact for /// unknown / legacy variants (harmless — the caller just gets /// "0 error" rather than a panic). @@ -52,7 +52,7 @@ use planner_types::post_asap::SketchParams as PlannerParams; /// ε_st`; the random parts compose in quadrature but the staleness part is /// adversarial, so linear addition is the honest envelope). δ is /// unchanged (staleness is not probabilistic). -pub fn derive(config: &AggregationConfig) -> AccuracyProfile { +pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { let mut profile = derive_sketch_only(config); let eps_st = config .parameters @@ -67,20 +67,20 @@ pub fn derive(config: &AggregationConfig) -> AccuracyProfile { /// Source adapter for installed aggregation configs. pub trait BackendAccuracyProfile { - fn derive(config: &AggregationConfig) -> Self; - fn derive_sketch_only(config: &AggregationConfig) -> Self; + fn derive(config: &PrecomputeMaterialization) -> Self; + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; } impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &AggregationConfig) -> Self { + fn derive(config: &PrecomputeMaterialization) -> Self { derive(config) } - fn derive_sketch_only(config: &AggregationConfig) -> Self { + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { derive_sketch_only(config) } } /// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { +fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { match config.aggregation_type { AggregationType::UnivMon => AccuracyProfile { epsilon: f64::MAX, @@ -91,13 +91,11 @@ fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { // `DeltaSetAggregator` exact-set-membership family lived // here too before its retirement.) AggregationType::Sum + | AggregationType::Count | AggregationType::Increase + | AggregationType::Rate | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax => AccuracyProfile::exact(), + | AggregationType::Max => AccuracyProfile::exact(), AggregationType::CountMinSketch => { let (rows, cols) = cms_params(config); @@ -220,7 +218,7 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { // authority on *accuracy*, not on *construction*. /// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &AggregationConfig) -> (u64, u64) { +fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { let rows = config .parameters .get("d") @@ -234,7 +232,7 @@ fn cms_params(config: &AggregationConfig) -> (u64, u64) { (rows, cols) } -fn hll_precision(config: &AggregationConfig) -> u32 { +fn hll_precision(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("precision") @@ -244,7 +242,7 @@ fn hll_precision(config: &AggregationConfig) -> u32 { .unwrap_or(14) } -fn kll_k(config: &AggregationConfig) -> u32 { +fn kll_k(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("K") @@ -254,7 +252,7 @@ fn kll_k(config: &AggregationConfig) -> u32 { .unwrap_or(200) } -fn ddsketch_alpha(config: &AggregationConfig) -> f64 { +fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { config .parameters .get("alpha") @@ -266,7 +264,7 @@ fn ddsketch_alpha(config: &AggregationConfig) -> f64 { /// from `parameters["heap_size"]` with a default of 100 — /// matches the default the control plane's planner uses when the /// caller didn't override. -fn cms_heap_size(config: &AggregationConfig) -> u64 { +fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { config .parameters .get("heap_size") @@ -373,8 +371,11 @@ mod tests { use serde_json::{json, Value}; use std::collections::HashMap; - fn base_config(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( + fn base_config( + agg_type: AggregationType, + params: HashMap, + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index 0e970b821..ce8e476d2 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use std::time::{SystemTime, UNIX_EPOCH}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; @@ -507,12 +507,12 @@ impl BackfillRegistry { /// control-plane-facing HTTP endpoint can return specific 404 / /// 409 / 400 statuses. `CreateError::UnknownAgg` is no longer /// returned from this method — the caller proves the agg - /// exists by holding the `AggregationConfig` — but the variant + /// exists by holding the `PrecomputeMaterialization` — but the variant /// is kept on the enum for HTTP error-mapping compatibility /// (the handler still produces it when its own lookup misses). pub fn create_checked( &self, - config: &AggregationConfig, + config: &PrecomputeMaterialization, created_at_ms: u64, time_range: (u64, u64), source: BackfillSource, @@ -883,7 +883,9 @@ pub use service::{ default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, BackfillServiceHandle, ReaderFactory, }; +#[cfg(test)] pub use window_builder::build_backfilled_accumulator; +pub use window_builder::build_dag_accumulator; pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; #[cfg(test)] diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 430aca740..8bfcf0cf0 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -22,10 +22,11 @@ use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use super::raw_sample_reader::RawSample; +#[cfg(test)] use super::window_builder::build_backfilled_accumulator; use super::worker::WindowProcessor; use super::BackfillRegistry; @@ -39,7 +40,7 @@ use super::BackfillRegistry; /// Kept local to the backfill module (not shared with live) per /// the §5e separation ask; the implementations must stay identical /// by convention. -fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { +fn extract_group_key(series_key: &str, config: &PrecomputeMaterialization) -> String { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); for label_name in &config.grouping_labels.names() { @@ -71,7 +72,7 @@ fn build_group_key_label_values(group_key: &str) -> KeyByLabelValues { /// live windows occupy the same storage row. fn resolve_backfill_bucket_sid( resolver: &SeriesIdResolver, - config: &AggregationConfig, + config: &PrecomputeMaterialization, series_key: &str, store: Option<&crate::storage_engines::sketch_db::index::SketchStore>, captured_generation: Option<&asap_types::sds::CatalogGeneration>, @@ -124,7 +125,7 @@ fn fallback_bucket_id(group_key: &str) -> u64 { pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest /// `StreamingConfig` at each window to find the - /// `AggregationConfig` for `agg_id`. The snapshot is cheap + /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. config: StreamingConfigHandle, /// Destination for rebuilt windows. Tests may omit it to record registry @@ -185,22 +186,6 @@ impl BackfillWindowProcessor { self.series_resolver = Some(series_resolver); self } - - /// Look up the `AggregationConfig` for `agg_id` in the current - /// `StreamingConfig` snapshot. Returns an error string if the - /// agg has been removed from the config since the job was - /// created — rare but worth handling (e.g. operator retired - /// the agg mid-backfill; the `BackfillWorker` will - /// `mark_failed` the job with this message). - fn config_for_agg( - &self, - agg_id: u64, - ) -> Result> { - let snap = self.config.snapshot(); - snap.get_aggregation_config(agg_id).cloned().ok_or_else(|| { - format!("agg_id {agg_id} not in current StreamingConfig — retired mid-backfill?").into() - }) - } } /// One per-sid bucket assembled by [`BackfillWindowProcessor::process_window`]. @@ -219,7 +204,16 @@ impl WindowProcessor for BackfillWindowProcessor { window_range: (u64, u64), samples: Vec, ) -> Result<(), Box> { - let config = self.config_for_agg(agg_id)?; + let snapshot = self.config.snapshot(); + let config = snapshot + .get_aggregation_config(agg_id) + .cloned() + .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + let program = snapshot.raw_programs.get(&agg_id).cloned(); + #[cfg(not(test))] + if program.is_none() { + return Err("backfill requires a post-ASAP DAG installation".into()); + } // B7.7 — sid-keyed bucketing. Per the schema-retirement #5 // step 6 plan, the backfill processor's per-window grouping is @@ -289,7 +283,18 @@ impl WindowProcessor for BackfillWindowProcessor { for (sid, bucket) in by_bucket { let SidBucket { group_key, samples } = bucket; - let accumulator = build_backfilled_accumulator(&config, &samples); + let accumulator = if let Some(program) = &program { + super::window_builder::build_dag_accumulator(program, &samples)? + } else { + #[cfg(test)] + { + build_backfilled_accumulator(&config, &samples) + } + #[cfg(not(test))] + { + return Err("missing backfill DAG producer".into()); + } + }; // Keyed accumulators (MultipleSubpopulation) carry their // subpopulation keys internally; the PrecomputedOutput's // `key` represents the *group* key (grouping_labels @@ -374,7 +379,7 @@ mod tests { use asap_types::KeyByLabelNames; use std::sync::Arc; - fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. let grouping_labels = if grouping.is_empty() { @@ -382,7 +387,7 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -401,7 +406,7 @@ mod tests { ) } - fn streaming_config_with(config: AggregationConfig) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); Arc::new(StreamingConfig::new(map)) @@ -567,7 +572,7 @@ mod tests { /// when given the same ordered samples. #[test] fn backfill_builds_bit_identical_sum_accumulator_to_live() { - use crate::precompute_engine::accumulator_factory::create_accumulator_updater; + use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; let cfg = sum_config(1, "m", vec![]); @@ -592,7 +597,7 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_accumulator_updater(&cfg); + let mut updater = create_fixture_accumulator(&cfg); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -612,7 +617,7 @@ mod tests { serialisations for SumAccumulator. \ If this test fails, something diverged — check:\n\ (1) Is `build_backfilled_accumulator` still calling \ - `create_accumulator_updater`?\n\ + `create_fixture_accumulator`?\n\ (2) Did a recent change to `SumAccumulator` introduce \ non-deterministic state (e.g. a seed)?\n\ (3) Does `serialize_to_bytes` include any timestamp \ diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index 7f9208a1e..79d9ccdb0 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -43,7 +43,7 @@ pub struct RawSample { /// honour. Exactly one metric name plus zero or more equality /// matchers on grouping labels — no regex, no negation, no /// lexicographic ranges. The control plane picks the subset of -/// `AggregationConfig.grouping_labels` that should gate the read. +/// `PrecomputeMaterialization.grouping_labels` that should gate the read. /// /// Rationale: every supported exact-DB backend (Prometheus, /// ClickHouse, S3+Gorilla) can evaluate this filter efficiently, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index c0b35971a..86d64fac3 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -342,15 +342,15 @@ mod tests { MockRawSampleReader, RawSample, }; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Mutex; - fn sum_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -369,7 +369,7 @@ mod tests { ) } - fn streaming_with(cfg: AggregationConfig) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); Arc::new(StreamingConfig::new(m)) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index 04a4eed39..6bf028c9c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -4,13 +4,16 @@ //! It shares the pure accumulator factory and update primitives with live ingest //! so both paths use the same sketch semantics. +#[cfg(test)] use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, + create_fixture_accumulator, AccumulatorUpdater, }; +#[cfg(test)] use crate::precompute_engine::worker::apply_sample; use crate::storage_engines::sketch_db::backfill::raw_sample_reader::RawSample; use crate::storage_engines::types::AggregateCore; -use asap_types::aggregation_config::AggregationConfig; +#[cfg(test)] +use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -25,11 +28,12 @@ use asap_types::aggregation_config::AggregationConfig; /// The function is synchronous + pure (no I/O, no async, no global /// state). Suitable to call from inside a `WindowProcessor` /// implementation without worrying about the async runtime. +#[cfg(test)] pub fn build_backfilled_accumulator( - config: &AggregationConfig, + config: &PrecomputeMaterialization, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_accumulator_updater(config); + let mut updater: Box = create_fixture_accumulator(config); for sample in samples { apply_sample( &mut *updater, @@ -45,14 +49,14 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_config() -> AggregationConfig { - AggregationConfig::new( + fn sum_config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), @@ -156,3 +160,28 @@ mod tests { assert!(aux.sum == Some(0.0) || aux.sum.is_none()); } } + +/// Backfill uses the same selected DAG producer and update expressions as live input. +pub fn build_dag_accumulator( + program: &crate::precompute_engine::raw_dag::RawDagProgram, + samples: &[RawSample], +) -> Result, String> { + let mut updater = program.updater()?; + let mut previous = std::collections::HashMap::new(); + for sample in samples { + let value = if program.uses_counter_delta() { + crate::precompute_engine::worker::reset_aware_counter_delta( + &mut previous, + &sample.labels, + sample.value, + sample.timestamp_ms, + ) + } else { + Some(sample.value) + }; + if let Some(value) = value { + program.apply(&mut *updater, &sample.labels, value, sample.timestamp_ms)?; + } + } + Ok(updater.take_accumulator()) +} diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index e8f3d873f..e696fe525 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -137,7 +137,7 @@ pub enum AggKind { /// Complete resolver identity for a configured materialization. All live and /// replay paths must include policy semantics, not just the sketch family. pub(crate) fn materialization_kind_for_config( - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, ) -> String { format!( "{}|{}", @@ -149,7 +149,9 @@ pub(crate) fn materialization_kind_for_config( /// Resolve the physical state family produced by a precompute policy. This is /// shared by SID minting and store registration so a sketch policy can never /// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config(config: &asap_types::aggregation_config::AggregationConfig) -> AggKind { +pub fn agg_kind_for_config( + config: &asap_types::aggregation_config::PrecomputeMaterialization, +) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; // HLL is intentionally absent from raw-value accumulator dispatch because @@ -592,7 +594,7 @@ mod tests { #[test] fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::AggregationConfig::new( + let config = asap_types::aggregation_config::PrecomputeMaterialization::new( AggregationType::HLL, String::new(), HashMap::from([("precision".to_string(), serde_json::json!(12))]), diff --git a/data_plane/src/storage_engines/sketch_db/index/admission.rs b/data_plane/src/storage_engines/sketch_db/index/admission.rs index dc30248ee..4ad4a5452 100644 --- a/data_plane/src/storage_engines/sketch_db/index/admission.rs +++ b/data_plane/src/storage_engines/sketch_db/index/admission.rs @@ -1,7 +1,7 @@ //! Store-owned tracking of accepted, not necessarily published summary updates. //! This records known work; it does not infer an event-time watermark. -use asap_types::sds::SummaryDefinitionId; +use asap_types::sds::StoredOutputId; use asap_types::sds::{CatalogGeneration, HalfOpenTimeRange, SummaryInstanceCoordinates}; use std::collections::{BTreeMap, BTreeSet}; @@ -27,7 +27,7 @@ pub(super) struct AdmissionInventory { revision: u64, windows: BTreeMap, metadata_bytes: usize, - replay_floors: BTreeMap, + replay_floors: BTreeMap, observed_extent: Option, finite_input: FiniteInputState, published_series: BTreeMap, @@ -70,7 +70,7 @@ impl AdmissionInventory { for coordinate in &coordinates { if self .replay_floors - .get(&coordinate.summary_definition_id) + .get(&coordinate.stored_output_id) .is_some_and(|floor| coordinate.time_range.end_ms <= *floor) { return Err("summary input precedes retained replay horizon".into()); @@ -252,7 +252,7 @@ impl AdmissionInventory { pub(super) fn known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -261,7 +261,7 @@ impl AdmissionInventory { pub(super) fn known_empty_with_layout( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, full_window: bool, @@ -276,7 +276,7 @@ impl AdmissionInventory { .get(&definition) .is_none_or(|floor| range.start_ms >= *floor) && !self.windows.iter().any(|(coordinate, state)| { - coordinate.summary_definition_id == definition + coordinate.stored_output_id == definition && (if full_window { coordinate.time_range == range } else { @@ -297,12 +297,12 @@ impl AdmissionInventory { pub(super) fn has_pending( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, range: HalfOpenTimeRange, full_window: bool, ) -> bool { self.windows.iter().any(|(coordinate, state)| { - coordinate.summary_definition_id == definition + coordinate.stored_output_id == definition && (if full_window { coordinate.time_range == range } else { @@ -315,11 +315,7 @@ impl AdmissionInventory { /// Advancing this configured replay floor also rejects future old admission. /// Pending work is never forgotten because a different series ran ahead. - pub(super) fn retire_completed_before( - &mut self, - definition: SummaryDefinitionId, - frontier_ms: i64, - ) { + pub(super) fn retire_completed_before(&mut self, definition: StoredOutputId, frontier_ms: i64) { let floor = self.replay_floors.entry(definition).or_insert(i64::MIN); *floor = (*floor).max(frontier_ms); let frontier_ms = *floor; @@ -329,7 +325,7 @@ impl AdmissionInventory { .flat_map(|state| state.pending.iter().copied()) .collect(); self.windows.retain(|coordinate, state| { - let remove = coordinate.summary_definition_id == definition + let remove = coordinate.stored_output_id == definition && coordinate.time_range.end_ms <= frontier_ms && state.published >= state.admitted && !pending.contains(&state.admitted); @@ -365,7 +361,7 @@ mod tests { } fn window(series: &str) -> SummaryInstanceCoordinates { SummaryInstanceCoordinates { - summary_definition_id: SummaryDefinitionId(asap_types::PolicyFingerprint(7)), + stored_output_id: StoredOutputId::from(asap_types::PolicyFingerprint(7)), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -385,9 +381,9 @@ mod tests { .admit(&generation, BTreeSet::from([a.clone(), b.clone()])) .unwrap(); inventory.acknowledge(&generation, &a, revision).unwrap(); - assert!(inventory.has_pending(a.summary_definition_id, a.time_range, false)); + assert!(inventory.has_pending(a.stored_output_id, a.time_range, false)); inventory.acknowledge(&generation, &b, revision).unwrap(); - assert!(!inventory.has_pending(a.summary_definition_id, a.time_range, false)); + assert!(!inventory.has_pending(a.stored_output_id, a.time_range, false)); } #[test] @@ -407,12 +403,8 @@ mod tests { .acknowledge(&generation, &coordinate, first) .unwrap(); assert_ne!(before, inventory.revision()); - assert!(inventory.has_pending( - coordinate.summary_definition_id, - coordinate.time_range, - false - )); - inventory.retire_completed_before(coordinate.summary_definition_id, 1000); + assert!(inventory.has_pending(coordinate.stored_output_id, coordinate.time_range, false)); + inventory.retire_completed_before(coordinate.stored_output_id, 1000); assert_eq!(inventory.windows.len(), 1); inventory .record_series(&generation, &coordinate, 42) @@ -420,7 +412,7 @@ mod tests { inventory .acknowledge(&generation, &coordinate, second) .unwrap(); - inventory.retire_completed_before(coordinate.summary_definition_id, 1000); + inventory.retire_completed_before(coordinate.stored_output_id, 1000); assert!(inventory.windows.is_empty()); assert_eq!( inventory.published_frontiers().get(&42), @@ -485,12 +477,12 @@ mod tests { let revision = inventory .admit(&generation, BTreeSet::from([current.clone(), future])) .unwrap(); - assert!(inventory.has_pending(current.summary_definition_id, current.time_range, true)); + assert!(inventory.has_pending(current.stored_output_id, current.time_range, true)); inventory .acknowledge(&generation, ¤t, revision) .unwrap(); - assert!(inventory.has_pending(current.summary_definition_id, current.time_range, false)); - assert!(!inventory.has_pending(current.summary_definition_id, current.time_range, true)); + assert!(inventory.has_pending(current.stored_output_id, current.time_range, false)); + assert!(!inventory.has_pending(current.stored_output_id, current.time_range, true)); } // A neighboring full snapshot may overlap an empty query population. @@ -525,28 +517,28 @@ mod tests { .unwrap(); } assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true )); inventory.seal_finite(&generation).unwrap(); - assert!(!inventory.known_empty(first.summary_definition_id, 1, other.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 1, other.time_range)); assert!(inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true )); assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 2, other.time_range, true )); - inventory.retire_completed_before(first.summary_definition_id, 2000); + inventory.retire_completed_before(first.stored_output_id, 2000); assert!(!inventory.known_empty_with_layout( - first.summary_definition_id, + first.stored_output_id, 1, other.time_range, true @@ -576,13 +568,13 @@ mod tests { inventory .acknowledge(&generation, &second, revision) .unwrap(); - assert!(!inventory.known_empty(first.summary_definition_id, 1, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 1, second.time_range)); inventory.seal_finite(&generation).unwrap(); - assert!(inventory.known_empty(first.summary_definition_id, 1, second.time_range)); - assert!(!inventory.known_empty(first.summary_definition_id, 2, second.time_range)); - assert!(!inventory.known_empty(first.summary_definition_id, 999, second.time_range)); + assert!(inventory.known_empty(first.stored_output_id, 1, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 2, second.time_range)); + assert!(!inventory.known_empty(first.stored_output_id, 999, second.time_range)); assert!(!inventory.known_empty( - first.summary_definition_id, + first.stored_output_id, 1, HalfOpenTimeRange { start_ms: 2000, diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index c1bc5f220..199b643d5 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -8,7 +8,7 @@ use crate::storage_engines::types::AggregateCore; pub(crate) struct FrozenExactWindows { pub(crate) sid: u64, - pub(crate) definition: SummaryDefinitionId, + pub(crate) definition: StoredOutputId, pub(crate) generation: Arc, pub(crate) group: BTreeMap, pub(crate) windows: BTreeMap<(u64, u64), Arc>, @@ -49,10 +49,10 @@ impl SketchStore { pub(crate) fn read_frozen_exact_cohort( &self, generation: &Arc, - expected_definitions: &BTreeSet, + expected_definitions: &BTreeSet, requests: &[( u64, - SummaryDefinitionId, + StoredOutputId, BTreeSet<(u64, u64)>, BTreeMap, )], @@ -89,7 +89,7 @@ impl SketchStore { pub(crate) fn read_complete_raw_maintenance_cohort( &self, generation: &Arc, - definitions: &BTreeSet, + definitions: &BTreeSet, window: (u64, u64), ) -> Result { if definitions.is_empty() || window.0 >= window.1 { @@ -113,7 +113,7 @@ impl SketchStore { fn durable_maintenance_population_ids( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, ) -> Result, String> { let metadata = self .persistence_metadata @@ -126,7 +126,7 @@ impl SketchStore { .load_strict() .map_err(|error| error.to_string())? .into_iter() - .filter(|record| !record.removed && record.summary_definition_id == Some(definition)) + .filter(|record| !record.removed && record.stored_output_id == Some(definition)) .map(|record| record.sid) .collect()) } @@ -135,7 +135,7 @@ impl SketchStore { /// coverage and incarnation before reading or publishing any state. pub(crate) fn completed_maintenance_coordinates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, ) -> Result, BTreeSet<(u64, u64)>>>, String> { @@ -147,7 +147,7 @@ impl SketchStore { /// bind them to the current catalog. pub(crate) fn complete_raw_maintenance_population( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, ) -> Result, BTreeSet<(u64, u64)>>>, String> { @@ -156,7 +156,7 @@ impl SketchStore { fn maintenance_coordinates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &CatalogGeneration, require_complete_population: bool, ) -> Result, BTreeSet<(u64, u64)>>>, String> @@ -260,7 +260,7 @@ impl SketchStore { pub(crate) fn read_frozen_exact_windows( &self, sid: u64, - definition: SummaryDefinitionId, + definition: StoredOutputId, generation: &Arc, expected_windows: &BTreeSet<(u64, u64)>, group: &BTreeMap, @@ -435,7 +435,7 @@ impl SketchStore { if !finite_complete { return Err("complete raw publication requires the original finite closure".into()); } - let mut supplied = BTreeMap::>::new(); + let mut supplied = BTreeMap::>::new(); for input in cohort.inputs() { supplied .entry(input.definition) @@ -775,7 +775,7 @@ mod tests { let definition = config.policy_fingerprint().into(); let population = BTreeMap::from([("instance".to_string(), index.to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: definition, + stored_output_id: definition, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -891,7 +891,7 @@ mod tests { for (index, config) in configs.iter().take(2).enumerate() { let definition = config.policy_fingerprint().into(); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: definition, + stored_output_id: definition, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 1000, @@ -926,7 +926,7 @@ mod tests { // Only the first population has the next window: completeness must // reject the partial cohort even after all writes are durably sealed. let extra_coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: configs[0].policy_fingerprint().into(), + stored_output_id: configs[0].policy_fingerprint().into(), time_range: HalfOpenTimeRange { start_ms: 1000, end_ms: 2000, @@ -1116,7 +1116,7 @@ mod tests { for (instance, value) in [("a", 5.0), ("b", 15.0)] { let population = BTreeMap::from([("instance".to_string(), instance.to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source.policy_fingerprint().into(), + stored_output_id: source.policy_fingerprint().into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, @@ -1315,7 +1315,7 @@ mod tests { let mut restored = restarted.start_persistence(restart_config).unwrap(); let population = BTreeMap::from([("instance".to_string(), "new".to_string())]); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: source_id, + stored_output_id: source_id, time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 60_000, diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e0c9ba55c..57fc7c953 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -25,7 +25,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use asap_types::sds::{ CatalogGeneration, HalfOpenTimeRange, InstanceCompleteness, InstanceLifecycle, - ObservedSummaryInventory, SummaryDefinitionId, SummaryInstance, SummaryInstanceStatus, + ObservedSummaryInventory, StoredOutputId, SummaryInstance, SummaryInstanceStatus, SummaryPlacement, SummaryStateReference, }; use asap_types::PolicyFingerprint; @@ -92,11 +92,12 @@ fn reconstruct_exact_agg( bytes: &[u8], ) -> Option> { use crate::precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleIncreaseAccumulator, - MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedCounterState, KeyedSumCountAccumulator, MaxAccumulator, + MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::AggregateCore; match type_name { + "PlannerExactAccumulatorV1" => crate::precompute_engine::operators::exact_accumulator::ExactAccumulator::deserialize_from_bytes(bytes).ok().map(|a|Box::new(a) as Box), "SumAccumulator" => SumAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), @@ -109,10 +110,12 @@ fn reconstruct_exact_agg( "MaxAccumulator" => MaxAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), - "MultipleSumAccumulator" => MultipleSumAccumulator::deserialize_from_bytes(bytes) - .ok() - .map(|a| Box::new(a) as Box), - "MultipleIncreaseAccumulator" => MultipleIncreaseAccumulator::deserialize_from_bytes(bytes) + "KeyedSumCountAccumulator" => { + KeyedSumCountAccumulator::deserialize_from_bytes(bytes) + .ok() + .map(|a| Box::new(a) as Box) + } + "KeyedCounterState" => KeyedCounterState::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), // The keyed `MultipleMin`/`MultipleMax` forms and the @@ -138,7 +141,7 @@ fn reconstruct_exact_agg( /// so the mint-driven path (B7.6) and the sid-direct path (B7.7) stay /// byte-identical on the values they hand to the index. fn build_attrs_fp_and_label_map( - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Result<(String, BTreeMap), String> { if let Some(labels) = &output.population_labels { @@ -232,7 +235,7 @@ pub struct SummarySeriesMetadata { /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source - /// `AggregationConfig` (test fixtures + the early-Phase-5 sketch + /// `PrecomputeMaterialization` (test fixtures + the early-Phase-5 sketch /// ingest path that didn't thread the config through); the index /// skips those entries — they're reachable through the legacy /// `instances_matching(metric, gbk)` walk if a query needs them. @@ -698,7 +701,7 @@ pub struct SketchStore { u64, ( Option>, - Option, + Option, ), >, >, @@ -866,6 +869,11 @@ impl SketchStore { catalog: Arc, ) -> Result<(), String> { let reference = catalog.reference().map_err(|error| error.to_string())?; + if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { + writer + .persist_catalog(&catalog) + .map_err(|e| e.to_string())?; + } let mut inventory = self.admission.write().unwrap(); let generation = CatalogGeneration { schema_version: reference.schema_version, @@ -901,11 +909,10 @@ impl SketchStore { .descriptors .authoritative_catalog() .ok_or("summary admission requires an installed catalog")?; - if coordinates.iter().any(|coordinate| { - !catalog - .materializations - .contains_key(&coordinate.summary_definition_id) - }) { + if coordinates + .iter() + .any(|coordinate| !catalog.outputs.contains_key(&coordinate.stored_output_id)) + { return Err("summary admission references an uninstalled definition".into()); } self.admission @@ -950,7 +957,7 @@ impl SketchStore { .time_range .end_ms .saturating_sub(i64::try_from(replay_horizon_ms).unwrap_or(i64::MAX)); - inventory.retire_completed_before(coordinate.summary_definition_id, floor); + inventory.retire_completed_before(coordinate.stored_output_id, floor); Ok(()) } @@ -1030,7 +1037,7 @@ impl SketchStore { pub(crate) fn summary_window_known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -1048,7 +1055,7 @@ impl SketchStore { /// Overlapping neighboring snapshots do not establish population in this one. pub(crate) fn full_summary_window_known_empty( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, series_id: u64, range: HalfOpenTimeRange, ) -> bool { @@ -1079,7 +1086,7 @@ impl SketchStore { pub(crate) fn has_pending_summary_updates( &self, - definition: SummaryDefinitionId, + definition: StoredOutputId, range: HalfOpenTimeRange, full_window: bool, ) -> bool { @@ -1143,11 +1150,13 @@ impl SketchStore { binding: &SdsBinding, generation: Option<&CatalogGeneration>, ) -> bool { - !matches!( - binding.data_descriptor.source, - asap_types::sds::DataSourceIdentity::Derived { .. } - ) || generation - .is_some_and(|generation| binding.catalog_generation.as_deref() == Some(generation)) + match generation { + Some(generation) => binding.catalog_generation.as_deref() == Some(generation), + None => !matches!( + binding.data_descriptor.source, + asap_types::sds::DataSourceIdentity::Derived { .. } + ), + } } #[cfg(test)] @@ -1196,7 +1205,7 @@ impl SketchStore { &self, reporter_id: &str, storage_node_id: &str, - producers: &BTreeMap, + producers: &BTreeMap, inventory_version: u64, observed_at_ms: i64, ) -> Result { @@ -1218,21 +1227,17 @@ impl SketchStore { if !Self::instance_visible_in_generation(binding, Some(&generation)) { continue; } - let summary_definition_id = SummaryDefinitionId::from(binding.metadata.policy_fp); + let stored_output_id = StoredOutputId::from(binding.metadata.policy_fp); if binding.metadata.policy_fp.is_unset() - || !catalog - .materializations - .contains_key(&summary_definition_id) + || !catalog.outputs.contains_key(&stored_output_id) { continue; } - let producer_id = producers - .get(&summary_definition_id) - .map(String::as_str) - .ok_or_else(|| { + let (stored_output_id, producer_id) = + producers.get(&stored_output_id).ok_or_else(|| { format!( "materialization {} has no producer in the active PrecomputePlan", - summary_definition_id.as_u64() + stored_output_id.as_u64() ) })?; let store = self @@ -1261,7 +1266,7 @@ impl SketchStore { 0, ); let instance_id = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id, + stored_output_id: *stored_output_id, time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values: group_values.clone(), } @@ -1269,14 +1274,20 @@ impl SketchStore { .map_err(|error| error.to_string())?; let instance = SummaryInstance { instance_id: instance_id.clone(), - summary_definition_id, + stored_output_id: *stored_output_id, + summary_definition_id: binding + .stored_output_reference + .as_ref() + .ok_or("missing semantic binding")? + .definition_id + .clone(), summary_descriptor_id: binding.summary_descriptor.id().clone(), data_descriptor_id: binding.data_descriptor.id().clone(), time_range: HalfOpenTimeRange { start_ms, end_ms }, group_values, catalog_generation: generation.clone(), placement: SummaryPlacement { - producer_id: producer_id.into(), + producer_id: producer_id.clone(), storage_node_id: storage_node_id.into(), }, state_reference: SummaryStateReference { @@ -1356,7 +1367,9 @@ impl SketchStore { observed_at_ms, instances: reported, }; - inventory.validate().map_err(|error| error.to_string())?; + inventory + .validate_against_catalog(&catalog) + .map_err(|error| error.to_string())?; Ok(inventory) } @@ -2242,7 +2255,7 @@ impl SketchStore { } /// Phase 5 M2.3.5 — query the precompute payloads across every sid - /// belonging to one `AggregationConfig` (identified by `metric` + + /// belonging to one `PrecomputeMaterialization` (identified by `metric` + /// `agg_cfg.aggregation_type`), shaped as the legacy `Store` /// trait's `TimestampedBucketsMap`. Lets the query engine swap /// `Store::query_precomputed_output` for `SketchStore` without @@ -2690,7 +2703,9 @@ impl SketchStore { m.first_seen_unix_ms, ); if !m.policy_fp.is_unset() { - record.summary_definition_id = Some(SummaryDefinitionId::from(m.policy_fp)); + record.stored_output_id = Some(StoredOutputId::from(m.policy_fp)); + record.summary_definition_id = + Some(m.stored_output_reference.as_ref()?.definition_id.clone()); record.catalog_generation = Some(Arc::clone(m.catalog_generation.as_ref()?)); } record.retired_at_ms = m.retired_at_ms; @@ -2793,7 +2808,7 @@ impl SketchStore { pub(crate) fn authorize_series_reactivation( &self, sid: u64, - definition: SummaryDefinitionId, + definition: StoredOutputId, ) -> Result>, String> { if let Some(binding) = self .instances @@ -2804,15 +2819,17 @@ impl SketchStore { if matches!( binding.data_descriptor.source, asap_types::sds::DataSourceIdentity::Derived { .. } - ) { + ) || binding.catalog_generation.as_deref() + != self.active_catalog_generation().as_deref() + { let (catalog, generation) = self .descriptors .authoritative_snapshot() - .ok_or("derived reactivation requires an authoritative catalog")?; + .ok_or("series reactivation requires an authoritative catalog")?; if binding.metadata.policy_fp != definition.fingerprint() - || !catalog.materializations.contains_key(&definition) + || !catalog.outputs.contains_key(&definition) { - return Err("derived reactivation differs from its installed definition".into()); + return Err("series reactivation differs from its installed definition".into()); } if binding.catalog_generation.as_deref() != Some(generation.as_ref()) { return Ok(Some(generation)); @@ -2830,9 +2847,7 @@ impl SketchStore { .descriptors .authoritative_snapshot() .ok_or("series reactivation requires an authoritative catalog")?; - if *old_definition != Some(definition) - || !catalog.materializations.contains_key(&definition) - { + if *old_definition != Some(definition) || !catalog.outputs.contains_key(&definition) { return Err("series reactivation does not match the installed materialization".into()); } let old_generation = old_generation @@ -2873,7 +2888,7 @@ impl SketchStore { record .as_ref() .and_then(|value| value.catalog_generation.clone()), - record.and_then(|value| value.summary_definition_id), + record.and_then(|value| value.stored_output_id), ), ); } @@ -2943,7 +2958,7 @@ impl SketchStore { } /// Phase 5 M2.3.6e — write-side helper. Given an - /// `AggregationConfig` and one `(PrecomputedOutput, AggregateCore)` + /// `PrecomputeMaterialization` and one `(PrecomputedOutput, AggregateCore)` /// pair (the shape both the live worker AND the backfill processor /// emit), compute the precompute sid, register a metadata entry on /// first sight, and append the payload window. Used by @@ -2963,7 +2978,7 @@ impl SketchStore { pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3095,7 +3110,7 @@ impl SketchStore { pub fn ingest_precompute_with_series_id( &self, sid: u64, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3113,6 +3128,18 @@ impl SketchStore { output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + let expected = agg_cfg.accumulator_spec().ok()?.family; + if matches!( + expected, + planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) + ) && accumulator + .get_accumulator_type() + .planner_exact_family() + .as_ref() + != Some(&expected) + { + return None; + } let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; // Keep the physical lifetime alive through publication. Removal takes @@ -3230,6 +3257,9 @@ impl SketchStore { // concurrent lifecycle operation cannot succeed without persistence. let metadata_writer = Arc::new(persistence::metadata::SidMetadataStore::new(&cfg.disk_path)); + if let Some(catalog) = self.descriptors.authoritative_catalog() { + metadata_writer.persist_catalog(&catalog)?; + } // Restore the generation-wide raw admission barrier before exposing // recovered state or accepting another producer after restart. if let Some(closed) = metadata_writer.load_finite_closure()? { @@ -3253,7 +3283,7 @@ impl SketchStore { .map(|record| { ( record.sid, - (record.catalog_generation, record.summary_definition_id), + (record.catalog_generation, record.stored_output_id), ) }), ); @@ -3275,7 +3305,7 @@ impl SketchStore { // early-returns on the missing `sid_group_by_keys` → "No result" // cluster-wide even though the data is durable on disk. Idempotent: // sids already registered (e.g. by an in-flight DataPoint) are kept. - let recovered = self.register_recovered_disk_series(&cfg.disk_path); + let recovered = self.register_recovered_disk_series(&cfg.disk_path)?; if recovered > 0 { tracing::info!( recovered_sids = recovered, @@ -3324,22 +3354,14 @@ impl SketchStore { /// metadata is authoritative, so we don't clobber it). Returns the /// number of sids freshly registered from disk. /// - /// `capability` / `accuracy` are re-derived from the persisted - /// `agg_kind` exactly as the ingest path derives them. The sidecar is - /// missing only for parts written before this feature landed (or a - /// fresh dir) — those sids stay invisible until a live DataPoint - /// re-registers them, the same as pre-fix behavior. - pub fn register_recovered_disk_series(&self, disk_path: &std::path::Path) -> usize { + /// Persisted metadata errors propagate to startup. Only an absent file is + /// treated as a fresh store. + pub fn register_recovered_disk_series( + &self, + disk_path: &std::path::Path, + ) -> persistence::PersistResult { use crate::storage_engines::sketch_db::index::persistence::metadata::SidMetadataStore; - - let store = SidMetadataStore::new(disk_path); - let records = match store.load() { - Ok(r) => r, - Err(e) => { - tracing::warn!(error = %e, "failed to load sid metadata sidecar on recovery"); - return 0; - } - }; + let records = SidMetadataStore::new(disk_path).load()?; let mut registered = 0usize; for rec in records { @@ -3359,15 +3381,28 @@ impl SketchStore { continue; } let catalog = self.descriptors.authoritative_snapshot(); - let policy_fp = match ( - &rec.summary_definition_id, - &rec.catalog_generation, - &catalog, - ) { + let policy_fp = match (&rec.stored_output_id, &rec.catalog_generation, &catalog) { (Some(definition), Some(generation), Some((catalog, installed_generation))) => { + let persisted = persistence::metadata::SidMetadataStore::new(disk_path) + .load_catalog(generation); if generation != installed_generation - || !catalog.materializations.contains_key(definition) + || persisted.as_ref().ok() != Some(catalog.as_ref()) + || !catalog.outputs.get(definition).is_some_and(|output| { + Some(&output.definition_id) == rec.summary_definition_id.as_ref() + }) { + // A new version must allocate a fresh physical series. Otherwise + // the persisted resolver can route fresh input back to this + // excluded series and its already-completed windows. + if generation.plan_id == installed_generation.plan_id + && generation.plan_version < installed_generation.plan_version + && catalog.outputs.contains_key(definition) + { + self.removed_sids + .write() + .unwrap() + .insert(rec.sid, (Some(Arc::clone(generation)), Some(*definition))); + } tracing::warn!( sid = rec.sid, "persisted summary catalog provenance differs; leaving state unbound" @@ -3409,7 +3444,7 @@ impl SketchStore { registered += 1; } } - registered + Ok(registered) } /// Switch the store into durable-tier mode: install the read handle @@ -3668,6 +3703,15 @@ mod tests { meta_with_policy(sid, asap_types::PolicyFingerprint::UNSET) } + fn meta_for_config( + sid: u64, + config: &asap_types::PrecomputeMaterialization, + ) -> SummarySeriesMetadata { + let mut metadata = meta_with_policy(sid, config.policy_fingerprint()); + metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(config); + metadata + } + fn meta_with_policy( sid: u64, policy_fp: asap_types::PolicyFingerprint, @@ -3725,6 +3769,26 @@ mod tests { assert!(!before.matches(store.summary_update_revision())); } + #[test] + fn recovery_rejects_unsupported_or_corrupt_metadata() { + let directory = tempfile::tempdir().unwrap(); + let store = SketchStore::new(); + let sidecar = persistence::metadata::SidMetadataStore::new(directory.path()); + assert_eq!( + store + .register_recovered_disk_series(directory.path()) + .unwrap(), + 0 + ); + for bytes in [b"{not json".as_slice(), b"{\"schema_version\":1}", b"{}"] { + std::fs::write(sidecar.path(), bytes).unwrap(); + assert!(store + .register_recovered_disk_series(directory.path()) + .is_err()); + assert!(store.snapshot_instances().is_empty()); + } + } + #[test] fn store_lookups_and_equivalent_sids_share_sds_allocations() { let store = SketchStore::new(); @@ -3752,12 +3816,24 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(41, fingerprint)); + store.register(meta_for_config(41, &state_config)); store.append_sample( 41, BTreeMap::from([("job".to_string(), "api".to_string())]), @@ -3772,8 +3848,12 @@ mod tests { ); let producers = BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), - "producer-a".to_string(), + StoredOutputId::from(fingerprint), + ( + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) + .stored_output_id, + "producer-a".to_string(), + ), )]); let inventory = store .observed_summary_inventory("backend-a", "store-a", &producers, 1, 100) @@ -3781,7 +3861,11 @@ mod tests { inventory.validate().unwrap(); assert_eq!(inventory.instances.len(), 2); let instance = inventory.instances.values().next().unwrap(); - assert_eq!(instance.summary_definition_id.fingerprint(), fingerprint); + assert_eq!(instance.stored_output_id.fingerprint(), fingerprint); + assert_eq!( + instance.stored_output_id, + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()).stored_output_id + ); assert_eq!(instance.status, SummaryInstanceStatus::Ready); assert_eq!(instance.completeness, InstanceCompleteness::Unknown); assert!(!instance.group_values.is_empty()); @@ -3805,8 +3889,7 @@ mod tests { inventory.instances.keys().collect::>(), next_inventory.instances.keys().collect::>() ); - let catalog_identity = - &plan.summary_catalog.materializations[&instance.summary_definition_id]; + let catalog_identity = &plan.summary_catalog.outputs[&instance.stored_output_id]; assert_eq!( instance.summary_descriptor_id, catalog_identity.summary_descriptor_id @@ -3827,15 +3910,31 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - store.register(meta_with_policy(42, fingerprint)); + store.register(meta_for_config(42, &state_config)); let producers = BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), - "producer-a".to_string(), + StoredOutputId::from(fingerprint), + ( + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) + .stored_output_id, + "producer-a".to_string(), + ), )]); let inventory = store .observed_summary_inventory("backend-a", "store-a", &producers, 1, 100) @@ -4987,8 +5086,20 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); - let metadata = meta_with_policy(507, fingerprint); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); + let metadata = meta_for_config(507, &state_config); let record = SidMetaRecord::new( metadata.sid, metadata.metric_name.clone(), @@ -5003,10 +5114,10 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - assert_eq!(store.register_recovered_disk_series(tmp.path()), 0); + assert_eq!(store.register_recovered_disk_series(tmp.path()).unwrap(), 0); assert!(store.instance(507).is_none()); let mut foreign = record; - foreign.summary_definition_id = Some(fingerprint.into()); + foreign.stored_output_id = Some(fingerprint.into()); let reference = plan.summary_catalog.reference().unwrap(); foreign.catalog_generation = Some(Arc::new(CatalogGeneration { schema_version: reference.schema_version, @@ -5015,7 +5126,80 @@ mod tests { snapshot_sha256: reference.snapshot_sha256, })); sidecar.upsert_all(&[foreign]).unwrap(); - assert_eq!(store.register_recovered_disk_series(tmp.path()), 0); + assert_eq!(store.register_recovered_disk_series(tmp.path()).unwrap(), 0); + assert!(store.series_ids_for_policy(fingerprint).is_empty()); + } + + #[test] + fn new_generation_requires_fresh_state_for_unchanged_definition() { + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); + let store = SketchStore::new(); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_for_config(509, &state_config)); + store.append_sample(509, BTreeMap::new(), (0, 10_000), sample(1)); + let mut next = plan.summary_catalog; + next.plan_version += 1; + store.install_summary_catalog(Arc::new(next)).unwrap(); + assert!( + store.series_ids_for_policy(fingerprint).is_empty(), + "new version must start cold" + ); + assert!( + store + .authorize_series_reactivation(509, fingerprint.into()) + .unwrap() + .is_some(), + "live version change must allocate a fresh physical series" + ); + let output = + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()).stored_output_id; + let inventory = store + .observed_summary_inventory( + "backend-a", + "store-a", + &BTreeMap::from([( + StoredOutputId::from(fingerprint), + (output, "producer-a".into()), + )]), + 1, + 100, + ) + .unwrap(); + assert!(inventory.instances.is_empty()); + store.register(meta_with_policy(510, fingerprint)); + store.append_sample(510, BTreeMap::new(), (0, 10_000), sample(2)); + assert_eq!(store.series_ids_for_policy(fingerprint), vec![510]); + let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( + plan.precompute_plan.envelope.plan_id, + plan.precompute_plan.envelope.plan_version + 2, + &[], + ) + .unwrap(); + store + .install_summary_catalog(Arc::new(incompatible)) + .unwrap(); assert!(store.series_ids_for_policy(fingerprint).is_empty()); } @@ -5030,7 +5214,19 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let definition = fingerprint.into(); let mut next_catalog = plan.summary_catalog.clone(); next_catalog.plan_version += 1; @@ -5047,7 +5243,7 @@ mod tests { .unwrap(); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); old_sid = resolver.resolve("metric", "group", "family"); - store.register(meta_with_policy(old_sid, fingerprint)); + store.register(meta_for_config(old_sid, &state_config)); for pane in 0..4 { store.append_sample( old_sid, @@ -5060,7 +5256,6 @@ mod tests { || !persistence.manifest.live_parts().is_empty(), Duration::from_secs(5) )); - let old_parts = persistence.manifest.live_parts().len(); store.remove_instance(old_sid).unwrap(); assert!(resolver .resolve_with_reactivation("metric", "group", "family", |sid| store @@ -5075,7 +5270,7 @@ mod tests { }) .unwrap(); assert_ne!(new_sid, old_sid); - store.register(meta_with_policy(new_sid, fingerprint)); + store.register(meta_for_config(new_sid, &state_config)); for pane in 0..4 { store.append_sample( new_sid, @@ -5085,7 +5280,20 @@ mod tests { ); } assert!(wait_until( - || persistence.manifest.live_parts().len() > old_parts, + || { + // Old-series epochs may still publish after reactivation. Wait + // for this series, not an unrelated increase in part count. + persistence.manifest.live_parts().iter().any(|part| { + let path = + persistence::part::part_dir_path(&persistence.parts_root, part.part_id); + persistence::part::PartReader::open(&path).is_ok_and(|reader| { + reader + .index_records() + .iter() + .any(|row| row.agg_id == new_sid && row.start_ts < 90_000) + }) + }) + }, Duration::from_secs(5) )); persistence.shutdown(); @@ -5131,20 +5339,32 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); let store = SketchStore::new(); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(850, fingerprint)); + store.register(meta_for_config(850, &state_config)); let generation = store.active_catalog_generation().unwrap(); let writer = Arc::new(persistence::metadata::SidMetadataStore::new( directory.path(), )); *store.persistence_metadata.write().unwrap() = Some(writer.clone()); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: fingerprint.into(), + stored_output_id: fingerprint.into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 30_000, @@ -5179,7 +5399,14 @@ mod tests { assert!(!store.completed_windows.read().unwrap().contains_key(&850)); std::fs::remove_dir(writer.path()).unwrap(); store.seal_finite_summary_input(&generation).unwrap(); - let producers = BTreeMap::from([(fingerprint.into(), "producer".to_string())]); + let producers = BTreeMap::from([( + fingerprint.into(), + ( + asap_types::sds::StoredOutputReference::for_output(fingerprint.into()) + .stored_output_id, + "producer".to_string(), + ), + )]); let inventory = store .observed_summary_inventory("backend", "store", &producers, 1, 30_000) .unwrap(); @@ -5202,7 +5429,9 @@ mod tests { restored .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - restored.register_recovered_disk_series(directory.path()); + restored + .register_recovered_disk_series(directory.path()) + .unwrap(); assert!(!restored.append_sample(850, BTreeMap::new(), (0, 30_000), sample(3))); assert!(restored.append_sample(850, BTreeMap::new(), (30_000, 60_000), sample(4))); } @@ -5219,21 +5448,33 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); { let store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_with_policy(851, fingerprint)); + store.register(meta_for_config(851, &state_config)); let mut config = durable_cfg(directory.path().to_path_buf()); config.hot_window_ms = None; config.seal_window_count = 100; let mut persistence = store.start_persistence(config).unwrap(); let generation = store.active_catalog_generation().unwrap(); let coordinate = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id: fingerprint.into(), + stored_output_id: fingerprint.into(), time_range: HalfOpenTimeRange { start_ms: 0, end_ms: 30_000, @@ -5329,7 +5570,19 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); let disk = directory.path().to_path_buf(); let expected_retirement; @@ -5339,7 +5592,7 @@ mod tests { .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); for sid in [801, 802, 803] { - store.register(meta_with_policy(sid, fingerprint)); + store.register(meta_for_config(sid, &state_config)); } let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for sid in [801, 802, 803] { @@ -5366,7 +5619,7 @@ mod tests { expected_retirement = store.force_retire(801, Duration::from_secs(3600)).unwrap(); assert!(store.force_expire(802).is_some()); assert!(store.remove_instance(803).is_some()); - store.register(meta_with_policy(803, fingerprint)); + store.register(meta_for_config(803, &state_config)); assert!( store.instance(803).is_none(), "removed SID reused before restart" @@ -5396,7 +5649,7 @@ mod tests { recovered.instance(803).is_none(), "removed state resurrected" ); - recovered.register(meta_with_policy(803, fingerprint)); + recovered.register(meta_for_config(803, &state_config)); assert!( recovered.instance(803).is_none(), "removed SID reused after restart" @@ -5413,12 +5666,30 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); - let definition_id = SummaryDefinitionId::from(fingerprint); - let producers = BTreeMap::from([(definition_id, "producer-a".to_string())]); + let state_config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let fingerprint = state_config.policy_fingerprint(); + let definition_id = StoredOutputId::from(fingerprint); + let producers = BTreeMap::from([( + definition_id, + ( + asap_types::sds::StoredOutputReference::for_output(definition_id).stored_output_id, + "producer-a".to_string(), + ), + )]); let tmp = tempfile::TempDir::new().unwrap(); let disk = tmp.path().to_path_buf(); - let mut metadata = meta_with_policy(506, fingerprint); + let mut metadata = meta_for_config(506, &state_config); metadata.group_by_keys = ["job".to_string()].into_iter().collect(); { @@ -6084,4 +6355,91 @@ mod tests { ); assert_eq!(idx.series.len(), 2); } + // Flush and reopen must preserve Planner family rather than reconstructing Rate as Increase. + #[test] + fn planner_exact_families_survive_disk_eviction_and_restart() { + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::storage_engines::types::{AggregateCore, AggregationType}; + let kinds = [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Min, + AggregationType::Max, + AggregationType::Rate, + AggregationType::Increase, + ]; + let stats = [ + asap_types::Statistic::Sum, + asap_types::Statistic::Count, + asap_types::Statistic::Min, + asap_types::Statistic::Max, + asap_types::Statistic::Rate, + asap_types::Statistic::Increase, + ]; + let expected = [16.0, 3.0, 2.0, 8.0, 3.0, 6.0]; + let temp = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + for (i, kind) in kinds.iter().enumerate() { + let mut metadata = meta(9000 + i as u64); + metadata.agg_kind = AggKind::ExactAgg { + agg_type: *kind, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }; + metadata.capability = Some(Capability::ExactAgg(*kind)); + metadata.accuracy = None; + store.register(metadata); + } + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + for window in 0..10u64 { + let mut state = + ExactAccumulator::new(kind.planner_exact_family().unwrap(), false).unwrap(); + for (time, value) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(None, value, time); + } + store.append_precompute( + 9000 + i as u64, + BTreeMap::new(), + (window * 30000, (window + 1) * 30000), + Box::new(state), + ); + } + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.approx_memory_bytes() == 0 + && store.list_sealed_epochs_len() == 0, + std::time::Duration::from_secs(5) + )); + persistence.shutdown(); + } + let store = Arc::new(SketchStore::new()); + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + let series = store.query_exact_agg_range(9000 + i as u64, 0, 30001); + assert_eq!(series.len(), 1, "{kind:?}"); + let state = &series[0].1[&30000]; + assert_eq!(state.get_accumulator_type(), *kind); + assert_eq!( + state + .query_statistic(stats[i], &None, &HashMap::new()) + .unwrap(), + expected[i] + ); + for (j, stat) in stats.iter().enumerate() { + if i != j { + assert!(state + .query_statistic(*stat, &None, &HashMap::new()) + .is_err()); + } + } + } + persistence.shutdown(); + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index d7de8c2bd..f82f06a17 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -195,13 +195,15 @@ mod tests { use super::*; use crate::precompute_engine::operators::SumAccumulator; use crate::storage_engines::types::{AggregationType, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> AggregationConfig { - AggregationConfig { + fn sum_agg_config(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index f9a834934..0b712f456 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::Duration; use crate::storage_engines::types::StreamingConfig; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; use crate::storage_engines::sketch_db::index::SketchStore; @@ -87,7 +87,7 @@ pub fn reconcile_from_streaming_config( } // `live_signatures` is built exclusively from // `signature_from_agg_config`, which canonicalizes every - // streaming-config `AggregationConfig` to an `AggKind::ExactAgg` + // streaming-config `PrecomputeMaterialization` to an `AggKind::ExactAgg` // signature (`P`-prefixed). An `AggKind::Sketch` sid (OTLP // modified-sketch ingest path: KLL / HLL / DDSketch / CMS / // CountSketch) always produces an `S`-prefixed signature, so it @@ -166,7 +166,7 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { +fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { let agg_kind = AggKind::ExactAgg { agg_type: cfg.aggregation_type, parameters_canonical: canonical_parameters(&cfg.parameters), @@ -264,7 +264,7 @@ mod tests { use super::*; use std::collections::HashMap; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -276,8 +276,8 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), HashMap::new(), @@ -321,7 +321,7 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> StreamingConfig { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); diff --git a/data_plane/src/storage_engines/sketch_db/mod.rs b/data_plane/src/storage_engines/sketch_db/mod.rs index e2aff1959..0ef39870c 100644 --- a/data_plane/src/storage_engines/sketch_db/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/mod.rs @@ -16,12 +16,12 @@ pub mod sds; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; pub use backfill::{ - build_backfilled_accumulator, clickhouse_reader_factory, default_reader_factory, - noop_reader_factory, BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, - BackfillWorkerError, ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, - MockRawSampleReader, PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, - ReaderFactory, WindowProcessor, + build_dag_accumulator, clickhouse_reader_factory, default_reader_factory, noop_reader_factory, + BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, BackfillServiceHandle, + BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, BackfillWorkerError, + ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, MockRawSampleReader, + PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, ReaderFactory, + WindowProcessor, }; pub use lifecycle::{ warn_if_retention_inverted, AggStatus, SchemaEvictionConfig, SchemaEvictionHandle, diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs index dcce818b9..4ea55e319 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -34,6 +34,7 @@ fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> Persist || current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() + || current.stored_output_id != record.stored_output_id || current.summary_definition_id != record.summary_definition_id || current.catalog_generation != record.catalog_generation || current.metric_name != record.metric_name diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 552883c1c..e6d8faa24 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -35,7 +35,7 @@ //! //! ## Format //! -//! A single JSON object `{ "": SidMetaRecord, ... }` written +//! A versioned JSON object containing descriptors and series bindings, written //! atomically (tmp + rename) on every upsert. JSON (not the custom //! binary part format) because the record count equals live sid //! cardinality (small) and the schema is human-inspectable for @@ -46,7 +46,7 @@ use std::collections::HashMap; use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -304,6 +304,8 @@ pub struct SidMetaRecord { pub sid: u64, /// Authoritative identity and provenance, absent on legacy sidecars. #[serde(default)] + pub stored_output_id: Option, + #[serde(default)] pub summary_definition_id: Option, #[serde(default)] pub catalog_generation: Option>, @@ -341,6 +343,7 @@ impl SidMetaRecord { ) -> Self { Self { sid, + stored_output_id: None, summary_definition_id: None, catalog_generation: None, metric_name, @@ -412,6 +415,8 @@ struct DataDescriptorRec { struct SidBindingRec { sid: u64, #[serde(default)] + stored_output_id: Option, + #[serde(default)] summary_definition_id: Option, #[serde(default)] catalog_generation_sha256: Option, @@ -432,7 +437,7 @@ struct SidBindingRec { last_immutable: Option, } -/// Version-3 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold +/// Version-4 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold /// foreign keys, mirroring the in-memory SDS registry. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SdsSidecar { @@ -449,7 +454,7 @@ impl SdsSidecar { use crate::storage_engines::sketch_db::sds::{data_descriptor_id, summary_descriptor_id}; let mut sidecar = Self { - schema_version: 3, + schema_version: 4, catalog_generations: HashMap::new(), summary_descriptors: HashMap::new(), data_descriptors: HashMap::new(), @@ -492,6 +497,7 @@ impl SdsSidecar { record.sid.to_string(), SidBindingRec { sid: record.sid, + stored_output_id: record.stored_output_id, summary_definition_id: record.summary_definition_id, catalog_generation_sha256: generation_sha256, summary_descriptor_id: summary_id, @@ -533,6 +539,7 @@ impl SdsSidecar { })?; Ok(SidMetaRecord { sid: binding.sid, + stored_output_id: binding.stored_output_id, summary_definition_id: binding.summary_definition_id, catalog_generation: binding .catalog_generation_sha256 @@ -585,6 +592,67 @@ impl SidMetadataStore { } } + /// Definitions are durable before any output may refer to this generation. + pub(crate) fn persist_catalog( + &self, + catalog: &asap_types::summary_catalog::SummaryCatalog, + ) -> PersistResult<()> { + let reference = catalog + .reference() + .map_err(|e| PersistError::Format(e.to_string()))?; + let _writer = self + .writer + .lock() + .map_err(|_| PersistError::Internal("SID metadata writer poisoned".into()))?; + let path = self.path.with_file_name(format!( + "sds-definitions-{}.json", + reference.snapshot_sha256 + )); + if path.exists() { + let existing = self.load_catalog(&reference)?; + if existing != *catalog { + return Err(PersistError::Format( + "immutable SDS definitions changed".into(), + )); + } + return Ok(()); + } + let bytes = + serde_json::to_vec(catalog).map_err(|e| PersistError::Serialize(e.to_string()))?; + Self::write_atomic_at(&path, &bytes) + } + + pub(crate) fn load_catalog( + &self, + generation: &asap_types::sds::CatalogGeneration, + ) -> PersistResult { + if generation.snapshot_sha256.len() != 64 + || !generation + .snapshot_sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(PersistError::Format("invalid SDS generation digest".into())); + } + let path = self.path.with_file_name(format!( + "sds-definitions-{}.json", + generation.snapshot_sha256 + )); + let catalog: asap_types::summary_catalog::SummaryCatalog = + serde_json::from_slice(&fs::read(path)?) + .map_err(|e| PersistError::Format(e.to_string()))?; + if catalog + .reference() + .map_err(|e| PersistError::Format(e.to_string()))? + != *generation + { + return Err(PersistError::Format( + "persisted SDS definitions differ from generation".into(), + )); + } + Ok(catalog) + } + /// One bounded generation checkpoint closes raw producers across restart. /// Catalog activation with another generation does not inherit this seal. pub(crate) fn persist_finite_closure( @@ -619,68 +687,10 @@ impl SidMetadataStore { &self.path } - /// Load every durable record. Returns an empty vec when the sidecar - /// doesn't exist yet (fresh dir, or parts written before this feature - /// landed) or when it is unparsable (treated as "no recoverable - /// metadata" — the live ingest path still re-registers on first DP). + /// Load current-format durable records. A missing file denotes a fresh store; + /// malformed or unsupported persisted metadata is an error. pub fn load(&self) -> PersistResult> { - let mut f = match File::open(&self.path) { - Ok(f) => f, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(e) => return Err(PersistError::Io(e)), - }; - let mut buf = String::new(); - f.read_to_string(&mut buf)?; - if buf.trim().is_empty() { - return Ok(Vec::new()); - } - let value: serde_json::Value = match serde_json::from_str(&buf) { - Ok(value) => value, - Err(e) => { - tracing::warn!( - path = %self.path.display(), - error = %e, - "sid metadata sidecar unparsable; ignoring (live ingest will re-register)" - ); - return Ok(Vec::new()); - } - }; - if matches!( - value.get("schema_version").and_then(|v| v.as_u64()), - Some(2 | 3) - ) { - let sidecar: SdsSidecar = match serde_json::from_value(value) { - Ok(sidecar) => sidecar, - Err(error) => { - tracing::warn!( - path = %self.path.display(), - %error, - "SDS metadata sidecar is invalid; ignoring" - ); - return Ok(Vec::new()); - } - }; - return match sidecar.into_records() { - Ok(records) => Ok(records), - Err(error) => { - tracing::warn!( - path = %self.path.display(), - %error, - "SDS metadata sidecar has broken descriptor references; ignoring" - ); - Ok(Vec::new()) - } - }; - } - // Version 1 was a flat SeriesId map. Read it and normalize on the next write. - let map: HashMap = match serde_json::from_value(value) { - Ok(map) => map, - Err(error) => { - tracing::warn!(path = %self.path.display(), %error, "legacy sid metadata is invalid; ignoring"); - return Ok(Vec::new()); - } - }; - Ok(map.into_values().collect()) + self.load_strict() } /// Upsert a batch of records, merging with whatever is already on @@ -738,20 +748,18 @@ impl SidMetadataStore { }; let value: serde_json::Value = serde_json::from_slice(&bytes) .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; - if let Some(version) = value.get("schema_version") { - if !matches!(version.as_u64(), Some(2 | 3)) { - return Err(PersistError::Format( - "unsupported SID metadata version".into(), - )); - } - let sidecar: SdsSidecar = serde_json::from_value(value) - .map_err(|error| PersistError::Format(error.to_string()))?; - sidecar.into_records() - } else { - let records: HashMap = serde_json::from_value(value) - .map_err(|error| PersistError::Format(error.to_string()))?; - Ok(records.into_values().collect()) + if value + .get("schema_version") + .and_then(serde_json::Value::as_u64) + != Some(4) + { + return Err(PersistError::Format( + "unsupported SID metadata version".into(), + )); } + let sidecar: SdsSidecar = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + sidecar.into_records() } pub(super) fn transaction( @@ -845,6 +853,26 @@ mod tests { assert!(s.load().unwrap().is_empty()); } + // Restart must validate the immutable semantic document, not just its filename. + #[test] + fn persisted_definitions_roundtrip_and_reject_tampering() { + let directory = tempfile::tempdir().unwrap(); + let store = SidMetadataStore::new(directory.path()); + let catalog = asap_types::summary_catalog::SummaryCatalog::build(7, 2, []).unwrap(); + store.persist_catalog(&catalog).unwrap(); + let generation = catalog.reference().unwrap(); + assert_eq!(store.load_catalog(&generation).unwrap(), catalog); + let path = store.path.with_file_name(format!( + "sds-definitions-{}.json", + generation.snapshot_sha256 + )); + let mut altered = serde_json::to_value(&catalog).unwrap(); + altered["plan_version"] = serde_json::json!(3); + std::fs::write(&path, serde_json::to_vec(&altered).unwrap()).unwrap(); + assert!(store.load_catalog(&generation).is_err()); + assert!(store.persist_catalog(&catalog).is_err()); + } + #[test] fn authoritative_bindings_share_one_persisted_catalog_generation() { let directory = tempfile::tempdir().unwrap(); @@ -856,7 +884,7 @@ mod tests { snapshot_sha256: "catalog".into(), }); let mut first = sketch_meta(1); - first.summary_definition_id = Some(asap_types::PolicyFingerprint(7).into()); + first.stored_output_id = Some(asap_types::PolicyFingerprint(7).into()); first.catalog_generation = Some(std::sync::Arc::clone(&generation)); let mut second = first.clone(); second.sid = 2; @@ -871,7 +899,7 @@ mod tests { records[1].catalog_generation.as_ref().unwrap() )); assert_eq!( - records[0].summary_definition_id, + records[0].stored_output_id, Some(asap_types::PolicyFingerprint(7).into()) ); } @@ -890,7 +918,7 @@ mod tests { let persisted: serde_json::Value = serde_json::from_slice(&std::fs::read(s.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); + assert_eq!(persisted["schema_version"], 4); assert_eq!( persisted["summary_descriptors"].as_object().unwrap().len(), 2 @@ -936,22 +964,25 @@ mod tests { .remove(&missing_id); std::fs::write(store.path(), serde_json::to_vec(&persisted).unwrap()).unwrap(); - assert!(store.load().unwrap().is_empty()); + assert!(store.load().is_err()); } #[test] - fn legacy_flat_sidecar_is_read_and_migrated_on_write() { + fn unsupported_sidecars_are_rejected_without_overwriting() { let tmp = TempDir::new().unwrap(); let store = SidMetadataStore::new(tmp.path()); - let legacy = HashMap::from([("1".to_string(), sketch_meta(1))]); - std::fs::write(store.path(), serde_json::to_vec(&legacy).unwrap()).unwrap(); - - assert_eq!(store.load().unwrap(), vec![sketch_meta(1)]); - store.upsert_all(&[exact_meta(2)]).unwrap(); - let persisted: serde_json::Value = - serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); - assert_eq!(store.load().unwrap().len(), 2); + let flat = serde_json::to_value(HashMap::from([("1", sketch_meta(1))])).unwrap(); + let mut cases = vec![flat]; + for version in [1, 2, 3, 5, 999] { + cases.push(serde_json::json!({"schema_version": version})); + } + for value in cases { + let bytes = serde_json::to_vec(&value).unwrap(); + std::fs::write(store.path(), &bytes).unwrap(); + assert!(store.load().is_err()); + assert!(store.upsert_all(&[exact_meta(2)]).is_err()); + assert_eq!(std::fs::read(store.path()).unwrap(), bytes); + } } #[test] @@ -989,10 +1020,10 @@ mod tests { } #[test] - fn unparsable_file_loads_as_empty() { + fn unparsable_file_is_rejected() { let tmp = TempDir::new().unwrap(); let s = SidMetadataStore::new(tmp.path()); std::fs::write(s.path(), b"{not json").unwrap(); - assert!(s.load().unwrap().is_empty()); + assert!(s.load().is_err()); } } diff --git a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs index 26341f942..894b15ee6 100644 --- a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs +++ b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs @@ -283,9 +283,8 @@ impl SummaryState { SummaryState::Dd(sk) => { match encoding { // PROTO_DELTA: dispatch on the payload SHAPE, mirroring the - // edge's own `DDSketchWrapper::apply_delta` - // (asap-precompute-rs/src/sketches/ddsketch.rs) — which - // tries the full-envelope decode first, then falls back to + // supported DDSketch frame decoder, which tries the + // full-envelope decode first, then falls back to // the bucket-delta proto. Two wire shapes can arrive on the // ProtoDelta channel: // diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 83778690e..ca7f7ec2d 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -75,12 +75,55 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { }) } +// A valid deployment reference does not authorize bytes in another state format. +fn operator_matches(summary: &SummaryDescriptor, actual: &AggKind) -> bool { + match &summary.operator { + SummaryOperator::Configured { + aggregation_type, + aggregation_sub_type, + parameters, + .. + } => { + let config = asap_types::PrecomputeMaterialization::new( + *aggregation_type, + aggregation_sub_type.clone(), + parameters.clone().into_iter().collect(), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::WindowKind::Tumbling, + String::new(), + String::new(), + None, + None, + None, + ); + super::data::agg_kind_for_config(&config).operator_canonical_string() + == actual.operator_canonical_string() + } + SummaryOperator::ExactAgg { + agg_type, + parameters_canonical, + } => matches!(actual, + AggKind::ExactAgg { agg_type: actual, parameters_canonical: parameters, .. } if actual == agg_type && parameters == parameters_canonical), + SummaryOperator::LegacyPartial { operator_canonical } => { + actual.operator_canonical_string() == *operator_canonical + } + // Bound runtime plans carry the complete Configured state contract. + SummaryOperator::Sketch { .. } => false, + } +} + /// Runtime foreign-key binding from one SeriesId to shared descriptors. Every pane /// row stored under the SeriesId is a Summary Instance: `(binding, interval, /// group-values, state)`. Descriptor references are normalized here instead of /// copied into every pane row. #[derive(Debug, Clone)] pub struct SdsBinding { + pub stored_output_reference: Option, pub metadata: Arc, pub summary_descriptor: Arc, pub data_descriptor: Arc, @@ -156,18 +199,19 @@ impl SummaryDescriptorRegistry { "materialization identity is required by the installed SummaryCatalog".into(), ); } - let materialization = asap_types::sds::SummaryDefinitionId::from(metadata.policy_fp); - let identity = catalog - .materializations - .get(&materialization) - .ok_or_else(|| { - format!( - "materialization {} is absent from the installed SummaryCatalog", - materialization.as_u64() - ) - })?; + let materialization = asap_types::sds::StoredOutputId::from(metadata.policy_fp); + let identity = catalog.outputs.get(&materialization).ok_or_else(|| { + format!( + "materialization {} is absent from the installed SummaryCatalog", + materialization.as_u64() + ) + })?; + let summary = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !operator_matches(summary, &metadata.agg_kind) { + return Err("stored output state format differs from installed definition".into()); + } Some(( - catalog.summary_descriptors[&identity.summary_descriptor_id].clone(), + summary.clone(), catalog.data_descriptors[&identity.data_descriptor_id].clone(), )) } else { @@ -208,7 +252,16 @@ impl SummaryDescriptorRegistry { } }; + let stored_output_reference = authoritative + .as_ref() + .map(|(catalog, _)| { + catalog + .output_reference(metadata.policy_fp.into()) + .map_err(|e| e.to_string()) + }) + .transpose()?; Ok(SdsBinding { + stored_output_reference, metadata: Arc::new(metadata), summary_descriptor, data_descriptor, @@ -385,7 +438,7 @@ mod tests { let summary = SummaryDescriptor::new( SummaryOperator::ExactAgg { agg_type: AggregationType::Sum, - parameters_canonical: "authoritative=true".into(), + parameters_canonical: "pane=5000;".into(), }, FidelityGuarantee::Exact, 1, @@ -399,15 +452,20 @@ mod tests { asap_types::PolicyFingerprint(7), summary.clone(), data.clone(), - asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, )], ) .unwrap(); let registry = SummaryDescriptorRegistry::default(); registry.install_catalog(Arc::new(catalog)).unwrap(); + assert!( + registry + .bind(metadata(1, "wrong-local-copy", "", AggregationType::Max, 7)) + .is_err(), + "an output ID must not authorize a different state family" + ); let binding = registry - .bind(metadata(1, "wrong-local-copy", "", AggregationType::Max, 7)) + .bind(metadata(1, "cpu", "", AggregationType::Sum, 7)) .unwrap(); assert_eq!(binding.summary_descriptor.as_ref(), &summary); assert_eq!(binding.data_descriptor.as_ref(), &data); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c679d0bcd..fc9062f1e 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -680,7 +680,7 @@ impl ActivePhysicalPlanHandle { #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::AggregationConfig; + use crate::storage_engines::types::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -710,11 +710,12 @@ mod tests { summary_catalog: None, envelope: envelope.clone(), ingest: asap_types::precompute_plan::IngestContract { + dataset_identity: None, protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "/v1/metrics".into(), timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: true, - require_summary_definition_identity: true, + require_stored_output_identity: true, require_registered_producer: true, }, schemas: Vec::new(), @@ -748,14 +749,15 @@ mod tests { plan_id, plan_version, clickhouse_context: None, + selected_dags: Default::default(), entries: Default::default(), }), storage_routing: Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), } } - fn dummy_agg(id: u64) -> AggregationConfig { - AggregationConfig::new( + fn dummy_agg(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 57d47470b..a91e83148 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -25,7 +25,7 @@ pub use streaming_config::*; pub use traits::*; // Cross-module re-export of asap_types data types so callers can -// write `crate::storage_engines::types::AggregationConfig` instead of +// write `crate::storage_engines::types::PrecomputeMaterialization` instead of // reaching across crates. pub use asap_types::aggregation_config::*; diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 4d268dc41..28e71b4c3 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -62,7 +62,7 @@ pub struct PrecomputedOutput { #[serde(default)] pub origin: Origin, /// Content-addressed policy identity. The data plane's only handle - /// on which source `AggregationConfig` produced this output. + /// on which source `PrecomputeMaterialization` produced this output. /// `#[serde(default)]` on read preserves forward-compat with /// PR-3 / PR-4-era records that may not have carried the field; /// sinks treat `PolicyFingerprint::UNSET` as "skip this output" @@ -75,7 +75,7 @@ impl PrecomputedOutput { /// Construct a `Native` precompute. /// /// `policy_fp` is the content-addressed handle on the source - /// [`asap_types::AggregationConfig`]; sinks use it to look up the + /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the /// config via `PolicyRegistry::get(policy_fp)`. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 430dd10bf..95055c2ea 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -6,31 +6,22 @@ use std::fs::File; use std::io::BufReader; use std::ops::Index; -use asap_types::enums::QueryLanguage; -use asap_types::{AggregationConfig, MonitorSpec, PolicyRegistry}; +use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; use super::storage_backend::StorageBackend; -/// The backend's active streaming policy config: every `AggregationConfig` -/// currently pushed by the controller, plus the storage-backend pin and CDM -/// monitor specs. -/// -/// Formerly `asap_types::streaming_config::StreamingConfig` — moved here -/// (see `scratchpad/artifacts/enum-unification-plan.md`) because -/// `control_plane` never actually depended on this type: its own -/// `StreamingConfigEmitter` hand-builds wire-compatible JSON independently, -/// and `PolicyRegistry::from_streaming_config` (the only thing that made -/// `asap_types::PolicyRegistry` -- genuinely shared -- look coupled to this -/// type) had exactly one real caller, this struct's own `policy_registry()` -/// method below. `asap_types` keeps the lower-level `PolicyRegistry:: -/// from_configs` primitive this method now calls directly. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// DAG installation plus a derived in-memory routing index. The flat index is +/// never serialized as executable configuration. Raw programs are validated +/// and shared once per installed producer across all of its population states. +#[derive(Debug, Clone, Serialize)] pub struct StreamingConfig { - #[serde( - rename = "aggregation_configs", - alias = "materializations_by_policy_fingerprint" - )] - pub materializations_by_policy_fingerprint: HashMap, + #[serde(skip)] + pub(crate) raw_programs: + HashMap>, + /// Authoritative execution configuration: Planner DAGs and physical bindings. + pub precompute_plan: Option, + #[serde(skip)] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -45,15 +36,60 @@ pub struct StreamingConfig { pub monitors: Vec, } +// Flat aggregation lists are deliberately not an accepted execution document. +impl<'de> Deserialize<'de> for StreamingConfig { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + precompute_plan: asap_types::precompute_plan::PrecomputePlan, + #[serde(default)] + storage_backend: StorageBackend, + #[serde(default)] + monitors: Vec, + } + let doc = Document::deserialize(deserializer)?; + let mut config = + Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; + config.storage_backend = doc.storage_backend; + config.monitors = doc.monitors; + Ok(config) + } +} + impl StreamingConfig { - pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { + pub fn new( + materializations_by_policy_fingerprint: HashMap, + ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } } + /// Build the routing projection only after validating the DAG installation. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|c| { + c.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::new(materializations); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + /// CDM monitor specs the data-plane coordinator should serve (may be empty). pub fn monitors(&self) -> &[MonitorSpec] { &self.monitors @@ -63,10 +99,12 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), @@ -80,12 +118,15 @@ impl StreamingConfig { self.storage_backend } - pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { + pub fn get_aggregation_config( + &self, + aggregation_id: u64, + ) -> Option<&PrecomputeMaterialization> { self.materializations_by_policy_fingerprint .get(&aggregation_id) } - pub fn materializations(&self) -> &HashMap { + pub fn materializations(&self) -> &HashMap { &self.materializations_by_policy_fingerprint } @@ -126,56 +167,12 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut materializations_by_policy_fingerprint: HashMap = - HashMap::new(); - - if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { - for aggregation_data in aggregations { - // Retention comes from each aggregation entry; identity is derived from - // its configuration content. - let num_aggregates_to_retain = aggregation_data - .get("numAggregatesToRetain") - .and_then(|v| v.as_u64()); - let config = AggregationConfig::from_yaml_data( - aggregation_data, - num_aggregates_to_retain, - QueryLanguage::PromQl, - )?; - if !config.population_key_encoding.is_legacy() { - anyhow::bail!( - "legacy streaming input does not support this population key encoding" - ); - } - if config.derived_input.is_some() { - anyhow::bail!( - "legacy streaming input cannot execute a derived summary program" - ); - } - // PR 5: the map key IS the policy-fingerprint u64. - // `AggregationConfig::policy_fp_u64()` is the canonical - // accessor for this value. - materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); - } - } - - let mut config = Self::new(materializations_by_policy_fingerprint); - // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each - // entry deserializing into a MonitorSpec. Absent → empty (the common - // case). The data-plane monitor coordinator reads these. - if let Some(monitors) = data.get("monitors").and_then(|v| v.as_sequence()) { - for m in monitors { - let spec: MonitorSpec = serde_yaml::from_value(m.clone()).map_err(|e| { - anyhow::anyhow!("invalid monitor spec in streaming-config: {e}") - })?; - config.monitors.push(spec); - } - } - Ok(config) + serde_yaml::from_value(data.clone()).map_err(Into::into) } } impl Index for StreamingConfig { - type Output = AggregationConfig; + type Output = PrecomputeMaterialization; fn index(&self, aggregation_id: u64) -> &Self::Output { &self.materializations_by_policy_fingerprint[&aggregation_id] @@ -190,7 +187,7 @@ impl Default for StreamingConfig { impl StreamingConfig { #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { + pub fn get_all_aggregation_configs(&self) -> &HashMap { self.materializations() } } @@ -199,153 +196,16 @@ impl StreamingConfig { mod tests { use super::*; - /// Pre-Phase-5 deploys serialize `StreamingConfig` without the - /// `storage_backend` field; deserialize must default to `SketchStore` - /// so the router keeps dispatching to `ASAPQueryEngine` unchanged. - #[test] - fn deserialize_legacy_yaml_defaults_to_asap_tier() { - let yaml = "{\"aggregation_configs\":{}}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("legacy decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::SketchStore); - } - - #[test] - fn deserialize_with_explicit_double_write_pin() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"double_write\"}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("Phase-5 decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::DoubleWrite); - } - - /// #746 deleted the archive tier; its storage-axis spelling is no longer - /// a known variant, so a stale config naming it fails to decode rather - /// than silently pinning some other tier. + // Old flat lists cannot become execution authority through JSON or YAML. #[test] - fn deserialize_rejects_the_removed_archive_axis() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_object_store\"}"; - assert!(serde_json::from_str::(yaml).is_err()); - } - - #[test] - fn legacy_yaml_rejects_derived_summary_input() { - let data = serde_yaml::from_str::(&format!( - "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) - )).unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!(error - .to_string() - .contains("legacy streaming input cannot execute")); - } - - #[test] - fn legacy_yaml_rejects_canonical_population_key_encoding() { - let data: Value = serde_yaml::from_str( - r#" -aggregations: -- aggregationType: Sum - aggregationSubType: '' - metric: m - population_key_encoding: canonical_labels_v1 - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#, - ) - .unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!( - error.to_string().contains("population key encoding"), - "{error}" - ); - } - - /// PR 5: a streaming-config YAML that omits `aggregationId` - /// parses correctly — the backend derives identity from content - /// via `PolicyFingerprint::from_config`. The map key is the - /// fingerprint's u64 form. - #[test] - fn from_yaml_data_accepts_entry_without_aggregation_id() { - let yaml = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); - let (k, v) = cfg - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_ne!(*k, 0, "derived id is not the 0 sentinel"); - assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); - assert_eq!(v.metric, "cpu_seconds"); - } - - /// PR 5: a streaming-config YAML that still spells out - /// `aggregationId: N` parses the SAME as one without — the field - /// is silently dropped. - #[test] - fn from_yaml_data_ignores_explicit_aggregation_id() { - let with = "\ -aggregations:\n\ -- aggregationId: 42\n aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let without = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let w: Value = serde_yaml::from_str(with).expect("with yaml ok"); - let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); - let cw = StreamingConfig::from_yaml_data(&w).expect("with"); - let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - let (kwo, _) = cwo - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_eq!( - kw, kwo, - "explicit aggregationId in YAML must not change identity" - ); - assert_ne!( - *kw, 42, - "the explicit value must NOT leak through as the map key" - ); - } - - #[test] - fn from_yaml_data_parses_monitors_section() { - // CDM monitor specs: a top-level `monitors:` array must populate - // StreamingConfig.monitors (the data-plane coordinator reads these). - let yaml = "\ -aggregations: []\n\ -monitors:\n\ -- agg_id: 16346598078036168951\n key: \"\"\n tau: 5000.0\n epsilon: 0.05\n window_ms: 10000\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode monitors"); - assert_eq!(cfg.monitors().len(), 1, "monitors: section must be parsed"); - let m = &cfg.monitors()[0]; - assert_eq!(m.agg_id, 16346598078036168951); - assert_eq!(m.tau, 5000.0); - assert_eq!(m.window_ms, 10000); - assert_eq!(m.epsilon, 0.05); - } - - #[test] - fn from_yaml_data_absent_monitors_is_empty() { - let yaml = "aggregations: []\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode"); - assert!( - cfg.monitors().is_empty(), - "no monitors: → empty (byte-compat)" - ); + fn rejects_flat_aggregation_documents() { + for text in [ + r#"{"aggregation_configs":{}}"#, + "aggregations: []", + "aggregations: [{aggregationType: Sum, metric: m}]", + ] { + let yaml = serde_yaml::from_str(text).unwrap(); + assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); + } } } diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index ed88c51d9..21cd2b115 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,7 +27,7 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -35,8 +35,8 @@ use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( +fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 5158398bd..974673941 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -2,14 +2,14 @@ //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore //! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build AggregationConfig with +//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with //! the correct aggregation_type string. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationConfig, AggregationType, KeyByLabelValues, PrecomputedOutput, QueryLanguage, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, StreamingConfig, WindowKind, }; use crate::AggregateCore; @@ -23,7 +23,7 @@ use std::collections::HashMap; fn ingest_with_fresh_resolver( summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, - agg_cfg: &AggregationConfig, + agg_cfg: &PrecomputeMaterialization, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { @@ -89,7 +89,9 @@ pub fn create_engine_single_pop_with_aggregated( .collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -119,6 +121,8 @@ pub fn create_engine_single_pop_with_aggregated( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -175,7 +179,9 @@ pub fn create_engine_dual_input( let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation - let value_agg_config = AggregationConfig { + let value_agg_config = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: value_agg_type, aggregation_sub_type: String::new(), @@ -205,7 +211,9 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation - let keys_agg_config = AggregationConfig { + let keys_agg_config = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: key_agg_type, aggregation_sub_type: String::new(), @@ -235,6 +243,8 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -300,7 +310,9 @@ pub fn create_engine_two_metrics( let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config_a = AggregationConfig { + let agg_config_a = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: aggregation_type_a, aggregation_sub_type: String::new(), @@ -329,7 +341,9 @@ pub fn create_engine_two_metrics( let id_a = agg_config_a.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_a, agg_config_a); - let agg_config_b = AggregationConfig { + let agg_config_b = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: aggregation_type_b, aggregation_sub_type: String::new(), @@ -359,6 +373,8 @@ pub fn create_engine_two_metrics( materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -434,7 +450,9 @@ pub fn create_engine_three_metrics( (aggregation_type_b, &labels_b, metric_b), (aggregation_type_c, &labels_c, metric_c), ] { - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type: agg_type, aggregation_sub_type: String::new(), @@ -466,6 +484,8 @@ pub fn create_engine_three_metrics( } let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -516,7 +536,9 @@ pub fn create_engine_multi_timestamp( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -546,6 +568,8 @@ pub fn create_engine_multi_timestamp( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -574,7 +598,7 @@ pub fn create_engine_multi_timestamp( /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the AggregationConfig (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -590,7 +614,9 @@ pub fn create_engine_multi_timestamp_with_window( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { + stored_output_id: None, + semantic_fragment: None, population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -620,6 +646,8 @@ pub fn create_engine_multi_timestamp_with_window( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), diff --git a/data_plane/src/tests/test_utilities/planning.rs b/data_plane/src/tests/test_utilities/planning.rs index f05dac22e..4a30ba1f0 100644 --- a/data_plane/src/tests/test_utilities/planning.rs +++ b/data_plane/src/tests/test_utilities/planning.rs @@ -1,6 +1,6 @@ //! Synthetic complete quotes for deployment fixtures; never used in production. use control_plane::physical::compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }; pub(crate) fn quoted_snapshot( @@ -20,9 +20,9 @@ pub(crate) fn quoted_snapshot( .enumerate() .filter_map(|(index, candidate)| { let plan = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b56408a24..a10cd3d14 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,4 +1,4 @@ -use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{KeyedSumCountAccumulator, SumAccumulator}; #[cfg(test)] use crate::storage_engines::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, @@ -18,7 +18,7 @@ fn test_single_subpopulation_interface() { #[test] fn test_multiple_subpopulation_interface() { // Multiple accumulator - matches Python behavior exactly - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let mut key = KeyByLabelValues::new(); key.insert("web".to_string()); @@ -43,7 +43,7 @@ fn test_interface_prevents_misuse() { let single_acc: Box = Box::new(SumAccumulator::with_sum(42.0)); let multi_acc: Box = - Box::new(MultipleSumAccumulator::new()); + Box::new(KeyedSumCountAccumulator::new()); // ✅ These work - correct usage let _result1 = single_acc.query(Statistic::Sum, None); @@ -68,7 +68,7 @@ fn test_python_alignment() { // Python: multiple_accumulator.query(Statistic.SUM, key) // Rust: multiple_accumulator.query(Statistic::Sum, &key) - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new(); multi_acc.add_sum(key.clone(), 100.0); let multi_trait: Box = Box::new(multi_acc); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 5150ddf0e..6f33ca87b 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -20,7 +20,7 @@ mod tests { use tempfile::NamedTempFile; #[test] - fn test_read_streaming_config() { + fn flat_streaming_file_is_rejected() { // PR 5: `aggregationId: 1` is silently dropped on read — the // streaming-config map key is the policy fingerprint derived // from content. The legacy field stays in this fixture to @@ -47,9 +47,6 @@ aggregations: let mut streaming_temp_file = NamedTempFile::new().unwrap(); write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.materializations_by_policy_fingerprint.is_empty()); - let agg = config.materializations().values().next().expect("one agg"); - assert_eq!(agg.num_aggregates_to_retain, Some(6)); + assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); } } diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index 9c8e442f0..9cee54846 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -20,7 +20,9 @@ use asap_otel_proto::tonic::metrics::v1::{ KllSketch as OtelKllSketch, KllSketchDataPoint, KllSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::{HllVariant as ProtoHllVariant, HyperLogLogState, KllState}; +use asap_sketchlib::proto::sketchlib::{ + sketch_envelope, HllVariant as ProtoHllVariant, HyperLogLogState, KllState, SketchEnvelope, +}; use asap_sketchlib::{CountMinSketch, CountSketch, HllSketch, HllVariant, MessagePackCodec}; use prost::Message; use serde_json::Value; @@ -107,22 +109,23 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(config_yaml: &str) -> Backend { +async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - config - .write_all(config_yaml.as_bytes()) - .expect("write streaming config"); - config.flush().expect("flush streaming config"); - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_str(config_yaml).unwrap(), + let mut install = + physical_fixture::artifact_from_materializations(vec![materialization.clone()]); + serde_yaml::to_writer( + &mut config, + &serde_json::json!({ + "precompute_plan": install.precompute_plan, + }), ) .unwrap(); + config.flush().unwrap(); let mut physical = tempfile::NamedTempFile::new().unwrap(); - let mut install = physical_fixture::artifact(&runtime); for rule in &mut install.transmission_plan.rules { if matches!( install @@ -292,9 +295,24 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { .collect() } -fn config(metric: &str, kind: &str, parameters: &str) -> String { - format!( - "aggregations:\n - aggregationType: {kind}\n aggregationSubType: ''\n labels:\n grouping: [service]\n rollup: []\n aggregated: []\n metric: {metric}\n parameters:\n{parameters}\n windowSize: 1\n windowType: tumbling\n spatialFilter: ''\n" +fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { + use asap_types::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + PrecomputeMaterialization::new( + kind.parse().unwrap(), + String::new(), + serde_yaml::from_str(parameters).unwrap(), + KeyByLabelNames::new(vec!["service".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 1, + 1, + WindowKind::Tumbling, + String::new(), + metric.into(), + None, + None, + None, ) } @@ -317,7 +335,12 @@ fn kll_export(metric: &str, timestamp_ns: u64, raw: &[f64]) -> ExportMetricsServ attributes: labels(), start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec(), encoding: KllSketchEncoding::Proto as i32, flags: 0, series_id: 0, diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 55d9b165f..99fae4b0a 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -46,7 +46,7 @@ fn quote_snapshot_for_frontend_test( metricsql: bool, ) -> control_plane::physical::compiler::BackendLocalPlanningInput { use control_plane::physical::{ - compiler::{PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, + compiler::{DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; let (request, environment) = snapshot @@ -59,9 +59,9 @@ fn quote_snapshot_for_frontend_test( .into_iter() .filter_map(|candidate| { let plan = if metricsql { - PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) + DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let unit_cost = if preferred { 1.0 } else { 1e12 }; @@ -313,17 +313,13 @@ fn is_warm(response: &Value) -> bool { }) } -// Measured ERP parameters must reach the real accumulator and answer held-out +// Confidence-sized KLL parameters must reach the real accumulator and answer // raw samples through the installed QueryPlan, without native fallback. +// Uncertified ERP maxima have separate exact-routing process coverage. #[tokio::test] -#[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] -async fn erp_measured_kll_collector_to_query_oracle() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; +async fn certified_kll_state_to_query_oracle() { + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; const QUERY: &str = "quantile_over_time(0.9, erp_latency[5s])"; - let artifact: Value = serde_json::from_str(include_str!( - "../../control_plane/tests/fixtures/erp-kll-measured.json" - )) - .unwrap(); let mut fixture: Value = serde_json::from_str(include_str!( "../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) @@ -332,15 +328,19 @@ async fn erp_measured_kll_collector_to_query_oracle() { entry["query"] = QUERY.into(); entry["requirements"]["accuracy"] = serde_json::json!({"explicit": {"Epsilon": 0.06}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + // This collector fixture exports KLL state only. An empty ERP artifact + // keeps runtime capability filtering while requiring theoretical sizing. fixture["implementation"]["erp"] = serde_json::json!({ - "distribution": artifact["records"][0]["distribution"], - "artifact": artifact, "implementation": "lib", "error_metric": "max_rank_err", + "distribution": {"workload": {"external": {"dataset": "kll-process-fixture"}}}, + "artifact": {"schema_version": 1, "producer_version": "test", "records": []}, + "implementation": "lib", "error_metric": "max_rank_err", "min_trials": 10, "expected_updates": 1000.0, "expected_queries": 10.0, "expected_merges": 0.0, "retention_seconds": 60.0, "cpu_weight": 1.0, "byte_second_weight": 1e-9, "mode": "hybrid", "runtime": {"allowed_algorithms": ["Kll"], "max_memory_bytes": null} }); let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let window_model = snapshot.physical_inputs.window_cost_model.clone(); let (mut request, mut environment) = snapshot.into_physical_compilation_request().unwrap(); request.allow_mixed_summary_and_exact_execution = false; request.queries[0].group_by_labels = vec!["service".into()]; @@ -355,11 +355,34 @@ async fn erp_measured_kll_collector_to_query_oracle() { environment.target = control_plane::physical::compiler::PhysicalDeploymentTarget::DistributedCollectors; environment.target_collector_ids = vec!["erp-collector".into()]; - let plan = PhysicalPlanCompiler + control_plane::physical::compiler::prepare_window_implementations( + &mut request.queries[0], + &window_model, + environment.target, + request.query_retention_margin_ms, + ) + .unwrap(); + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); + let k = plan.precompute_plan.materializations[0].parameters["k"] + .as_u64() + .unwrap() as u32; + let guarantee = asap_aware_mapping::DefaultAccuracyModel::sketch_guarantee( + &planner_types::post_asap::SketchAlgorithm::Kll, + &planner_types::post_asap::SketchParams::Kll { k }, + &planner_types::post_asap::SketchQuery::Quantile { q: 0.9 }, + ) + .unwrap(); + assert!(asap_aware_mapping::AccuracyModel::satisfies( + &asap_aware_mapping::DefaultAccuracyModel, + &guarantee, + &planner_types::types::AccuracyTarget::EpsilonDelta { + epsilon: 0.06, + delta: 0.01 + } + )); let collector = serde_json::to_value(&plan.collector_plans[0]).unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -379,7 +402,7 @@ async fn erp_measured_kll_collector_to_query_oracle() { let mut bootstrap_config = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer( &mut bootstrap_config, - &serde_json::json!({"aggregations": []}), + &serde_json::json!({"precompute_plan": install.precompute_plan}), ) .unwrap(); let mut child = ChildGuard( @@ -416,8 +439,7 @@ async fn erp_measured_kll_collector_to_query_oracle() { .unwrap() .as_millis() as i64; let base = now - now.rem_euclid(5000) - 20000; - // A different deterministic stream from training seed 42; the oracle - // evaluates rank error, not the unrelated relative error of the value. + // The oracle evaluates rank error, not relative error of the value. let raw: Vec = (0..1000) .map(|i| ((i * 7919 + 17) % 1009) as f64 / 1009.0) .collect(); @@ -457,7 +479,7 @@ async fn erp_measured_kll_collector_to_query_oracle() { } }) .await - .expect("ERP plan must answer without exact fallback"); + .expect("certified KLL plan must answer without exact fallback"); let estimate = first_value(&response, "value").expect("numeric estimate"); let rank = raw.iter().filter(|v| **v <= estimate).count() as f64 / raw.len() as f64; assert!( @@ -475,45 +497,20 @@ fn erp_collector_kll_export(plan: &Value, end_ms: u64, raw: &[f64], sequence: u6 ResourceMetrics, ScopeMetrics, }, }; - use asap_precompute_rs::Precompute; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; - let decoded = asap_precompute_rs::CollectorPlan::from_json( - &serde_json::to_vec(plan).unwrap(), - "erp-collector", - ) - .unwrap(); - let config = decoded - .to_precompute_config_set() + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_value(plan.clone()).unwrap(); + assert_eq!(decoded.materializations.len(), 1); + let k = decoded.materializations[0].parameters["k"] + .as_u64() .unwrap() - .configs - .remove(0); - let k = config.sketch_params["k"] as i32; - assert_eq!(k, 32); - let runtime = asap_precompute_rs::precompute::PrecomputeImpl::new( - Some(config), - Some(Box::new(move || { - Box::new(asap_precompute_rs::sketches::KLLWrapper::new(k, Some(123))) - })), - Some(Box::new(asap_precompute_rs::sketches::KLLObserver)), - ); + .try_into() + .unwrap(); + let mut sketch = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(k, 123); for value in raw { - runtime - .observe(&asap_precompute_rs::Observation::new( - end_ms - 500, - "erp_latency", - vec![], - vec![asap_precompute_rs::KeyValue::new("service", "erp")], - asap_precompute_rs::ObservationValue { - kind: asap_precompute_rs::ObservationValueKind::Float, - float: *value, - ..Default::default() - }, - )) - .unwrap(); + sketch.update(value); } - let envelopes = runtime.tick(end_ms); - assert_eq!(envelopes.len(), 1); - let wire = SketchEnvelope::decode(envelopes[0].payload.as_slice()).unwrap(); + let wire = SketchEnvelope::decode(asap_sketch_codec::encode_kll(&sketch).as_slice()).unwrap(); let Some(sketch_envelope::SketchState::Kll(state)) = wire.sketch_state else { panic!("KLL state required") }; @@ -576,7 +573,12 @@ fn erp_collector_kll_export(plan: &Value, end_ms: u64, raw: &[f64], sequence: u6 attributes, start_time_unix_nano: (end_ms - 5000) * 1_000_000, time_unix_nano: end_ms * 1_000_000, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Kll(state)), + ..Default::default() + } + .encode_to_vec(), encoding: KllSketchEncoding::Proto as i32, flags: 0, series_id: 0, @@ -602,7 +604,7 @@ async fn registered_temporal_topk_count_sketch_heap() { } async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlgorithm) { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; use planner_types::post_asap::{CompositionOperator, SketchQuery, SummaryFamilyType}; const QUERY: &str = "topk(3, count_over_time(top_endpoint_qps[5s]))"; struct Evidence; @@ -649,10 +651,47 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg query.accuracy_target.clone(), ) .unwrap(); - let model = control_plane::physical::post_asap::cost_model::ForcedFamilyCostModel::new( - query.accuracy_target.clone(), - algorithm.clone(), - ); + // Family ordering alone does not select a whole DAG. Price the desired + // legal heap candidate explicitly so this process test exercises its runtime. + struct HeapFixtureCost(planner_types::post_asap::SketchAlgorithm); + impl asap_aware_mapping::CostModel for HeapFixtureCost { + fn rank_candidates( + &self, + _: &planner_types::pre_asap::AggIntent, + candidates: &[planner_types::post_asap::SketchAlgorithm], + ) -> Vec { + let mut ranked = candidates.to_vec(); + ranked.sort_by_key(|kind| kind != &self.0); + ranked + } + fn candidate_cost( + &self, + candidate: &asap_aware_mapping::ReplacementSubDAG, + _: &asap_aware_mapping::TargetSubDAG<'_>, + ) -> Option { + let heap = + if let asap_aware_mapping::Replacement::Summary(root) = &candidate.replacement { + planner_types::post_asap::compile_executable_dag(root) + .unwrap() + .nodes + .iter() + .any(|node| { + matches!(&node.payload, + planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), .. + } if kind.algorithm() == &self.0) + }) + } else { + false + }; + Some(asap_aware_mapping::cost_model::Cost(if heap { + 1.0 + } else { + 1e12 + })) + } + } + let model = HeapFixtureCost(algorithm.clone()); query.selected_plan_root = control_plane::planner_selection::select_query_with_models( &expr, &model, @@ -660,7 +699,7 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg &Evidence, ) .unwrap(); - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -922,7 +961,7 @@ async fn run_shared_dashboard(multi_pane: bool) { .into_iter() .enumerate() .map(|(index, candidate)| { - let plan = control_plane::physical::compiler::PhysicalPlanCompiler + let plan = control_plane::physical::compiler::DeploymentPlanCompiler .compile_promql(candidate.clone(), environment.clone()) .unwrap(); let manifest = @@ -940,7 +979,6 @@ async fn run_shared_dashboard(multi_pane: bool) { } }) .collect(); - typed.schema_version = 2; typed.workload_cost_evidence = Some( control_plane::physical::workload_cost::WorkloadCostEvidence { backend_revision: control_plane::physical::compiler::BACKEND_REVISION.into(), @@ -955,8 +993,21 @@ async fn run_shared_dashboard(multi_pane: bool) { snapshot = serde_json::to_value(&typed).unwrap(); let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); - assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!(plan.precompute_plan.materializations.len(), 2); + let families = plan + .precompute_plan + .materializations + .iter() + .map(|m| m.aggregation_type.as_str()) + .collect::>(); + assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); + assert!(plan + .precompute_plan + .executable_dags + .values() + .all(|installed| installed.document.schema_version + == asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION)); if multi_pane { assert!(plan.lifecycle_estimates[0] .window_realization_id @@ -1532,12 +1583,12 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() Some(1) ); assert_eq!( - topk_sum["data"]["result"][0]["metric"]["item"], - "asap_demo_gauge{job=\"worker\"}" + topk_sum["data"]["result"][0]["metric"]["job"], "worker", + "TopK must preserve the selected series labels: {topk_sum}" ); assert_eq!( - topk_count["data"]["result"][0]["metric"]["item"], - "asap_demo_gauge{job=\"api\"}" + topk_count["data"]["result"][0]["metric"]["job"], "api", + "TopK must preserve the selected series labels: {topk_count}" ); assert!((first_value(&sum, "value").expect("sum value") - 240.0).abs() < 1e-9); diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 29a5b1789..910025999 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -5,7 +5,9 @@ //! data plane, sends a modified-OTLP DDSketch, and verifies the resulting //! PromQL value. No server or planner is constructed in the test process. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -16,7 +18,6 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_precompute_rs::Precompute; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope as ProtoEnvelope}; use control_plane::opamp::{ opamp_proto, CollectorPlanStatus, CollectorPlanStatusKind, COLLECTOR_PLAN_CAPABILITY, @@ -76,49 +77,20 @@ fn ddsketch_export( plan: &serde_json::Value, sequence: u64, ) -> Vec { - let decoded = asap_precompute_rs::CollectorPlan::from_json( - &serde_json::to_vec(plan).unwrap(), - "whole-e2e-collector", - ) - .unwrap(); - let mut configs = decoded.to_precompute_config_set().unwrap().configs; - assert_eq!( - configs.len(), - 1, - "two query roots must create only one producer" - ); - let config = configs.remove(0); - assert_eq!(config.sketch_params["relative_accuracy"], alpha); - let runtime = asap_precompute_rs::precompute::PrecomputeImpl::new( - Some(config), - Some(Box::new(move || { - Box::new(asap_precompute_rs::sketches::DDSketchWrapper::new(alpha)) - })), - Some(Box::new(asap_precompute_rs::sketches::DDSketchObserver)), - ); + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_value(plan.clone()).unwrap(); + assert_eq!(decoded.materializations.len(), 1); + let mut sketch = asap_sketchlib::DdSketch::new(alpha); for value in values { - runtime - .observe(&asap_precompute_rs::Observation::new( - timestamp_ns / 1_000_000 - 500, - metric, - vec![], - vec![asap_precompute_rs::KeyValue::new("service", "whole-e2e")], - asap_precompute_rs::ObservationValue { - kind: asap_precompute_rs::ObservationValueKind::Float, - float: *value, - ..Default::default() - }, - )) - .unwrap(); + sketch.update(*value); } - let envelopes = runtime.tick(timestamp_ns / 1_000_000); - assert_eq!(runtime.stats().input_observations, values.len() as u64); - assert_eq!(envelopes.len(), 1); - assert_eq!(envelopes[0].count, values.len() as u64); - let wire = ProtoEnvelope::decode(envelopes[0].payload.as_slice()).unwrap(); - let Some(sketch_envelope::SketchState::Ddsketch(state)) = wire.sketch_state else { - panic!("expected actual Collector DDSketch state") - }; + assert_eq!(sketch.total_count(), values.len() as u64); + let sketch_bytes = asap_sketch_codec::encode_ddsketch(&sketch); + let wire = ProtoEnvelope::decode(sketch_bytes.as_slice()).unwrap(); + assert!(matches!( + wire.sketch_state, + Some(sketch_envelope::SketchState::Ddsketch(_)) + )); let materialization = plan["materializations"][0]["materialization"] .as_u64() .unwrap(); @@ -166,7 +138,7 @@ fn ddsketch_export( attributes, start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: sketch_bytes, encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, @@ -297,12 +269,9 @@ async fn respond_next_collector_plan( .expect("collector-plan custom message"); assert_eq!(custom.capability, COLLECTOR_PLAN_CAPABILITY); assert_eq!(custom.r#type, COLLECTOR_PLAN_MESSAGE); - let decoded = asap_precompute_rs::collector_plan::CollectorPlan::from_json( - &custom.data, - "whole-e2e-collector", - ) - .expect("actual Collector validator accepts the emitted plan"); - assert_eq!(decoded.to_precompute_config_set().unwrap().configs.len(), 1); + let decoded: asap_types::producer_plan::CollectorPlan = + serde_json::from_slice(&custom.data).expect("decode backend CollectorPlan"); + assert_eq!(decoded.materializations.len(), 1); let plan: serde_json::Value = serde_json::from_slice(&custom.data).expect("decode collector physical plan"); let plan_id = plan["envelope"]["plan_id"] @@ -395,7 +364,6 @@ async fn quote_workload( } #[tokio::test] -#[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] async fn production_control_plane_to_data_plane_otlp_to_promql() { let control_binary = std::env::var("ASAP_E2E_CONTROL_PLANE_BIN") .expect("ASAP_E2E_CONTROL_PLANE_BIN is set by scripts/e2e.sh whole"); @@ -408,7 +376,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut bootstrap = tempfile::NamedTempFile::new().expect("create bootstrap config"); - writeln!(bootstrap, "aggregations: []").expect("write bootstrap config"); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let data_child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--streaming-config") @@ -510,6 +478,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { } } }], + "dataset_identity": {"namespace": "process-e2e", "dataset": "metrics"}, "collector_ids": ["whole-e2e-collector"], "capability_snapshot_id": "whole-e2e-capabilities", "evidence": {}, @@ -521,6 +490,12 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { "backend_compat": control_plane::physical::compiler::BACKEND_COMPAT, "apply_timeout_ms": 10000 }); + let workload: serde_json::Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + request["data_workload"] = workload["data_workload"].clone(); + request["data_workload"]["data_ingestion_interval"]["value"] = 1_000.into(); let mut second = request["queries"][0].clone(); second["query_id"] = "whole-process-e2e-median".into(); second["query_string"] = "quantile_over_time(0.5, whole_process_e2e_latency_ms[1s])".into(); @@ -575,9 +550,9 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { active["aggregation_count"], 1, "physical plan was not installed: {active}" ); - let installed_aggregation = active["streaming_config"]["aggregation_configs"] - .as_object() - .and_then(|configs| configs.values().next()) + let installed_aggregation = active["streaming_config"]["precompute_plan"]["materializations"] + .as_array() + .and_then(|configs| configs.first()) .expect("installed aggregation details"); let planned_alpha = installed_aggregation["parameters"]["alpha"] .as_f64() @@ -730,9 +705,14 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!( - still_active, physical_plan_status, + still_active["plans"], physical_plan_status["plans"], "failed rollout changed active plan" ); + assert_eq!( + still_active["materializations"][0]["materialization"], + physical_plan_status["materializations"][0]["materialization"], + "failed rollout changed the installed materialization" + ); let still_warm: serde_json::Value = client .get(format!("{data_base}/api/v1/query")) .query(&[ @@ -746,8 +726,9 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!(first_scalar(&still_warm), Some(value)); - // Retry the staged successor while queries are in flight. Each - // request must retain a complete active snapshot through cutover. + // Retry while queries are in flight. Old-generation reads may finish, + // but the successor must stay cold until its own output is published. + // Reusing the old payload would violate StoredOutputReference identity. request["activation_unix_ms"] = (std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() @@ -770,11 +751,12 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .json() .await .unwrap(); - assert_eq!( - first_scalar(&response), - Some(value), - "torn serving snapshot: {response}" - ); + if let Some(actual) = first_scalar(&response) { + assert_eq!(actual, value, "incorrect old-generation result: {response}"); + } else { + assert_eq!(response["status"], "error", "{response}"); + assert_eq!(response["error"], "No result for query", "{response}"); + } tokio::time::sleep(Duration::from_millis(10)).await; } }); @@ -797,6 +779,27 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { assert_eq!(activated["plan_version"], 2); let (successor, _collector_socket) = collector.await.unwrap(); readers.await.unwrap(); + let cold_successor: serde_json::Value = client + .get(format!("{data_base}/api/v1/query")) + .query(&[ + ("query", query.to_string()), + ("time", (window_end_ms as f64 / 1000.0).to_string()), + ]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(cold_successor["status"], "error", "{cold_successor}"); + assert_eq!( + cold_successor["error"], "No result for query", + "{cold_successor}" + ); + assert!( + first_scalar(&cold_successor).is_none(), + "successor reused old state" + ); let old_frame = client .post(format!("http://{otlp_http}/v1/metrics")) .header("content-type", "application/x-protobuf") diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 64ad1e5e5..471b71530 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -2,9 +2,11 @@ //! //! Set `CLICKHOUSE_URL` (for example `http://127.0.0.1:8123`) to run it. +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::{ collections::HashMap, - io::Write, net::TcpListener, process::{Child, Command, Stdio}, sync::Arc, @@ -345,7 +347,7 @@ async fn run_mixed_aggregate(aggregate: &str) { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -540,7 +542,7 @@ async fn moving_windows_match_clickhouse_after_automatic_publication() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -724,7 +726,7 @@ async fn collection_sql_executes_local_elements_after_typed_exact_leaf() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index d35f6cae4..8f0d89dc6 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -7,7 +7,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -18,7 +17,7 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::DdSketchState; +use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; struct ChildGuard(Child); @@ -53,7 +52,12 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec) -> Vec }], start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec(), encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, @@ -120,30 +124,29 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: component_process_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), + let materialization = asap_types::PrecomputeMaterialization::new( + asap_types::AggregationType::DDSketch, + String::new(), + std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(0.01))]), + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::WindowKind::Tumbling, + String::new(), + "component_process_e2e_latency_ms".into(), + None, + None, + None, + ); + let install = physical_fixture::artifact_from_materializations(vec![materialization]); + serde_yaml::to_writer( + &mut config, + &serde_json::json!({"precompute_plan": install.precompute_plan}), ) .unwrap(); - let install = physical_fixture::artifact(&runtime); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); diff --git a/data_plane/tests/disable_query_forwarding_process_e2e.rs b/data_plane/tests/disable_query_forwarding_process_e2e.rs index 7c8c65855..529134fc9 100644 --- a/data_plane/tests/disable_query_forwarding_process_e2e.rs +++ b/data_plane/tests/disable_query_forwarding_process_e2e.rs @@ -1,6 +1,8 @@ //! The production CLI's no-forwarding mode keeps query traffic inside the backend. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::sync::{ @@ -61,7 +63,7 @@ async fn cli_mode_blocks_instant_and_range_forwarding() { }); let mut config = tempfile::NamedTempFile::new().unwrap(); - writeln!(config, "aggregations: []").unwrap(); + serde_yaml::to_writer(&mut config, &streaming_config::empty()).unwrap(); let output = tempfile::tempdir().unwrap(); let port = unused_port(); let mut child = ChildGuard( diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 84e7af7ea..8c078ec49 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -10,15 +10,14 @@ //! //! The control plane drives the plan: a PromQL query and an accuracy target //! go through `BackendLocalPlanningInput::planning_request` → -//! `PhysicalPlanCompiler::compile`, and the resulting materializations are +//! `DeploymentPlanCompiler::compile`, and the resulting materializations are //! projected into a physical-plan artifact with QueryPlan/SummaryCatalog //! bindings, then staged and activated before ingest. //! -//! Planner owns the summary choice. These tests declare an accuracy target and -//! build their payloads from whichever family and parameters it committed to — -//! `materializations[0].aggregation_type` and `.parameters` — rather than -//! pinning a family. Family selection itself is covered by the control-plane -//! compiler tests. +//! Quantile fixtures use Planner-selected materializations. CMS wire fixtures +//! explicitly declare the imported payload family; they do not assert that a +//! total-count query selects CMS. HLL and CountSketch production oracle tests +//! live in `all_sketches_process_oracle_e2e`. //! //! Queries are registered with the grouping the producer's attribute set //! carries (`sum by (service) (...)`), because the population key the backend @@ -37,7 +36,7 @@ //! the GET endpoint reflects the registered aggregation. //! * Test 2 — same shape with `group_by_labels: ["zone"]`; verifies //! #245's grouping plumb survives the round-trip into the backend's -//! `AggregationConfig.grouping_labels`. +//! `PrecomputeMaterialization.grouping_labels`. //! * Test 3 — full controller-to-query roundtrip: harness simulates //! the agent (builds DDSketch state with `asap_sketchlib`, encodes //! as a modified-OTLP `DdSketchDataPoint`), POSTs sketches to the @@ -45,7 +44,7 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use asap_types::AggregationConfig; +use asap_types::PrecomputeMaterialization; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -76,7 +75,7 @@ fn phase_aligned_now_ns() -> u64 { async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[AggregationConfig], + materializations: &[PrecomputeMaterialization], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the @@ -157,7 +156,7 @@ use asap_otel_proto::tonic::metrics::v1::{ Metric, ResourceMetrics, ScopeMetrics, }; use asap_sketchlib::proto::sketchlib::{ - CountMinState, CountSketchState, CounterType, DdSketchState, + sketch_envelope, CountMinState, CountSketchState, CounterType, DdSketchState, SketchEnvelope, }; use prost::Message; @@ -171,8 +170,8 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( "../../docs/examples/asapquery-compatibility-demo-snapshot.json" @@ -202,12 +201,37 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec Vec { + vec![PrecomputeMaterialization::new( + asap_types::AggregationType::CountMinSketch, + String::new(), + std::collections::HashMap::from([ + ("w".into(), serde_json::json!(512)), + ("d".into(), serde_json::json!(5)), + ]), + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 5, + 5, + asap_types::enums::WindowKind::Tumbling, + String::new(), + metric.into(), + Some(12), + None, + None, + )] +} + /// Epsilon-delta accuracy target in the shape `QueryRequirements` expects. fn epsilon_delta(epsilon: f64, delta: f64) -> JsonValue { serde_json::json!({ "explicit": { "EpsilonDelta": { "epsilon": epsilon, "delta": delta } } }) @@ -349,6 +373,15 @@ fn build_dd_sketch_state(alpha: f64, store_counts: Vec, store_offset: i32) } } +fn encode_dd_full_state(state: DdSketchState) -> Vec { + SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec() +} + /// Build an OTLP `ExportMetricsServiceRequest` wrapping a single DDSketch /// data point. fn build_dd_sketch_export( @@ -632,7 +665,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { // Verifies #245's grouping plumb survives the controller → backend // round-trip. The workload carries `group_by_labels: ["zone"]`; the // emitted JSON must surface `["zone"]` in `labels.grouping`, the -// backend's parser must materialise it into `AggregationConfig. +// backend's parser must materialise it into `PrecomputeMaterialization. // grouping_labels`, and the active-config snapshot must reflect that. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -745,7 +778,7 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); // ── 3. POST the sketch DP via OTLP HTTP ──────────────────────────── // @@ -785,7 +818,7 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -869,15 +902,13 @@ async fn controller_plan_to_query_full_roundtrip_kll() { "sum by (service) (quantile_over_time(0.5, request_size_bytes[1s]))", epsilon_delta(0.05, 0.05), ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. post_full_config(&client, &stack, &materializations).await; let alpha = materializations[0].parameters["alpha"] .as_f64() .expect("planner sized a relative-accuracy quantile summary"); let dd_state = build_dd_sketch_state(alpha, vec![5u64, 10, 15, 20], -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -897,7 +928,7 @@ async fn controller_plan_to_query_full_roundtrip_kll() { "request_size_bytes", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -927,42 +958,16 @@ async fn controller_plan_to_query_full_roundtrip_kll() { ); } -// ── Test 5 — full roundtrip with HLL (cardinality) ────────────────────────── -// -// HLL backs the cardinality readout. The workload pins HLL via -// `sketch_type_override: Some(SketchType::HLL)`. The OTLP DP carries -// a `HllSketchDataPoint` with `HyperLogLogState`. PromQL's -// `count(metric)` is the spec's distinct-counting idiom — returns -// the number of distinct label sets in the result vector — which -// the analyzer routes to `Capability::CardinalityApprox` and the -// reducer dispatches to the HLL cardinality readout. -// -// Closed by a chain of fixes: -// * `count(metric)` analyzer fix (PR #255) -// * `count` reducer alias (PR #255) -// * Vector-vs-Matrix instant-query response shape fix (this PR) +// An imported CMS state remains readable through its installed count binding. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_hll() { +async fn imported_cms_state_serves_count_query() { let stack = start_full_stack(19_565, 19_566).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "sum by (service) (count_over_time(unique_users_per_min[1s]))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("unique_users_per_min"); post_full_config(&client, &stack, &materializations).await; - // Precision must match what the controller plans for this - // workload (`HLLDefaults` in `control_plane::types`). The - // accuracy_sla=0.05 above is > the precision_threshold (0.02), - // so the planner picks `precision_coarse = 10`. If the OTLP DP - // were sent with a different precision, the backend would - // register two separate sids for the same metric — one with - // policy_fp=UNSET (no matching policy params) — and the query - // wouldn't find the policy-tagged one. let (w, d) = extract_w_d(&materializations[0]); let cells = (w as usize) * (d as usize); let mut counts = vec![0i64; cells]; @@ -1022,42 +1027,19 @@ async fn controller_plan_to_query_full_roundtrip_hll() { assert_eq!( status, "success", - "HLL cardinality query did not succeed:\n{}", + "Imported CMS count query did not succeed:\n{}", serde_json::to_string_pretty(&response).unwrap_or_default() ); } -// ── Test 6 — wire-format roundtrip with CountSketch (frequency) ───────────── -// -// CountSketch backs FREQUENCY estimation — signed-counter matrix -// producing approximate point-frequency answers. `top_endpoint_qps` -// is the canonical TopK metric, so the planner pins -// `with_heap: true` and the controller emits `CountSketchWithHeap` -// (regardless of override). To match, the wire DP carries a -// msgpack-encoded heap envelope, but the query -// uses `count_over_time(...)` instead of `topk(...)` — the -// reducer's `decode_frequency_total` reads row-0 of the underlying -// matrix for heap-bearing variants too, so FrequencyEstimate works -// on a heap-bearing SID. -// -// **Strict-success: `count_over_time(top_endpoint_qps[1s])`** binds -// to `Capability::FrequencyEstimate(Any)`, which -// `is_satisfied_by` accepts against -// `FrequencyTopk(CountSketchWithHeap)` (heap is additional info -// layered over the matrix — the matrix is a fully valid frequency -// sketch on its own). +// CMS transport binds the configured dimensions for the endpoint metric. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_count_sketch() { +async fn imported_cms_top_endpoint_wire_roundtrip() { let stack = start_full_stack(19_567, 19_568).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(top_endpoint_qps[1s])))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("top_endpoint_qps"); post_full_config(&client, &stack, &materializations).await; // Use the planner-picked `(w, d)` so the OTLP DP's wire-level @@ -1149,16 +1131,11 @@ async fn controller_plan_to_query_full_roundtrip_count_sketch() { // matrix and returns the per-window total count. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { +async fn imported_cms_frequency_wire_roundtrip() { let stack = start_full_stack(19_569, 19_570).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(endpoint_request_freq[1s])))", - epsilon_delta(0.05, 0.05), - ); - // Planner owns the family choice; the payload below is built from what it - // committed to. Family selection is covered by the compiler tests. + let materializations = imported_cms_materializations("endpoint_request_freq"); post_full_config(&client, &stack, &materializations).await; // Use planner-picked `(w, d)` so the wire DP's `rows`/`cols` @@ -1252,7 +1229,7 @@ async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &AggregationConfig) -> (u32, u32) { +fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { let w = agg.parameters["w"] .as_u64() .expect("materialization must carry parameters.w") as u32; @@ -1283,14 +1260,11 @@ fn extract_w_d(agg: &AggregationConfig) -> (u32, u32) { // (Prometheus spec for range queries). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_plan_to_range_query_count_over_time_cms() { +async fn imported_cms_state_serves_range_query() { let stack = start_full_stack(19_575, 19_576).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "topk(3, sum by (service) (count_over_time(endpoint_request_freq[1s])))", - epsilon_delta(0.05, 0.05), - ); + let materializations = imported_cms_materializations("endpoint_request_freq"); post_full_config(&client, &stack, &materializations).await; let (w, d) = extract_w_d(&materializations[0]); @@ -1725,7 +1699,7 @@ async fn shadow_mode_does_not_change_served_ddsketch_quantile() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -1745,7 +1719,7 @@ async fn shadow_mode_does_not_change_served_ddsketch_quantile() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -1835,7 +1809,7 @@ async fn live_serve_actually_answers_ddsketch_quantile() { let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); + let sketch_bytes = encode_dd_full_state(dd_state); let now_ns = phase_aligned_now_ns(); let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); @@ -1855,7 +1829,7 @@ async fn live_serve_actually_answers_ddsketch_quantile() { "http_latency_ms", &[("service", "e2e-test")], watermark_t_ns, - watermark_state.encode_to_vec(), + encode_dd_full_state(watermark_state), alpha, ); post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; @@ -1891,34 +1865,15 @@ async fn live_serve_actually_answers_ddsketch_quantile() { ); } -// ── Test — the live serving cutover MERGES the global-merge shape ───────── -// correctly, end to end (ASAPController#163/#165) -// -// `count(hll_metric)` with NO `by (...)` and MULTIPLE distinct-service HLL -// sids used to be the ambiguous shape the design doc's "Grouping -// semantics" section described: `SummaryAgg{by: []}` couldn't tell "no -// grouping concept" from "reduce everything," so `live_serve.rs`'s -// `ambiguous_merge_risk` gate DECLINED to serve it from the new path and -// fell back to the legacy `evaluate_cardinality_global` special case. -// -// `Reduction` (ASAPController#165) resolves that: `count(...)` is a -// genuine aggregation operator, so it lowers to `Reduce([])` and -// `resolve_group_key` gives both sids the same group key -- the new path -// merges them itself. The gate is gone; this SHOULD exercise the new -// path serving the shape directly, not a fallback. -// -// The installed cardinality readout merges all bound series and windows. +// Live serving keeps the two imported CMS series available. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn live_serve_hll_global_count_merges_across_sids() { +async fn live_serve_cms_reads_independent_series() { let _live = LiveServeEnvGuard::enable(); let stack = start_full_stack(19_595, 19_596).await; let client = reqwest::Client::new(); - let materializations = plan_materializations( - "sum by (service) (count_over_time(unique_users_per_min[1s]))", - epsilon_delta(0.05, 0.05), - ); + let materializations = imported_cms_materializations("unique_users_per_min"); post_full_config(&client, &stack, &materializations).await; let (w, d) = extract_w_d(&materializations[0]); @@ -1997,7 +1952,7 @@ async fn live_serve_hll_global_count_merges_across_sids() { #[test] fn probe_queryplan() { - use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; for q in [ "sum by (service) (quantile_over_time(0.99, http_latency_ms[1s]))", "quantile_over_time(0.99, http_latency_ms[1s])", @@ -2012,7 +1967,7 @@ fn probe_queryplan() { fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); let snap: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (req, env) = snap.into_physical_compilation_request().unwrap(); - let plan = PhysicalPlanCompiler.compile_promql(req, env).unwrap(); + let plan = DeploymentPlanCompiler.compile_promql(req, env).unwrap(); eprintln!("PROBE {q}"); for (id, e) in plan.query_plan.entries.iter() { eprintln!( diff --git a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs deleted file mode 100644 index e4623e6bb..000000000 --- a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Phase 3 step 3 acceptance tests: backend ingest **consumes** -//! `asap-precompute-rs` for the shared envelope-parsing, -//! sketch-reconstruction, and merge logic. -//! -//! Each test produces an envelope via `asap-precompute-rs`'s wrappers -//! (the canonical Rust edge runtime) and routes the bytes through the -//! backend's runtime adapter (`precompute_operators::edge_runtime_adapter`). -//! Successful round-trips prove that the asap-precompute-rs `Sketch` -//! trait family is sitting in the backend's ingest path — i.e. the -//! shared logic actually runs in this repo, not just in agents. -//! -//! Sketch coverage: -//! - **DDSketch**: round-trip + structural assertions are live — DDSketch -//! is a deterministic histogram, so `snapshot → reconstruct → snapshot` -//! is byte-identical (`asap_sketchlib`#40). -//! - **KLL**: structural envelope compatibility is live. Byte identity is -//! not a supported contract because reconstruction replays retained items -//! through randomized, lossy compaction. -//! - **HLL + CountSketch + CountMinSketch**: the shared runtime adapter does -//! not support these families; their production decoders are tested at the -//! backend accumulator boundary instead. - -use asap_precompute_rs::sketches::{DDSketchWrapper, KLLWrapper}; -use asap_precompute_rs::Sketch; - -use data_plane::precompute_engine::operators::edge_runtime_adapter::{ - encode_ddsketch_envelope, reconstruct_via_runtime, snapshot_ddsketch_via_runtime, - unwrap_envelope_state, ReconstructedSketch, SketchType, -}; -use data_plane::storage_engines::types::AggregateCore; - -// --- DDSketch ----------------------------------------------------- - -/// Round-trip: an envelope produced by asap-precompute-rs's -/// `DDSketchWrapper` is reconstructed by the backend's runtime adapter -/// to a backend-shaped `DdSketch`, re-encoded through the same -/// envelope shape, and the resulting bytes match the original. -/// -/// This proves the **shared envelope wire format** flows through both -/// crates with byte parity — the dedup target. -#[test] -fn ddsketch_envelope_round_trip_through_backend_adapter() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=200 { - w.update(i as f64); - } - let original = w.snapshot().expect("DDSketchWrapper snapshot"); - assert!(!original.is_empty()); - - let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &original) - .expect("runtime adapter reconstruction"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!("expected DDSketch reconstruction"), - }; - // `count` is recovered from the bucket store now that the scalar was - // dropped (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57). - assert_eq!( - dd.total_count(), - 200, - "count preserved through runtime adapter" - ); - - let re_encoded = encode_ddsketch_envelope(&dd); - assert_eq!( - re_encoded, original, - "envelope round-trip via asap-precompute-rs runtime must be byte-identical" - ); -} - -/// Structural: an envelope produced by asap-precompute-rs's -/// `DDSketchWrapper` is unwrapped via the backend's -/// `unwrap_envelope_state` (which itself goes through asap-precompute-rs's -/// `ProtoSketchEnvelope`), and the typed inner state has the expected -/// count + alpha shape. -#[test] -fn ddsketch_envelope_structural_assertions() { - use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; - - let mut w = DDSketchWrapper::new(0.005); - w.update(1.0); - w.update(2.0); - w.update(3.0); - let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes) - .expect("unwrap") - .expect("state"); - match state { - SketchState::Ddsketch(s) => { - // `count` was dropped from `DdSketchState` - // (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57); it is - // recovered by summing the bucket store counts. - assert_eq!(s.store_counts.iter().sum::(), 3, "structural count"); - assert!( - s.alpha > 0.0 && s.alpha < 1.0, - "alpha within (0,1): got {}", - s.alpha - ); - } - other => panic!("expected DDSketch state, got {other:?}"), - } -} - -/// End-to-end: DDSketch envelope → backend `AggregateCore` (via the -/// runtime adapter), then the backend's `query_statistic` API answers a -/// quantile query on the reconstructed accumulator. This proves the -/// **adapter integration is live** — backend ingest goes through -/// asap-precompute-rs and the resulting accumulator works on the -/// query-side surface. -#[test] -fn ddsketch_envelope_ends_up_in_backend_accumulator() { - use data_plane::precompute_engine::operators::DDSketchAccumulator; - - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=100 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot"); - - let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &bytes).expect("reconstruct"); - let dd = match reconstructed { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let acc = DDSketchAccumulator { - inner: dd, - sample_p: 1.0, - }; - - let q = acc - .query_statistic( - asap_types::Statistic::Quantile, - &None, - &[("quantile".to_string(), "0.5".to_string())] - .into_iter() - .collect(), - ) - .expect("quantile query"); - assert!( - (q - 50.0).abs() / 50.0 < 0.05, - "median estimate close to 50: got {q}" - ); - let count = acc - .query_statistic(asap_types::Statistic::Count, &None, &Default::default()) - .expect("count query"); - assert_eq!(count as u64, 100); -} - -/// Snapshot a backend-side `DdSketch` *back through* -/// asap-precompute-rs's `Sketch::snapshot` and assert byte-equality -/// with the canonical envelope bytes. Closes the round-trip -/// (encode side) — proves backend can EMIT the same wire bytes as -/// asap-precompute-rs. -#[test] -fn ddsketch_backend_sketch_snapshots_to_canonical_envelope_bytes() { - let mut w = DDSketchWrapper::new(0.01); - for i in 1..=50 { - w.update(i as f64); - } - let canonical = w.snapshot().expect("snapshot"); - let dd = match reconstruct_via_runtime(SketchType::DDSketch, &canonical).unwrap() { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let via_runtime = snapshot_ddsketch_via_runtime(&dd).expect("runtime snapshot"); - assert_eq!( - via_runtime, canonical, - "snapshot via runtime adapter is byte-identical to source envelope" - ); -} - -// --- KLL ---------------------------------------------------------- - -/// Structural: KLL envelope unwraps to the expected oneof variant via -/// the shared `unwrap_envelope_state` helper. -#[test] -fn kll_envelope_structural_assertions() { - use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; - - let mut w = KLLWrapper::new(200, Some(7)); - for i in 1..=10 { - w.update(i as f64); - } - let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes) - .expect("unwrap") - .expect("state"); - match state { - SketchState::Kll(s) => { - assert_eq!(s.k, 200, "structural k"); - assert_eq!(s.items.len(), 10, "all 10 items retained"); - } - other => panic!("expected KLL state, got {other:?}"), - } -} diff --git a/data_plane/tests/edge_sketch_codec.rs b/data_plane/tests/edge_sketch_codec.rs new file mode 100644 index 000000000..4f94d9fee --- /dev/null +++ b/data_plane/tests/edge_sketch_codec.rs @@ -0,0 +1,75 @@ +//! Portable edge sketch envelopes reconstruct through the neutral codec and +//! remain readable by the backend's query accumulators. + +use asap_sketch_codec::{encode_ddsketch, reconstruct_ddsketch}; +use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; +use data_plane::storage_engines::types::AggregateCore; + +#[test] +fn ddsketch_full_envelope_round_trips_without_collector_runtime() { + let mut source = asap_sketchlib::DdSketch::new(0.01); + for value in 1..=200 { + source.update(value as f64); + } + let bytes = asap_sketch_codec::encode_ddsketch(&source); + assert!(matches!( + asap_sketch_codec::envelope_state(&bytes).unwrap(), + Some(SketchState::Ddsketch(_)) + )); + let (decoded, _) = reconstruct_ddsketch(&bytes).unwrap(); + assert_eq!(decoded.total_count(), 200); + assert_eq!(encode_ddsketch(&decoded), bytes); +} + +#[test] +fn ddsketch_bare_state_is_rejected_and_envelope_supports_query_readout() { + let mut source = asap_sketchlib::DdSketch::new(0.01); + for value in 1..=100 { + source.update(value as f64); + } + let envelope = asap_sketch_codec::encode_ddsketch(&source); + let Some(SketchState::Ddsketch(state)) = asap_sketch_codec::envelope_state(&envelope).unwrap() + else { + panic!("DDSketch state required") + }; + let bare = prost::Message::encode_to_vec(&state); + assert!(asap_sketch_codec::reconstruct_ddsketch(&bare).is_err()); + let (decoded, _) = asap_sketch_codec::reconstruct_ddsketch(&envelope).unwrap(); + let accumulator = data_plane::precompute_engine::operators::DDSketchAccumulator { + inner: decoded, + sample_p: 1.0, + }; + let median = accumulator + .query_statistic( + asap_types::Statistic::Quantile, + &None, + &[("quantile".to_string(), "0.5".to_string())] + .into_iter() + .collect(), + ) + .unwrap(); + assert!((median - 50.0).abs() / 50.0 < 0.05); +} + +#[test] +fn kll_envelope_keeps_level_layout_for_backend_readout() { + let mut source = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(200, 7); + for value in 1..=50 { + source.update(&(value as f64)); + } + let bytes = asap_sketch_codec::encode_kll(&source); + let state = asap_sketch_codec::kll_state(&bytes).unwrap(); + assert_eq!(state.k, 200); + assert_eq!(state.items.len(), 50); + let snapshot_bytes = bytes; + let accumulator = data_plane::precompute_engine::operators::DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&snapshot_bytes).unwrap(); + assert!(accumulator.get_quantile(0.5).is_finite()); +} + +#[test] +fn a_different_sketch_family_cannot_be_decoded_as_ddsketch() { + let mut kll = asap_sketchlib::sketches::kll::KLL::::init_kll_with_seed(200, 7); + kll.update(&42.0); + let bytes = asap_sketch_codec::encode_kll(&kll); + assert!(asap_sketch_codec::reconstruct_ddsketch(&bytes).is_err()); +} diff --git a/data_plane/tests/monitor_process_e2e.rs b/data_plane/tests/monitor_process_e2e.rs index bf7bedbfd..2a515ad0b 100644 --- a/data_plane/tests/monitor_process_e2e.rs +++ b/data_plane/tests/monitor_process_e2e.rs @@ -4,7 +4,9 @@ //! data-plane executable, register, report different rates, and receive //! differentiated sampling grants over bidirectional gRPC streams. -use std::io::Write; +#[path = "support/streaming_config.rs"] +mod streaming_config; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -110,11 +112,11 @@ async fn production_coordinator_differentiates_edge_sampling_grants() { let monitor_port = unused_port(); let output_dir = tempfile::tempdir().expect("create output directory"); let mut config = tempfile::NamedTempFile::new().expect("create monitor config"); - write!( - config, - "aggregations: []\nmonitors:\n - agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n" - ) - .expect("write monitor config"); + let mut startup = streaming_config::empty(); + startup["monitors"] = serde_json::json!([{ + "agg_id": 1, "key": "", "tau": 5000.0, "epsilon": 0.05, "window_ms": 60000 + }]); + serde_yaml::to_writer(&mut config, &startup).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--streaming-config") diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index 6830697b5..59fc84659 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -8,7 +8,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -19,7 +18,7 @@ use asap_otel_proto::tonic::metrics::v1::{ metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, }; -use asap_sketchlib::proto::sketchlib::DdSketchState; +use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; use prost::Message; use serde_json::Value; @@ -80,7 +79,12 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, values: &[f64]) -> Vec { }], start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, - sketch: state.encode_to_vec(), + sketch: SketchEnvelope { + format_version: 1, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + ..Default::default() + } + .encode_to_vec(), encoding: DdSketchEncoding::DdsketchEncodingProto as i32, exemplars: Vec::new(), flags: 0, @@ -182,39 +186,17 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: differential_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), - ) - .unwrap(); - let install = physical_fixture::artifact(&runtime); + let install = physical_fixture::artifact_from_materializations(vec![ddsketch_config()]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); + let bootstrap = output_dir.path().join("bootstrap.json"); + std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) .arg("--streaming-config") - .arg(config.path()) + .arg(&bootstrap) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") @@ -355,3 +337,194 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { ); } } + +fn ddsketch_config() -> asap_types::PrecomputeMaterialization { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + PrecomputeMaterialization::new( + AggregationType::DDSketch, + String::new(), + std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(ALPHA))]), + KeyByLabelNames::new(vec!["service".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 1, + 1, + WindowKind::Tumbling, + String::new(), + METRIC.into(), + None, + None, + None, + ) +} + +// A real process must range-read the bound output, never a semantic substitute. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn bound_sds_keeps_hot_and_rebuild_outputs_and_groups_isolated() { + use asap_types::query_plan::QueryPlanNode; + use asap_types::sds::StoredOutputId; + let mut hot = ddsketch_config(); + hot.stored_output_id = Some(StoredOutputId(1001)); + let mut rebuild = hot.clone(); + rebuild.stored_output_id = Some(StoredOutputId(1002)); + let mut install = physical_fixture::artifact_from_materializations(vec![hot, rebuild]); + assert_eq!(install.summary_catalog.definitions.len(), 1); + assert_eq!(install.summary_catalog.outputs.len(), 2); + for entry in install.query_plan.entries.values_mut() { + let output = if entry.canonical_query.contains("0.99") { + 1002 + } else { + 1001 + }; + for node in entry.nodes.values_mut() { + if let QueryPlanNode::ReadMaterialization { binding } = node { + binding.materialization = StoredOutputId(output); + } + } + } + install + .query_plan + .bind_catalog(&install.summary_catalog) + .unwrap(); + let query_port = unused_port(); + let otlp_http_port = unused_port(); + let directory = tempfile::tempdir().unwrap(); + let plan = directory.path().join("plan.json"); + std::fs::write(&plan, serde_json::to_vec(&install).unwrap()).unwrap(); + let bootstrap = directory.path().join("bootstrap.json"); + std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); + let mut child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--physical-plan") + .arg(&plan) + .arg("--streaming-config") + .arg(&bootstrap) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(directory.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(otlp_http_port.to_string()) + .arg("--otel-grpc-port") + .arg(unused_port().to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{query_port}"); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut child.0).await; + let end = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; + let send = |output: u64, second: u64, value: f64, service: &'static str| { + let mut stamp_plan = install.clone(); + stamp_plan + .precompute_plan + .schemas + .retain(|s| s.materialization == StoredOutputId(output)); + let client = client.clone(); + async move { + let bytes = ddsketch_export(METRIC, second * 1_000_000_000, &[value; 32]); + let mut message = ExportMetricsServiceRequest::decode(bytes.as_slice()).unwrap(); + let Some(Data::Ddsketch(sketch)) = + &mut message.resource_metrics[0].scope_metrics[0].metrics[0].data + else { + panic!("fixture") + }; + sketch.data_points[0].attributes[0].value = Some(AnyValue { + value: Some(any_value::Value::StringValue(service.into())), + }); + physical_fixture::stamp(&mut message, &stamp_plan); + let response = client + .post(format!("http://127.0.0.1:{otlp_http_port}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(message.encode_to_vec()) + .send() + .await + .unwrap(); + let status = response.status(); + assert!( + status.is_success(), + "output {output}: {status}: {}", + response.text().await.unwrap() + ); + } + }; + let query = |quantile: &str, range: &str| { + let expression = format!("quantile_over_time({quantile}, {METRIC}[{range}])"); + let client = client.clone(); + let base = base.clone(); + async move { + get_json( + &client, + &format!("{base}/api/v1/query"), + &[("query", expression), ("time", end.to_string())], + ) + .await + } + }; + for second in end - 2..=end { + send(1002, second, 100.0, SERVICE).await; + } + let mut rebuilt = Value::Null; + for _ in 0..50 { + rebuilt = query("0.99", "3s").await; + if first_instant(&rebuilt).is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_approx( + 100.0, + first_instant(&rebuilt).expect("bound rebuild result").2, + "rebuild", + ); + assert!( + first_instant(&query("0.5", "3s").await).is_none(), + "hot must not substitute rebuild state" + ); + send(1001, end - 2, 1.0, SERVICE).await; + send(1001, end, 1.0, SERVICE).await; + assert!( + first_instant(&query("0.5", "3s").await).is_none(), + "missing middle pane must not be an empty input" + ); + send(1001, end - 1, 1.0, SERVICE).await; + for second in end - 2..=end { + send(1001, second, 200.0, "payments").await; + } + let mut answer = Value::Null; + for _ in 0..50 { + answer = query("0.5", "3s").await; + if answer["data"]["result"] + .as_array() + .is_some_and(|r| r.len() == 2) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let rows = answer["data"]["result"] + .as_array() + .unwrap_or_else(|| panic!("{answer}")); + assert_eq!(rows.len(), 2, "{answer}"); + for row in rows { + let expected = if row["metric"]["service"] == SERVICE { + 1.0 + } else { + assert_eq!(row["metric"]["service"], "payments"); + 200.0 + }; + assert_approx( + expected, + row["value"][1].as_str().unwrap().parse().unwrap(), + "isolated group", + ); + } +} diff --git a/data_plane/tests/support/current_series_process.rs b/data_plane/tests/support/current_series_process.rs index b0dabba0e..4f0dc6992 100644 --- a/data_plane/tests/support/current_series_process.rs +++ b/data_plane/tests/support/current_series_process.rs @@ -1,7 +1,7 @@ use super::*; use control_plane::physical::{ compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; @@ -27,7 +27,6 @@ async fn current_series_quantiles_topk_share_and_replace_values() { "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - snapshot.schema_version = 2; // Exercise a short declared horizon; native differential mode keeps its five-minute contract. let horizon_ms: i64 = if native.is_some() { 300_000 } else { 5_000 }; let scrape_ms = horizon_ms / 5; @@ -68,7 +67,7 @@ async fn current_series_quantiles_topk_share_and_replace_values() { let quotes = candidates .into_iter() .filter_map(|candidate| { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate.clone(), env.clone()) .ok()?; let warm = candidate.queries.iter().all(|query| { diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index b79aad913..909752886 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -18,7 +18,9 @@ async fn persisted_summary_restarts_without_live_reregistration() { serde_json::json!(5000); let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); + let plan = quote_snapshot_for_test(snapshot.clone()) + .compile_promql() + .unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -134,7 +136,10 @@ async fn persisted_summary_restarts_without_live_reregistration() { .as_object() .unwrap() .values() - .all(|binding| !binding["summary_definition_id"].is_null() + .all(|binding| !binding["stored_output_id"].is_null() + && binding["summary_definition_id"] + .as_str() + .is_some_and(|id| id.starts_with("sds-v1:")) && !binding["catalog_generation_sha256"].is_null())); drop(first); let port = unused_port(); @@ -154,4 +159,72 @@ async fn persisted_summary_restarts_without_live_reregistration() { assert!(is_warm(&after), "{after}"); assert_eq!(after["data"]["result"], before["data"]["result"]); assert_eq!(after["data"]["result"][0]["value"][1], "15"); + drop(second); + + // Equal semantics do not grant a new plan version access to old disk state. + let mut next_snapshot = snapshot; + next_snapshot.environment.plan_version += 1; + let next = quote_snapshot_for_test(next_snapshot) + .compile_promql() + .unwrap(); + assert_eq!( + install.summary_catalog.definitions, + next.summary_catalog.definitions + ); + let next_install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: next.summary_catalog, + collector_plans: next.collector_plans, + precompute_plan: next.precompute_plan, + transmission_plan: next.transmission_plan, + query_plan: next.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + }; + std::fs::write(&artifact, serde_json::to_vec(&next_install).unwrap()).unwrap(); + let port = unused_port(); + let base = format!("http://127.0.0.1:{port}"); + let mut third = spawn(port); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut third.0).await; + let cold: Value = client + .get(format!("{base}/api/v1/query")) + .query(&[("query", query.as_str()), ("time", "5")]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(!is_warm(&cold), "new version adopted old state: {cold}"); + assert_eq!( + remote_write( + &client, + &base, + &WriteRequest { + timeseries: vec![series( + "asap_demo_gauge", + &[ + (1000, 2.0), + (2000, 4.0), + (3000, 6.0), + (4000, 8.0), + (5000, 10.0) + ] + )], + } + ) + .await, + 204 + ); + drain_precompute(&client, &base).await; + let rebuilt: Value = client + .get(format!("{base}/api/v1/query")) + .query(&[("query", query.as_str()), ("time", "5")]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(is_warm(&rebuilt), "{rebuilt}"); + assert_eq!(rebuilt["data"]["result"][0]["value"][1], "30"); } diff --git a/data_plane/tests/support/issue_701_702_process.rs b/data_plane/tests/support/issue_701_702_process.rs index cdd251578..f2dbe6b6c 100644 --- a/data_plane/tests/support/issue_701_702_process.rs +++ b/data_plane/tests/support/issue_701_702_process.rs @@ -2,7 +2,7 @@ use super::*; use control_plane::physical::{ compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; @@ -152,14 +152,15 @@ async fn run_warm_workload(queries: Vec<(String, u64, u64)>) { let quotes = candidates .into_iter() .filter_map(|candidate| { - let plan = - match PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) { - Ok(plan) => plan, - Err(error) => { - errors.push(error.to_string()); - return None; - } - }; + let plan = match DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + { + Ok(plan) => plan, + Err(error) => { + errors.push(error.to_string()); + return None; + } + }; let warm = fully_warm(&plan); found |= warm; @@ -346,7 +347,7 @@ fn issue_701_702_uncertified_ratios_require_exact_fallback() { workload_cost::enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(!candidates.is_empty()); for candidate in candidates { - let plan = PhysicalPlanCompiler + let plan = DeploymentPlanCompiler .compile_promql(candidate, environment.clone()) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{query}"); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 403296140..609176084 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -38,6 +38,15 @@ pub fn artifact_from_materializations( // identities and bindings. for config in &mut configs { config.pane_origin_ms.get_or_insert(0); + // These fixtures import synthetic states and replace the producer's + // window layout. They do not install the original Planner DAG, so + // describe the supplied source/configuration instead of claiming its + // persisted-output closure. Derived inputs require the real DAG. + assert!( + config.derived_input.is_none(), + "use the Planner plan for derived inputs" + ); + config.semantic_fragment = None; } let catalog = control_plane::physical::summary_catalog::SummaryCatalog::from_materializations( 1, 1, &configs, @@ -45,7 +54,7 @@ pub fn artifact_from_materializations( .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); - precompute.summary_catalog = Some(catalog.reference().unwrap()); + precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, @@ -57,6 +66,7 @@ pub fn artifact_from_materializations( plan_id: 1, plan_version: 1, clickhouse_context: None, + selected_dags: Default::default(), entries: BTreeMap::new(), }; for config in &precompute.materializations { @@ -129,6 +139,10 @@ pub fn artifact_from_materializations( binding: MaterializationBinding { full_window_slide_ms: None, materialization: config.policy_fingerprint().into(), + stored_output_reference: + asap_types::sds::StoredOutputReference::for_output( + config.policy_fingerprint().into(), + ), output_grouping, item_labels: config.aggregated_labels.labels.clone(), window_ms: config.slide_interval * 1000, @@ -155,6 +169,7 @@ pub fn artifact_from_materializations( ); } } + query_plan.bind_catalog(&catalog).unwrap(); PhysicalPlanInstallRequest { summary_catalog: catalog, collector_plans: vec![], @@ -228,7 +243,7 @@ pub fn stamp( ("backend_compat", BACKEND_COMPAT.into()), ( "materialization", - schema.materialization.0.as_u64().to_string(), + schema.materialization.as_u64().to_string(), ), ("schema_id", schema.schema_id.clone()), ("producer_id", "fixture".into()), diff --git a/data_plane/tests/support/streaming_config.rs b/data_plane/tests/support/streaming_config.rs new file mode 100644 index 000000000..ccd35c12e --- /dev/null +++ b/data_plane/tests/support/streaming_config.rs @@ -0,0 +1,16 @@ +//! Current typed startup configuration for processes awaiting plan installation. +pub fn empty() -> serde_json::Value { + use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "process-fixture".into(), + }; + let plan = asap_types::precompute_plan::PrecomputePlan::build(envelope, vec![], &[]).unwrap(); + serde_json::json!({"precompute_plan": plan}) +} diff --git a/docs/design_docs/asapplanner-migration-plan.md b/docs/design_docs/asapplanner-migration-plan.md index 0c51cda9a..78b909e68 100644 --- a/docs/design_docs/asapplanner-migration-plan.md +++ b/docs/design_docs/asapplanner-migration-plan.md @@ -165,8 +165,8 @@ Ad-hoc discovery is deferred. | Implementation owner | Required change | Regression/acceptance gate | | --- | --- | --- | | Planner shared types and physical integration (#462) | Export a versioned canonical semantic description for a selected persisted output; exclude placement and temporary node IDs. | Different input expressions differ; renumbering preserves identity; state definitions exclude downstream readout parameters. | -| Backend plan/schema foundation (#749), completed with the shared semantic contract in #774 | Separate semantic definitions from deployed-output bindings; remove the requirement that stored-output ID equals definition ID; version the changed plan contract. | Same-version hot/rebuild outputs can share one definition without aliasing; tampered definitions and mismatched bindings fail installation. | -| Planner dependency integration (#774) | Consume the shared semantic export and propagate it from selected physical outputs into deployment compilation. | No backend expression normalization or synthetic semantic fingerprint from incomplete config fields. | +| Backend plan/schema and SDS identity foundation (#749) | Separate semantic definitions from deployed-output bindings; remove the requirement that stored-output ID equals definition ID; version the changed plan contract. | Same-version hot/rebuild outputs can share one definition without aliasing; tampered definitions and mismatched bindings fail installation. | +| Planner semantic integration (#749); shared execution integration (#774) | #749 consumes the shared semantic export and propagates dataset-bound definitions from selected outputs. #774 integrates the shared physical executor. | No backend expression normalization or synthetic semantic fingerprint from incomplete config fields. | | Precompute/storage integration (#763) | Persist definitions and output-scoped records; authorize writes against installed bindings and recover them consistently. | Restart retains semantic descriptions; wrong-output writes fail; replacement metadata and payload remain consistent. | | Query integration (#765) | Resolve the installed deployed output and validate definition, revision, format and coverage before invoking shared execution. | A hot-bound query never reads rebuild state; stale, missing or incompatible records take the explicit failure route. | | Acceptance PRs (#728, #742, #775) | Update fixtures and process tests for the new contract; validate individual queries and ensembles using synthetic costs. | End-to-end producer → persisted definition/record → recovery → bound read, with negative identity and coverage cases. | @@ -183,6 +183,6 @@ each stage lands; then update the APIs, persistence descriptions and test eviden in the same implementation PR. The open shared-library integration PR is #774, replacing the already merged -#770. The active order after #771 is #774 → #763 → #765 → #761 → #728 +#770. The active order after #749 is #774 → #763 → #765 → #761 → #728 → #742 → #775. Real-evidence work in #776, #777, #778 and #759 is deferred; old #770 base metadata is not part of this chain. diff --git a/docs/design_docs/continuous-summary-completeness.md b/docs/design_docs/continuous-summary-completeness.md index 34549b8df..040378591 100644 --- a/docs/design_docs/continuous-summary-completeness.md +++ b/docs/design_docs/continuous-summary-completeness.md @@ -10,6 +10,6 @@ Finite drain closes input, waits for all workers and certifies that every accept Admission metadata has bounded coordinate, pending-revision and byte budgets. Completed receipts expire with configured materialization retention; pending work and the published prefixes of pending admissions remain protected. Already admitted slow-worker outputs can finish behind another worker's maintenance replay frontier. Unsolicited expired input and expired untagged replay remain rejected. -This inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and version-3 persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and leaves incompatible or legacy records unbound under an authoritative catalog. Reuse across changed catalog generations requires a separate explicit compatibility decision. General multi-input maintenance transforms and durable producer watermarks remain separate work. +This inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and current-version persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and rejects unsupported persisted metadata formats. A changed catalog generation starts cold and requires fresh state; cross-generation adoption is not supported. General multi-input maintenance transforms and durable producer watermarks remain separate work. Durable SID bindings also preserve retirement and expiry timestamps and a removal tombstone. Lifecycle changes publish through the same serialized metadata writer as the flusher before changing in-memory visibility. A stale flush snapshot cannot clear those fields. Recovery leaves removed and expired instances unregistered and preserves a still-retired instance's expiry deadline. Tombstones remain until durable state is explicitly reclaimed; this change does not claim automatic tombstone garbage collection or cross-generation reactivation. diff --git a/docs/developer_docs/control-plane/architecture-naming-review.zh.md b/docs/developer_docs/control-plane/architecture-naming-review.zh.md index ef6ae4c64..5cedf4e26 100644 --- a/docs/developer_docs/control-plane/architecture-naming-review.zh.md +++ b/docs/developer_docs/control-plane/architecture-naming-review.zh.md @@ -37,7 +37,7 @@ | 所属边界 / 源码 | 原名称 → 最终名称 | |---|---| -| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`；`BackendLocalImplementation` → `BackendLocalPhysicalInputs`；`PlanningQuery` → `QueryCompilationInput`；`PlanningRequest` → `PhysicalCompilationRequest`；`PhysicalCompiler` → `PhysicalPlanCompiler`；`PhysicalPlan` → `CompiledPhysicalPlan` | +| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`；`BackendLocalImplementation` → `BackendLocalPhysicalInputs`；`PlanningQuery` → `QueryCompilationInput`；`PlanningRequest` → `PhysicalCompilationRequest`；`PhysicalCompiler` → `DeploymentPlanCompiler`；`PhysicalPlan` → `CompiledPhysicalPlan` | | 同上：查询语义 | `post_asap` → `selected_plan_root`；`source` → `legacy_query_source`；`window_secs` → `query_lookback_seconds`；`group_by` → `group_by_labels`；`accuracy` → `accuracy_target`；`lifecycle` → `summary_lifecycle_inputs`；`runtime_policy` → `materialization_runtime_policy` | | 同上：候选与证据 | `logical_selection` → `planner_selection_trace`；`materialization_policy` → `enabled_materialization_keys`；`evidence` → `topk_membership_evidence_by_query_id`；`hybrid_execution` → `allow_mixed_summary_and_exact_execution`；`synthesized_window_queries` 删除：编译器来源标记改为每个候选的 `derived` / `cohort_only`，不接受序列化输入 | | 同上：窗口与成本 | `WindowImplementationCandidate` → `WindowRealizationCandidate`；`ImplementationCostEvidence` → `WindowRealizationCostQuote`；`LifecycleCostEvidence` → `LifecycleUnitCosts`；`LifecyclePlanningInput` → `SummaryLifecyclePlanningInputs`；`window_implementations` → `window_realization_candidates` | @@ -63,7 +63,7 @@ |---|---|---| | 外部 ASAPPlanner | 解析与语义 IR，合法 summary/exact 候选，精度推理，逻辑选择；backend 提供具体成本和能力约束 | Planner 的语义选择与 backend 的物理候选比较是不同层次，不是两个重复 planner | | `control_plane::planner_selection` | 适配 Planner 的选择调用、精度及证据；输出语义 DAG 与诊断 trace | `selection` 必须说明是 logical 还是 physical；trace 不是决定执行行为的配置 | -| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择，以及绑定具体物理实现，生成多个一致的计划投影 | `PhysicalPlanCompiler` 比 `PhysicalCompiler` 清楚；整个模块当前职责仍比单纯 lowering 更宽 | +| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择，以及绑定具体物理实现，生成多个一致的计划投影 | `DeploymentPlanCompiler` 比 `PhysicalCompiler` 清楚；整个模块当前职责仍比单纯 lowering 更宽 | | `physical::workload_cost` | 枚举工作负载级候选、编译、生成报价清单、核验报价、选择最低成本可行候选 | manifest、quote、evaluation、selection report 不应相互替代 | | `physical::erp` | Error–Resource Profile 的部署适配、分布匹配、经验参数与资源估计 | `empirical_runtime_profile` 是错误展开；输入还包含策略与观测，不只一个 profile | | `control_plane::clickhouse` | SQL frontend、逻辑选择、物理绑定；支持已有 catalog 输入与自动生成 materialization 两条路径 | `ClickHouseSqlWorkload.sds` 实际是 `SummaryCatalog`；SQL 当前没有走同一套 `workload_cost::select` 整计划报价流程 | @@ -126,7 +126,7 @@ ClickHouse 的自动路径直接从 SQL 选择与绑定构建 `PhysicalPlanPubli | 当前名称 | 建议名称 / 约束 | |---|---| | `PlanningRequest` | `PhysicalCompilationRequest`；包含 workload 上下文，不是单 query | -| `PhysicalCompiler` | `PhysicalPlanCompiler` | +| `PhysicalCompiler` | `DeploymentPlanCompiler` | | `PhysicalPlan` | `CompiledPhysicalPlan`；候选和获选结果可继续复用此类型，无须新增 `SelectedPhysicalPlan` wrapper | | `logical_selection` | `planner_selection_trace`；说明只做诊断 | | `window_implementations` | `window_realization_candidates`；对象 `WindowImplementationCandidate` 也应相应命名 | diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 44f9e3ae8..7f029b8b5 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -5,8 +5,8 @@ ## Current implementation boundary -The compiler consumes ASAPPlanner types pinned to the revision exposed as -`physical::compiler::PLANNER_REVISION`, selects +The compiler consumes ASAPPlanner types from `main`, with the resolved revision +exposed as `physical::compiler::PLANNER_REVISION`, and selects from Planner's legal candidate space with backend-owned cost and evidence inputs, and emits one `CompiledPhysicalPlan`. The plan contains one SummaryCatalog plus CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan projections @@ -32,10 +32,10 @@ PhysicalCompilationRequest + DataWorkload + concrete implementation evidence | ^ | abstract candidates | complete physical costs v | -ASAPPlanner selection <---------- PhysicalPlanCompiler +ASAPPlanner selection <---------- DeploymentPlanCompiler | v -PhysicalPlanCompiler -------> CompiledPhysicalPlan +DeploymentPlanCompiler -------> CompiledPhysicalPlan | | | | v v v v Collector Precompute Backend Query @@ -112,7 +112,7 @@ identity. Physical identities never enter post-ASAP IR. ### Physical compiler ```rust -impl PhysicalPlanCompiler { +impl DeploymentPlanCompiler { pub fn compile_promql( &self, request: PhysicalCompilationRequest, @@ -338,7 +338,7 @@ identity. ### Add a deployment topology 1. Add a public `PhysicalDeploymentTarget` variant and its required target fields. -2. Teach `PhysicalPlanCompiler::compile_promql` how selected operators can be placed on it. +2. Teach `DeploymentPlanCompiler::compile_promql` how selected operators can be placed on it. 3. Reject plans requiring an unavailable stage/capability. 4. Verify the output contains one complete CollectorPlan for every producer and one SummaryCatalog and matching QueryPlan referencing all produced materializations. diff --git a/docs/developer_docs/control-plane/planning-terminology.md b/docs/developer_docs/control-plane/planning-terminology.md index 3a4dbcd97..b2076814b 100644 --- a/docs/developer_docs/control-plane/planning-terminology.md +++ b/docs/developer_docs/control-plane/planning-terminology.md @@ -23,7 +23,7 @@ flowchart TB LOGICAL["ASAPPlanner + selection adapter
Legal semantic DAG selection"] REQUEST["PhysicalCompilationRequest
QueryCompilationInput + enabled materialization keys"] WINDOWS["Generate window candidates
Cadence + evaluation phase + WindowCostModel"] - COMPILE["PhysicalPlanCompiler
Compile concrete candidate plans"] + COMPILE["DeploymentPlanCompiler
Compile concrete candidate plans"] MANIFEST["WorkloadCostManifest
Component implementations and pricing basis"] QUOTE["WorkloadQuote
Provider feasibility and component prices"] EVALUATE["CandidatePlanEvaluation
Select the lowest-cost feasible enumerated candidate"] diff --git a/docs/developer_docs/planning/repeated-dashboard-panes.md b/docs/developer_docs/planning/repeated-dashboard-panes.md index 119c2014d..36827f25b 100644 --- a/docs/developer_docs/planning/repeated-dashboard-panes.md +++ b/docs/developer_docs/planning/repeated-dashboard-panes.md @@ -23,7 +23,7 @@ pane width, without bypassing capability checks. Serialization does not confer compiler provenance: incoming quotes are always measured/provider evidence. Unquoted layouts use supplied lifecycle unit costs multiplied by structural counts. Layout changes never inherit another layout's measured scalar cost. -Workload-versus-exact deployment evidence is still required by snapshot version 2. +Workload-versus-exact deployment evidence is still required by snapshot version 3. Temporal requirements are derived from PromQL. A range selector supplies its own readout window; each rangeless source uses required diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 5a8b044ea..890ae627f 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -4,14 +4,51 @@ removed. The backend now stages and atomically activates one physical-plan envelope containing the authoritative `SummaryCatalog` snapshot plus the `PrecomputePlan`, optional `CollectorPlan`, `TransmissionPlan`, and `QueryPlan` -that reference catalog materialization IDs. - -The target semantic-definition and deployed-output separation is described in -[SDS architecture](../../design_docs/summary-catalog-sds-architecture.md). It is a -design contract, not evidence that the current catalog/storage path already -implements canonical semantic fingerprints or independent stored-output IDs. -The [migration gates](../../design_docs/asapplanner-migration-plan.md#5-bound-query-sds-implementation-across-the-pr-stack) -track the required code and recovery changes. +that reference installed stored outputs and their semantic definitions. + +Catalog schema 6 separates `StoredOutputId` (deployment routing) from +`SummaryDefinitionId` (versioned semantic SHA-256). `StoredOutputReference` +binds both. Planner exports the persisted output's typed semantic dependency +closure; explicit raw summary configurations use a restricted typed description. +Derived outputs require the complete Planner closure at installation. + +Writes must match the installed output's operator and format. Durable metadata +records both identities and the immutable catalog snapshot. Recovery validates +that snapshot before restoring authoritative state. Only the current metadata +schema is accepted. Unsupported versions, flat-map sidecars, corrupt metadata, +obsolete identity aliases, and untyped projection encodings are rejected. There +is no automatic format migration or cross-version state-adoption path. Rebuild +unsupported persisted state using the installed plan. + +See [SDS architecture](../../design_docs/summary-catalog-sds-architecture.md) +for the contract and [migration gates](../../design_docs/asapplanner-migration-plan.md#5-bound-query-sds-implementation-across-the-pr-stack) +for the downstream acceptance requirements. Ad-hoc discovery is not implemented. The HTTP lifecycle is exposed through `/api/v1/physical-plan`, `/api/v1/physical-plan/activate`, `/api/v1/physical-plan/discard`, and `/api/v1/physical-plan/status`. + +## Foundation scope (#749) + +This branch validates that maintenance and query plans belong to one installed +catalog generation, retain the selected DAG provenance, and agree on writer/read +bindings and physical windows. It removes the Collector runtime dependency. + +#749 establishes the SDS identity contract. Planner-generated definitions contain +an explicit logical dataset/tenant identity and the canonical typed dependency +closure of the persisted output. Endpoint relocation preserves this identity; +a different dataset changes it even when the metric and expression are equal. +The installed input binding must agree with the exported dataset identity. + +Semantic definition IDs and deployed output IDs are independent. Hot and rebuild +outputs can share one definition, but an installed query reads only its selected +output. Catalog schema 6 and planning snapshot schema 3 reject older metadata +rather than inventing a missing semantic identity. Later runtime PRs consume this +contract; they do not replace its identity model. + +Recovery at this stage is limited to the same installed catalog generation. +Installing a new plan version never adopts previous-version state, even for an +unchanged definition. New input must populate that version before it can serve +accelerated results; the query follows its installed fallback/unavailability +policy while cold. Fresh writes allocate a new physical series instead of using +the previous generation's completed series, both after restart and during live +activation. Cross-version adoption metadata is rejected. diff --git a/docs/evaluation/e2e-physical-dag.md b/docs/evaluation/e2e-physical-dag.md index a9cbfe108..0bcc13a67 100644 --- a/docs/evaluation/e2e-physical-dag.md +++ b/docs/evaluation/e2e-physical-dag.md @@ -228,7 +228,7 @@ cargo build --locked -p control_plane --example compile_workload_artifact cargo build --locked -p data_plane --bin data_plane python3 tools/o11y-execution/replay.py \ --metrics /path/metrics.txt --queries /path/queries.json \ - --snapshot /path/costed-version2-snapshot.json \ + --snapshot /path/costed-version3-snapshot.json \ --compiler target/debug/examples/compile_workload_artifact \ --data-plane target/debug/data_plane \ --exact-url http://127.0.0.1:9090 --output /path/new-run diff --git a/docs/examples/asapquery-compatibility-demo-snapshot.json b/docs/examples/asapquery-compatibility-demo-snapshot.json index e381d536e..c24e976c4 100644 --- a/docs/examples/asapquery-compatibility-demo-snapshot.json +++ b/docs/examples/asapquery-compatibility-demo-snapshot.json @@ -1,5 +1,5 @@ { - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": { "language": "promql", "query_batch": null, @@ -111,6 +111,7 @@ } }, "environment": { + "dataset_identity": {"namespace": "example", "dataset": "metrics"}, "target": "backend_local_remote_write", "collector_ids": [], "capability_snapshot_id": "asapquery-compatibility-demo-v1", diff --git a/docs/examples/asapquery-planning-snapshot.json b/docs/examples/asapquery-planning-snapshot.json index 471c8ad02..736857c49 100644 --- a/docs/examples/asapquery-planning-snapshot.json +++ b/docs/examples/asapquery-planning-snapshot.json @@ -1,5 +1,5 @@ { - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": { "language": "promql", "query_batch": null, @@ -62,6 +62,7 @@ "scrape_interval_ms": 5000 }, "environment": { + "dataset_identity": {"namespace": "example", "dataset": "metrics"}, "target": "backend_local_remote_write", "collector_ids": [], "capability_snapshot_id": "asapquery-local-v1", diff --git a/docs/examples/workload-cost-evidence.md b/docs/examples/workload-cost-evidence.md index 64e60498c..fc5430d55 100644 --- a/docs/examples/workload-cost-evidence.md +++ b/docs/examples/workload-cost-evidence.md @@ -1,6 +1,6 @@ # Complete workload cost evidence -Planning snapshots use one schema, `snapshot_version: 2`. Version 1 is rejected. +Planning snapshots use one schema, `snapshot_version: 3`. Versions 1 and 2 are rejected; the environment must declare its logical dataset identity. Candidate discovery may omit `workload_cost_evidence`; compiling a deployable snapshot requires complete, valid quotes and selects by complete workload cost. There is no unquoted snapshot deployment path. The checked-in JSON examples are @@ -9,7 +9,7 @@ discovery templates, not ready-to-deploy plans. ## Workflow 1. Prepare the canonical workload, deployment capabilities and implementation - evidence as in `asapquery-planning-snapshot.json`. Set version 2. + evidence as in `asapquery-planning-snapshot.json`. Set version 3. 2. Obtain requirements without deploying: ```sh diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 96491c76b..2600999f9 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -99,7 +99,7 @@ data_plane() { CURRENT_STAGE="data-plane/edge-runtime-wire" say "data-plane: edge runtime sketch envelope -> backend accumulator" - rust_test data_plane --test edge_runtime_consumes_precompute_rs + rust_test data_plane --test edge_sketch_codec CURRENT_STAGE="data-plane/production-process" say "data-plane: production binary -> modified OTLP -> SketchStore -> PromQL" diff --git a/tools/o11y-execution/CALIBRATION.md b/tools/o11y-execution/CALIBRATION.md index b95ec262b..cefa91fec 100644 --- a/tools/o11y-execution/CALIBRATION.md +++ b/tools/o11y-execution/CALIBRATION.md @@ -13,7 +13,7 @@ compares its complete quotes and installs the selected artifact. Duplicate corpus occurrences increase their registered query frequency, so the manifest accounts for all 28 occurrences, not merely 24 unique strings. Provenance preserves input hashes and source counts. This discovery snapshot contains an **uncalibrated unit enumeration seed**, - not cost evidence; version 2 with no quotes cannot deploy it. + not cost evidence; version 3 with no quotes cannot deploy it. 2. Run `calibration_candidates DISCOVERY`. It calls the control plane and Planner, and exports every bindable candidate with its manifest and install request. Errors remain in the output. These artifacts are solely for calibration. diff --git a/tools/o11y-execution/README.md b/tools/o11y-execution/README.md index b95e487b5..46a10e662 100644 --- a/tools/o11y-execution/README.md +++ b/tools/o11y-execution/README.md @@ -15,7 +15,7 @@ No family override, candidate index, or benchmark-selected winner is accepted. Export the pinned upstream task queries, not the historical 27-query fixture. Provide generator revision, command and seed separately; a file hash cannot establish that provenance. -- A canonical **version 2** planning snapshot registering exactly the corpus's +- A canonical **version 3** planning snapshot registering exactly the corpus's unique queries and containing valid complete-workload provider cost evidence. Collect evidence using `workload_cost_manifest`; keep calibration inputs and evaluation inputs distinct. Do not reuse the demo snapshot's declared costs. diff --git a/tools/o11y-execution/calibrate.py b/tools/o11y-execution/calibrate.py index dc5e7d1a7..17ec9eada 100644 --- a/tools/o11y-execution/calibrate.py +++ b/tools/o11y-execution/calibrate.py @@ -101,10 +101,11 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() snapshot = json.loads(args.snapshot.read_text()) + if snapshot.get("snapshot_version") != 3: + raise ValueError("snapshot_version 3 with explicit dataset identity is required") evidence, audit = calibrate(json.loads(args.candidates.read_text()), json.loads(args.measurements.read_text()), "sha256:" + digest(args.metrics), snapshot["environment"]["observed_at_unix_ms"], snapshot["environment"]["max_evidence_age_ms"]) - snapshot["snapshot_version"] = 2 snapshot["workload_cost_evidence"] = evidence args.output.write_text(json.dumps(snapshot, indent=2, allow_nan=False) + "\n") args.output.with_suffix(".calibration.json").write_text(json.dumps(audit, indent=2, allow_nan=False) + "\n") diff --git a/tools/o11y-execution/calibrate_runtime.py b/tools/o11y-execution/calibrate_runtime.py index 5587dca6e..7b263db7c 100644 --- a/tools/o11y-execution/calibrate_runtime.py +++ b/tools/o11y-execution/calibrate_runtime.py @@ -212,7 +212,7 @@ def launch(name, command): query_phase = phase(folder, "query-" + qid, before, after, time.perf_counter_ns() - start) raw = folder / f"queries-{qid}.json" runner.write_json(raw, records) - validate_candidate_topk_execution(artifact, records) + validate_membership_filter_execution(artifact, records) routes = {record["execution"] for record in records} correct = all(record["comparison"]["equal"] and record["exact"]["http_status"] == 200 for record in records) row["queries"][qid] = {"cpu_ns": query_phase["cpu_ns"], "evaluations": len(records), @@ -267,20 +267,20 @@ def launch(name, command): -def _candidate_topk_inputs(nodes, root): +def _membership_filter_inputs(nodes, root): bindings, visiting, visited = set(), set(), set() def visit(node_id): node_id = str(node_id) if node_id in visiting: - raise ValueError("CandidateTopK input DAG contains a cycle") + raise ValueError("MembershipFilter input DAG contains a cycle") if node_id in visited: return if node_id not in nodes: - raise ValueError(f"CandidateTopK input DAG references missing node {node_id}") + raise ValueError(f"MembershipFilter input DAG references missing node {node_id}") visiting.add(node_id) node = nodes[node_id] if node.get("op") == "exact_fallback": - raise ValueError("CandidateTopK input contains ExactFallback") + raise ValueError("MembershipFilter input contains ExactFallback") if node.get("op") == "read_materialization": bindings.add(str(node["binding"]["materialization"])) children = [str(value) for value in node.get("inputs", [])] @@ -294,21 +294,21 @@ def visit(node_id): return bindings -def validate_candidate_topk_artifact(artifact): - """Reject CandidateTopK plans whose membership sidecar is not locally installed.""" +def validate_membership_filter_artifact(artifact): + """Reject MembershipFilter plans whose membership sidecar is not locally installed.""" request = artifact.get("install_request", {}) schemas = {str(row["materialization"]): row for row in request.get("precompute_plan", {}).get("schemas", [])} modes = set() for entry in request.get("query_plan", {}).get("entries", {}).values(): nodes = entry.get("nodes", {}) for node in nodes.values(): - if node.get("op") != "candidate_top_k": + if node.get("op") != "membership_filter": continue inputs = node.get("inputs", []) if len(inputs) != 2: - raise ValueError("CandidateTopK requires membership and exact-value inputs") - membership_bindings = _candidate_topk_inputs(nodes, inputs[0]) - value_bindings = _candidate_topk_inputs(nodes, inputs[1]) + raise ValueError("MembershipFilter requires membership and exact-value inputs") + membership_bindings = _membership_filter_inputs(nodes, inputs[0]) + value_bindings = _membership_filter_inputs(nodes, inputs[1]) heap_bindings = [] for materialization in membership_bindings: schema = schemas.get(materialization) @@ -316,7 +316,7 @@ def validate_candidate_topk_artifact(artifact): if "CmsWithHeap" in family or "CountSketchWithHeap" in family: heap_bindings.append(materialization) if not heap_bindings: - raise ValueError("CandidateTopK membership input has no installed heap materialization") + raise ValueError("MembershipFilter membership input has no installed heap materialization") value_node = nodes.get(str(inputs[1]), {}) operator = value_node.get("operator", {}) if value_node.get("op") == "logical" else {} if operator.get("kind") == "candidate_exact_subquery": @@ -339,21 +339,21 @@ def validate_candidate_topk_artifact(artifact): and any(kind in json.dumps((schemas.get(mid) or {}).get("family", {})).lower() for kind in ("counter", "rate", "increase")) for mid in value_bindings): - raise ValueError("CandidateTopK value input has no installed ExactCounter materialization") + raise ValueError("MembershipFilter value input has no installed ExactCounter materialization") return modes -def validate_candidate_topk_execution(artifact, records): - modes = validate_candidate_topk_artifact(artifact) +def validate_membership_filter_execution(artifact, records): + modes = validate_membership_filter_artifact(artifact) if not modes: return if len(modes) != 1: - raise ValueError("mixed CandidateTopK execution contracts are not calibratable together") + raise ValueError("mixed MembershipFilter execution contracts are not calibratable together") mode = next(iter(modes)) for record in records: provenance = record.get("execution_provenance", {}) if provenance.get("raw_scan_evaluations", 0) not in (0, None): - raise ValueError("CandidateTopK execution used a forbidden local raw scan") + raise ValueError("MembershipFilter execution used a forbidden local raw scan") if mode == "candidate_filtered_exact": if record.get("execution") != "hybrid" or provenance.get("detail") != "hybrid": raise ValueError("candidate-filtered TopK did not report hybrid execution") @@ -364,12 +364,12 @@ def validate_candidate_topk_execution(artifact, records): raise ValueError(f"candidate-filtered TopK has invalid provenance: {key}") else: if record.get("execution") != "warm" or provenance.get("detail") not in (None, "asap"): - raise ValueError("local CandidateTopK execution was not warm") + raise ValueError("local MembershipFilter execution was not warm") for key in ("exact_subquery_rpcs", "exact_subquery_evaluations", "exact_branch_evaluations"): if provenance.get(key, 0) != 0: - raise ValueError(f"CandidateTopK execution used exact path: {key}") + raise ValueError(f"MembershipFilter execution used exact path: {key}") if provenance.get("summary_readout_evaluations", 0) < 2: - raise ValueError("CandidateTopK execution did not read both summary branches") + raise ValueError("MembershipFilter execution did not read both summary branches") def main(): @@ -413,7 +413,7 @@ def main(): candidates = candidate_document["candidates"] for candidate in candidates: if "manifest" in candidate and "install_request" in candidate: - validate_candidate_topk_artifact(candidate) + validate_membership_filter_artifact(candidate) for index, candidate in enumerate(candidates): if "manifest" not in candidate or "install_request" not in candidate: continue diff --git a/tools/o11y-execution/discover_snapshot.py b/tools/o11y-execution/discover_snapshot.py index 9b2ec9a65..8cf15e7ad 100644 --- a/tools/o11y-execution/discover_snapshot.py +++ b/tools/o11y-execution/discover_snapshot.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Register the complete real corpus for candidate discovery, never cost selection. -Unit discovery costs are an explicit uncalibrated enumeration seed. Version 2 +Unit discovery costs are an explicit uncalibrated enumeration seed. Version 3 without quotes cannot select/deploy. Replace implementation evidence with measured calibration and re-export candidates before producing final deployment quotes. """ @@ -47,6 +47,8 @@ def main(): first_timestamp_ms = timestamp if first_timestamp_ms is None else first_timestamp_ms last_timestamp_ms = timestamp snapshot = json.loads(args.template.read_text()) + if snapshot.get("snapshot_version") != 3: + raise ValueError("snapshot_version 3 with explicit dataset identity is required") now = int(time.time() * 1000) input_span = (last_timestamp_ms - first_timestamp_ms) / 1000 horizon = args.repetitions * args.interval_ms / 1000 @@ -77,7 +79,6 @@ def evidence(value): "declared_interval_ms": interval, "evaluation_phase_ms": phase, "lookback_method": "derived by the backend compiler from PromQL and the declared scrape cadence"}) snapshot["query_workload"].update(repeating_queries=registrations, query_batch=None) - snapshot["snapshot_version"] = 2 snapshot.pop("workload_cost_evidence", None) implementation = snapshot["implementation"] implementation.pop("source_sample_interval_ms", None) diff --git a/tools/o11y-execution/test_calibrate.py b/tools/o11y-execution/test_calibrate.py index b98a77fbd..8b9eefc06 100644 --- a/tools/o11y-execution/test_calibrate.py +++ b/tools/o11y-execution/test_calibrate.py @@ -1,4 +1,9 @@ import copy +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile import unittest from calibrate import calibrate @@ -21,6 +26,36 @@ def setUp(self): "queries": {"q": {"cpu_ns": 100, "evaluations": 10, "classification": "warm", "correct": True, "raw_measurement_file": "q.json"}}}]} + def test_cli_preserves_current_snapshot_and_dataset_identity(self): + # Calibration adds quotes without downgrading or inventing dataset semantics. + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + metrics = root / "metrics.prom" + metrics.write_text("m 1\n") + measurements = copy.deepcopy(self.measurements) + measurements["data_snapshot_id"] = "sha256:" + hashlib.sha256(metrics.read_bytes()).hexdigest() + dataset = {"namespace": "test-tenant", "dataset": "metrics"} + snapshot = {"snapshot_version": 3, "environment": { + "observed_at_unix_ms": 1000, "max_evidence_age_ms": 1000, + "dataset_identity": dataset}} + for name, value in [("candidates", self.candidates), ("measurements", measurements), ("snapshot", snapshot)]: + (root / (name + ".json")).write_text(json.dumps(value)) + command = ["python3", str(Path(__file__).with_name("calibrate.py")), + "--candidates", str(root / "candidates.json"), + "--measurements", str(root / "measurements.json"), + "--metrics", str(metrics), "--snapshot", str(root / "snapshot.json"), + "--output", str(root / "output.json")] + subprocess.run(command, check=True, capture_output=True) + output = json.loads((root / "output.json").read_text()) + self.assertEqual(output["snapshot_version"], 3) + self.assertEqual(output["environment"]["dataset_identity"], dataset) + self.assertTrue(output["workload_cost_evidence"]["quotes"]) + snapshot["snapshot_version"] = 2 + (root / "snapshot.json").write_text(json.dumps(snapshot)) + result = subprocess.run(command, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("snapshot_version 3", result.stderr) + def run_provider(self): return calibrate(self.candidates, self.measurements, "d", 1000, 1000) diff --git a/tools/o11y-execution/test_calibrate_runtime.py b/tools/o11y-execution/test_calibrate_runtime.py index 74a0931dc..891829f36 100644 --- a/tools/o11y-execution/test_calibrate_runtime.py +++ b/tools/o11y-execution/test_calibrate_runtime.py @@ -1,14 +1,14 @@ -"""Fail-closed validation for calibrated CandidateTopK artifacts.""" +"""Fail-closed validation for calibrated MembershipFilter artifacts.""" import unittest -from calibrate_runtime import validate_candidate_topk_artifact, validate_candidate_topk_execution +from calibrate_runtime import validate_membership_filter_artifact, validate_membership_filter_execution -class CandidateTopKArtifactTests(unittest.TestCase): +class MembershipFilterArtifactTests(unittest.TestCase): def artifact(self, membership): return {"install_request": { "query_plan": {"entries": {"q": {"nodes": { - "0": {"op": "candidate_top_k", "inputs": [1, 3]}, + "0": {"op": "membership_filter", "inputs": [1, 3]}, "1": {"op": "summary_estimate", "input": 2}, "2": membership, "3": {"op": "exact_readout", "input": 4}, @@ -35,20 +35,20 @@ def candidate_filtered_artifact(self): def test_rejects_exact_membership_fallback(self): with self.assertRaisesRegex(ValueError, "contains ExactFallback"): - validate_candidate_topk_artifact(self.artifact({"op": "exact_fallback", "reason": "unsupported"})) + validate_membership_filter_artifact(self.artifact({"op": "exact_fallback", "reason": "unsupported"})) def test_rejects_uninstalled_heap_membership(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 9}}) with self.assertRaisesRegex(ValueError, "no installed heap"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_accepts_heap_membership_and_exact_values(self): - validate_candidate_topk_artifact( + validate_membership_filter_artifact( self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) ) - def test_ignores_plans_without_candidate_topk(self): - validate_candidate_topk_artifact({"install_request": { + def test_ignores_plans_without_membership_filter(self): + validate_membership_filter_artifact({"install_request": { "query_plan": {"entries": {"q": {"nodes": {"0": {"op": "exact_fallback"}}}}}, "precompute_plan": {"schemas": []}, }}) @@ -57,56 +57,56 @@ def test_rejects_exact_value_fallback(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) artifact["install_request"]["query_plan"]["entries"]["q"]["nodes"]["3"] = {"op": "exact_fallback"} with self.assertRaisesRegex(ValueError, "contains ExactFallback"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_rejects_missing_or_cyclic_input_nodes(self): artifact = self.artifact({"op": "summary_estimate", "input": 99}) with self.assertRaisesRegex(ValueError, "missing node 99"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) artifact = self.artifact({"op": "summary_estimate", "input": 2}) with self.assertRaisesRegex(ValueError, "contains a cycle"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_requires_two_local_summary_readouts_at_runtime(self): artifact = self.artifact({"op": "read_materialization", "binding": {"materialization": 7}}) provenance = {"summary_readout_evaluations": 2, "exact_subquery_rpcs": 0, "exact_subquery_evaluations": 0, "exact_branch_evaluations": 0} - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": provenance}]) + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": provenance}]) with self.assertRaisesRegex(ValueError, "both summary branches"): - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": { + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": { **provenance, "summary_readout_evaluations": 1}}]) with self.assertRaisesRegex(ValueError, "used exact path"): - validate_candidate_topk_execution(artifact, [{"execution": "warm", "execution_provenance": { + validate_membership_filter_execution(artifact, [{"execution": "warm", "execution_provenance": { **provenance, "exact_subquery_rpcs": 1}}]) def test_accepts_one_heap_and_candidate_filtered_external_exact(self): - validate_candidate_topk_artifact(self.candidate_filtered_artifact()) + validate_membership_filter_artifact(self.candidate_filtered_artifact()) def test_candidate_filtered_contract_rejects_local_exact_state_or_unshared_input(self): artifact = self.candidate_filtered_artifact() artifact["install_request"]["precompute_plan"]["schemas"].append( {"materialization": 8, "family": {"family": "exact", "kind": "increase"}}) with self.assertRaisesRegex(ValueError, "must not install"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) artifact = self.candidate_filtered_artifact() artifact["install_request"]["query_plan"]["entries"]["q"]["nodes"]["3"]["inputs"] = [2] with self.assertRaisesRegex(ValueError, "shared membership"): - validate_candidate_topk_artifact(artifact) + validate_membership_filter_artifact(artifact) def test_candidate_filtered_execution_requires_hybrid_one_rpc_and_one_summary_read(self): artifact = self.candidate_filtered_artifact() provenance = {"detail": "hybrid", "raw_scan_evaluations": 0, "summary_readout_evaluations": 1, "exact_subquery_rpcs": 1, "exact_subquery_evaluations": 1, "exact_branch_evaluations": 1} - validate_candidate_topk_execution( + validate_membership_filter_execution( artifact, [{"execution": "hybrid", "execution_provenance": provenance}]) for key in ("summary_readout_evaluations", "exact_subquery_rpcs", "exact_subquery_evaluations", "exact_branch_evaluations"): with self.subTest(key=key), self.assertRaisesRegex(ValueError, "invalid provenance"): - validate_candidate_topk_execution(artifact, [{"execution": "hybrid", + validate_membership_filter_execution(artifact, [{"execution": "hybrid", "execution_provenance": {**provenance, key: 0}}]) with self.assertRaisesRegex(ValueError, "hybrid execution"): - validate_candidate_topk_execution( + validate_membership_filter_execution( artifact, [{"execution": "hybrid", "execution_provenance": { **provenance, "detail": "external_exact"}}]) diff --git a/tools/o11y-execution/test_discover_snapshot.py b/tools/o11y-execution/test_discover_snapshot.py index f195eb77b..e062430d0 100644 --- a/tools/o11y-execution/test_discover_snapshot.py +++ b/tools/o11y-execution/test_discover_snapshot.py @@ -24,6 +24,9 @@ def test_historical_input_does_not_backdate_plan_activation(self): "--template", str(template), "--output", str(output), "--repetitions", "1", ], check=True) snapshot = json.loads(output.read_text()) + self.assertEqual(snapshot["snapshot_version"], 3) + self.assertEqual(snapshot["environment"]["dataset_identity"], + json.loads(template.read_text())["environment"]["dataset_identity"]) self.assertEqual(snapshot["query_workload"]["repeating_queries"][0]["demand"], {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}) # Discovery output must obey the same schema as the compiler input. self.assertIsNone(snapshot["query_workload"]["repeating_queries"][0]["time_selection"].get("lookback")) diff --git a/tools/shared-workload/ACCURACY_E2E.md b/tools/shared-workload/ACCURACY_E2E.md index 73fc5f5aa..7867a534c 100644 --- a/tools/shared-workload/ACCURACY_E2E.md +++ b/tools/shared-workload/ACCURACY_E2E.md @@ -14,8 +14,8 @@ The repository's existing backend CI is unchanged. Runtime scope is **backend-local precompute**. No ASAPCollector service is started or called: the generator sends Remote Write directly to the backend. -The `asap-precompute-rs` build dependency and offline Google mapper source happen -to live in the ASAPCollector repository; they are not an extra running collector. +The optional offline Google mapper source lives in the ASAPCollector repository; +the backend build and runtime do not depend on that repository. ## Query matrix @@ -328,7 +328,7 @@ a scheduled live-load or throughput benchmark. `planned_run.py` connects one selected corpus query to the existing production planning/replay workflow. It owns fresh, separate Prometheus baseline/fallback -stores, invokes the normal compiler on measured version-2 cost evidence, installs +stores, invokes the normal compiler on measured version-3 cost evidence, installs its selected artifact, loads data, drains finite-input materialization and probes **every evaluation window** before timed replay. A readiness probe requires a nonempty warm response, positive summary-readout count and zero exact-subquery RPCs; diff --git a/tools/shared-workload/planned_run.py b/tools/shared-workload/planned_run.py index 2f7c57cdd..5e0c8b57c 100644 --- a/tools/shared-workload/planned_run.py +++ b/tools/shared-workload/planned_run.py @@ -48,7 +48,7 @@ def prepare(data, manifest, query_id, snapshot, end_ms, repetitions): registered = snapshot["query_workload"] if registered.get("query_batch") or {q["query"] for q in registered["repeating_queries"]} != {query["promql"]}: raise ValueError("costed snapshot must register exactly the selected query") - if snapshot.get("snapshot_version") != 2 or not snapshot.get("workload_cost_evidence", {}).get("quotes"): + if snapshot.get("snapshot_version") != 3 or not snapshot.get("workload_cost_evidence", {}).get("quotes"): raise ValueError("measured complete-workload cost evidence is required; discovery/demo costs are not deployment quotes") return query, {"upstream_revision": "shared-workload:" + hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest(), "queries": [{"id": query_id, "query": query["promql"], "eval_timestamp_ms": end}]} diff --git a/tools/shared-workload/test_planned.py b/tools/shared-workload/test_planned.py index 6b5bb327c..565caab6c 100644 --- a/tools/shared-workload/test_planned.py +++ b/tools/shared-workload/test_planned.py @@ -27,7 +27,7 @@ def test_series_ordered_trace_reaches_production_replay(self): manifest = accuracy_suite.corpus("google") (root / "queries.json").write_text(json.dumps(manifest)) (root / "snapshot.json").write_text(json.dumps({ - "snapshot_version": 2, + "snapshot_version": 3, "query_workload": {"repeating_queries": [{"query": "sum_over_time(google_cluster_cpu_rate[1m])"}]}, "workload_cost_evidence": {"quotes": [{"test_placeholder": True}]}})) argv = ["planned_run.py", "--data", str(root / "data"), "--manifest", str(root / "queries.json"), @@ -73,11 +73,13 @@ def test_complete_window_and_costed_registration_are_required(self): root = Path(tmp) dataset.write(root / "data", dataset.synthetic(1, 1, 0, 120000), 2402, {"dataset": "synthetic"}) manifest = accuracy_suite.corpus("synthetic") - snapshot = {"snapshot_version": 2, "query_workload": {"repeating_queries": [{"query": "sum_over_time(fake_metric[1m])"}]}} + snapshot = {"snapshot_version": 3, "query_workload": {"repeating_queries": [{"query": "sum_over_time(fake_metric[1m])"}]}} qid = "synthetic/1m/False/temporal_sum" with self.assertRaisesRegex(ValueError, "cost evidence"): prepare(root / "data", manifest, qid, snapshot, None, 2) snapshot["workload_cost_evidence"] = {"quotes": [{"test_placeholder": True}]} + with self.assertRaisesRegex(ValueError, "cost evidence"): + prepare(root / "data", manifest, qid, {**snapshot, "snapshot_version": 2}, None, 2) _, corpus = prepare(root / "data", manifest, qid, snapshot, None, 2) self.assertEqual(corpus["queries"][0]["eval_timestamp_ms"], 120000) with self.assertRaisesRegex(ValueError, "full temporal history"): diff --git a/tools/test_shared_panes.py b/tools/test_shared_panes.py index 30b3cc73e..551b10d47 100644 --- a/tools/test_shared_panes.py +++ b/tools/test_shared_panes.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -"""Validate the unmerged Planner/backend pair without changing immutable IR pins.""" +"""Validate the unmerged Planner/backend pair against Planner main.""" import argparse import json from pathlib import Path -import re import shutil import subprocess import tempfile @@ -17,17 +16,6 @@ def main(): parser.add_argument("cargo_args", nargs=argparse.REMAINDER) args = parser.parse_args() backend = Path(__file__).resolve().parents[1] - manifest = (backend / "control_plane/Cargo.toml").read_text() - declaration = re.search(r"^planner-types(?:\.workspace)?\s*=\s*(.+)$", manifest, re.MULTILINE) - if declaration is None: - raise RuntimeError("planner-types dependency is missing") - if re.search(r"workspace\s*=\s*true", declaration.group(0)): - manifest = (backend / "Cargo.toml").read_text() - declaration = re.search(r"^planner-types\s*=\s*(.+)$", manifest, re.MULTILINE) - revision_match = re.search(r'rev\s*=\s*"([0-9a-f]+)"', declaration.group(1)) if declaration else None - if revision_match is None: - raise RuntimeError("planner-types must declare a pinned Git revision") - revision = revision_match.group(1) lock = backend / "Cargo.lock" original_lock = lock.read_bytes() with tempfile.TemporaryDirectory(prefix="asap-pane-reuse-") as temporary: @@ -39,7 +27,7 @@ def main(): old = 'asap-types = { path = "../types" }' if old not in source: raise RuntimeError("unexpected Planner dependency declaration") - cargo_toml.write_text(source.replace(old, 'asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "' + revision + '" }')) + cargo_toml.write_text(source.replace(old, 'asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", branch = "main" }')) config = root / "validation.toml" text = "" if args.sketchlib: