From 3aba47c1f92e1bd0d1e7e04188a22baf8860d3a6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 03:12:51 +0000 Subject: [PATCH 01/45] Restack PR #761 with implementation before standalone acceptance --- README.md | 17 +- control_plane/src/main.rs | 77 +- control_plane/src/physical/compiler.rs | 815 +++++++++++++++--- .../src/physical/compiler/windows.rs | 4 +- control_plane/src/physical/erp.rs | 175 +++- .../src/physical/post_asap/cost_model.rs | 251 +++++- control_plane/src/physical/workload_cost.rs | 226 ++++- .../src/physical/workload_cost/automatic.rs | 667 ++++++++++++++ control_plane/src/planner_selection.rs | 8 +- control_plane/tests/discovery_snapshot.rs | 2 +- .../support/distinct_planning_process.rs | 179 ++++ docs/design_docs/README.md | 1 + docs/design_docs/asapplanner-integration.md | 10 + .../evidence-dependent-candidates.md | 319 +++++++ docs/design_docs/shape-aware-erp-v1.md | 8 +- docs/evaluation/e2e-physical-dag.md | 18 +- docs/examples/workload-cost-evidence.md | 18 +- 17 files changed, 2629 insertions(+), 166 deletions(-) create mode 100644 control_plane/src/physical/workload_cost/automatic.rs create mode 100644 docs/design_docs/evidence-dependent-candidates.md diff --git a/README.md b/README.md index f9391cbc1..ae06b77f7 100644 --- a/README.md +++ b/README.md @@ -302,13 +302,16 @@ dot -Tsvg target/readme-evidence/selected.dot \ -o target/readme-evidence/selected.svg ``` -Candidate discovery accepts the checked-in unquoted templates. Deployment and -selected-plan inspection require `ASAPQUERY_PLANNING_SNAPSHOT` to point to a -snapshot with complete, valid workload cost evidence. Prepare that input using -the [cost evidence workflow](docs/examples/workload-cost-evidence.md). -There is one snapshot compiler: it compares complete executable alternatives, -including exact fallback. Materialization IDs are definitions, not physical SIDs. -For `--metricsql`, collect quotes for the MetricsQL frontend. +Candidate discovery and deployment accept snapshots without external workload +quotes. The backend combines applicable ERP resources or analytical estimates +with data size, query frequency and physical-plan structure, then compares +complete candidate costs, including exact fallback. The selected plan includes +the resource breakdown and assumptions in `cost_comparison`. + +Point `ASAPQUERY_PLANNING_SNAPSHOT` to your workload snapshot with current data +and capability inputs. Optional calibrated provider quotes can override automatic +costing through the [cost evidence workflow](docs/examples/workload-cost-evidence.md). +Materialization IDs are definitions, not physical SIDs. ## Prometheus runbook diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 41dd15050..303caaef3 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -203,8 +203,12 @@ struct CompileAndPublishPhysicalPlanRequest { target_collector_ids: Vec, capability_snapshot_id: String, #[serde(default)] + data_snapshot_id: Option, + #[serde(default)] evidence: HashMap, #[serde(default)] + accuracy_evidence: HashMap, + #[serde(default)] exact_composition_costs: HashMap>, #[serde(default)] @@ -588,7 +592,7 @@ fn compile_physical_plan_request( legacy_query_source: planner_types::pre_asap::Source::TimeSeries { metric: query.metric, }, - query_lookback_seconds: query.window_secs, + query_lookback_ms: query.window_secs.saturating_mul(1_000), group_by_labels: query.group_by, accuracy_target: query.accuracy, summary_lifecycle_inputs: query.lifecycle, @@ -597,16 +601,54 @@ fn compile_physical_plan_request( }); } - let planner_selection_trace = match physical::compiler::select_logical_roots_with_trace( - &mut queries, - canonical_roots.clone(), - &request.evidence, - &request.exact_composition_costs, - request.erp.as_ref(), - ) { - Ok(trace) => trace, - Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), - }; + let scoped_snapshot_id = request.data_snapshot_id.as_deref().or_else(|| { + request + .workload_cost_evidence + .as_ref() + .map(|evidence| evidence.data_snapshot_id.as_str()) + }); + if request.data_snapshot_id.as_ref().is_some_and(|id| { + request + .workload_cost_evidence + .as_ref() + .is_some_and(|evidence| evidence.data_snapshot_id != *id) + }) { + return Err(( + StatusCode::UNPROCESSABLE_ENTITY, + "accuracy evidence data snapshot differs from workload cost evidence".into(), + )); + } + for (query_id, evidence) in &request.accuracy_evidence { + let Some(query) = queries.iter().find(|query| &query.query_id == query_id) else { + return Err(( + StatusCode::UNPROCESSABLE_ENTITY, + format!("accuracy evidence names unknown query {query_id}").into(), + )); + }; + evidence + .validate( + query_id, + &query.query_string, + &request.data_workload, + scoped_snapshot_id, + now, + request.max_evidence_age_ms, + ) + .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; + } + let planner_selection_trace = + match physical::compiler::select_logical_roots_with_scoped_evidence_and_trace( + &mut queries, + canonical_roots.clone(), + &request.evidence, + &request.accuracy_evidence, + &request.exact_composition_costs, + request.erp.as_ref(), + now, + ) { + Ok(trace) => trace, + Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), + }; for (query, model) in queries.iter_mut().zip(window_models) { physical::compiler::prepare_window_implementations(query, &model, request.target, 0) @@ -685,7 +727,7 @@ fn compile_physical_plan_request( ) } }, - None => frontend.compile(compilation_request, environment), + None => physical::workload_cost::select_candidates(candidates, environment, None, frontend), }; let bundle = match compiled { Ok(bundle) => bundle, @@ -926,7 +968,7 @@ mod api_tests { "target": "backend_local_remote_write", "queries": [{ "query_id": query.query_id, "query_string": query.query_string, - "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, + "metric": metric, "window_secs": query.query_lookback_ms / 1_000, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } }], "collector_ids": [], "capability_snapshot_id": "test", @@ -938,6 +980,15 @@ mod api_tests { let (plan, collectors, _, _, _) = compile_physical_plan_request(request, false, QueryFrontend::PromQl).unwrap(); let plan = plan.unwrap(); + let report = plan + .cost_comparison + .as_ref() + .expect("HTTP must compare automatically priced candidates"); + assert_eq!(report.model_version, "backend-workload-resources-v1"); + assert!(report + .candidate_evaluations + .iter() + .any(|candidate| candidate.automatic_cost.is_some())); assert!(collectors.is_empty()); assert!(plan.collector_plans.is_empty()); assert_eq!( diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index c5e93e98a..934da0b8c 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -71,7 +71,7 @@ pub struct QueryCompilationInput { /// materialization sources are derived from each post-ASAP SummaryAgg; /// it also remains part of the cost manifest workload identity. pub legacy_query_source: Source, - pub query_lookback_seconds: u64, + pub query_lookback_ms: u64, /// Label names are deployment metadata because Planner's canonical IR /// currently carries positional column IDs at this boundary. pub group_by_labels: Vec, @@ -196,6 +196,134 @@ pub struct TopKMembershipEvidence { pub source: String, } +/// A proof for one exact Planner operand, including the enforced source +/// domain rather than an observed sample minimum or maximum. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QuantileOperandDomainEvidence { + pub operand: Value, + pub lower: f64, + pub upper: f64, + pub max_samples: u64, + pub contract: String, +} + +/// Optional accuracy facts bound to one registered query and data snapshot. +/// Missing fields stay unknown to Planner. The data workload and snapshot +/// identity prevent reusing a proof for a different source population. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ScopedAccuracyEvidence { + pub query_string: String, + pub data_snapshot_id: String, + pub data_workload: DataWorkload, + pub source: String, + pub observed_at_unix_ms: u64, + pub valid_for_ms: u64, + #[serde(default)] + pub quantile_operand_domains: Vec, + #[serde(default)] + pub hll: Option, + #[serde(default)] + pub values_non_negative: Option, + #[serde(default)] + pub input_row_count: Option, + #[serde(default)] + pub hydra_shared_grid_collision_bound: Option, + #[serde(default)] + pub hydra_shared_grid_failure_probability: Option, + #[serde(default)] + pub topk_selected_lower_bound: Option, + #[serde(default)] + pub topk_excluded_upper_bound: Option, + #[serde(default)] + pub topk_interval_failure_probability: Option, +} + +impl ScopedAccuracyEvidence { + pub fn validate( + &self, + query_id: &str, + query_string: &str, + data: &DataWorkload, + snapshot_id: Option<&str>, + now_ms: u64, + max_age_ms: u64, + ) -> Result<(), CompileError> { + let valid_scope = self.query_string == query_string + && &self.data_workload == data + && snapshot_id.is_some_and(|id| id == self.data_snapshot_id) + && !self.data_snapshot_id.trim().is_empty() + && !self.source.trim().is_empty(); + let valid_time = self.observed_at_unix_ms <= now_ms + && self.valid_for_ms > 0 + && now_ms - self.observed_at_unix_ms <= self.valid_for_ms.min(max_age_ms); + let topk_bounds = ( + self.topk_selected_lower_bound, + self.topk_excluded_upper_bound, + self.topk_interval_failure_probability, + ); + let valid_topk = match topk_bounds { + (None, None, None) => true, + (Some(lower), Some(upper), Some(failure)) => { + lower.is_finite() + && upper.is_finite() + && lower > upper + && (0.0..=1.0).contains(&failure) + } + _ => false, + }; + let valid_stats = valid_topk + && self + .hll + .as_ref() + .is_none_or(super::erp::HllConfidenceContract::valid) + && self.input_row_count != Some(0) + && self + .hydra_shared_grid_collision_bound + .is_none_or(|v| v.is_finite() && v >= 0.0) + && self + .hydra_shared_grid_failure_probability + .is_none_or(|v| (0.0..=1.0).contains(&v)) + && self + .quantile_operand_domains + .iter() + .enumerate() + .all(|(index, domain)| { + domain.lower.is_finite() + && domain.upper.is_finite() + && domain.lower <= domain.upper + && (1..=(1u64 << 53)).contains(&domain.max_samples) + && !domain.contract.trim().is_empty() + && !self.quantile_operand_domains[..index] + .iter() + .any(|earlier| earlier.operand == domain.operand) + }); + if valid_scope && valid_time && valid_stats { + return Ok(()); + } + let reason = if self.query_string != query_string { + "accuracy evidence belongs to a different query" + } else if &self.data_workload != data { + "accuracy evidence belongs to a different data workload" + } else if !snapshot_id.is_some_and(|id| id == self.data_snapshot_id) + || self.data_snapshot_id.trim().is_empty() + { + "accuracy evidence belongs to a different or unspecified data snapshot" + } else if self.source.trim().is_empty() { + "accuracy evidence has no provenance source" + } else if !valid_time { + "accuracy evidence is expired, future-dated, or has no validity window" + } else { + "accuracy evidence contains invalid or ambiguous bounds" + }; + Err(CompileError::InvalidEvidence { + query_id: query_id.into(), + reason: reason.into(), + }) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PhysicalDeploymentContext { @@ -219,7 +347,7 @@ pub enum PhysicalDeploymentTarget { } /// Startup and candidate-discovery input for backend-local planning. -/// Version 2 is the sole supported schema; deployment always requires quotes. +/// Version 2 is the sole supported schema; deployment compares complete workload costs. /// Query/data semantics use ASAPPlanner's canonical workload types directly; /// this wrapper adds only backend-owned implementation evidence and lifecycle /// identity required to choose a concrete physical realization. @@ -228,7 +356,7 @@ pub enum PhysicalDeploymentTarget { pub struct BackendLocalPlanningInput { #[serde(rename = "snapshot_version")] pub schema_version: u32, - /// May be absent during candidate discovery, never during deployment. + /// Optional complete provider override; absent evidence uses backend ERP/analytical workload costing. #[serde(default, skip_serializing_if = "Option::is_none")] pub workload_cost_evidence: Option, #[serde(deserialize_with = "deserialize_snapshot_query_workload")] @@ -266,6 +394,12 @@ pub struct BackendLocalPhysicalInputs { /// before workload selection so one query cannot borrow another's evidence. #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub topk_evidence: HashMap, + /// Data generation used by scoped accuracy certificates during candidate + /// discovery, before complete workload quotes are available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_snapshot_id: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub accuracy_evidence: HashMap, /// Measured exact/summary composition profiles keyed by registered /// PromQL. Missing rows keep the corresponding Planner site opaque. #[serde(default, skip_serializing_if = "HashMap::is_empty")] @@ -505,24 +639,75 @@ pub enum CompileError { QueryPlan(#[from] crate::query_plan::QueryPlanError), } -struct QueryEvidence<'a>(Option<&'a TopKMembershipEvidence>); +struct QueryEvidence<'a> { + topk: Option<&'a TopKMembershipEvidence>, + scoped: Option<&'a ScopedAccuracyEvidence>, + now_ms: u64, +} impl AccuracyEvidenceProvider for QueryEvidence<'_> { + fn quantile_input_domain( + &self, + operand: &QueryExpr, + ) -> Option { + let operand = serde_json::to_value(operand).ok()?; + let domain = self + .scoped? + .quantile_operand_domains + .iter() + .find(|entry| entry.operand == operand)?; + Some(asap_aware_mapping::accuracy::QuantileInputDomain { + lower: domain.lower, + upper: domain.upper, + max_samples: domain.max_samples, + contract: domain.contract.clone(), + }) + } + fn propagation_stats( &self, op: &CompositionOperator, _family: &SummaryFamilyType, _query: Option<&SketchQuery>, ) -> PropagationStats { - match (op, self.0) { - (CompositionOperator::TopKSelection, Some(e)) => PropagationStats { - topk_selected_lower_bound: Some(e.selected_lower_bound), - topk_excluded_upper_bound: Some(e.excluded_upper_bound), - topk_interval_failure_probability: Some(e.interval_failure_probability), + let mut stats = self + .scoped + .map_or_else(PropagationStats::default, |evidence| PropagationStats { + values_non_negative: evidence.values_non_negative, + input_row_count: evidence.input_row_count.or_else(|| { + evidence + .data_workload + .input_cardinality + .value_at(self.now_ms) + .copied() + }), + data_distribution: evidence + .data_workload + .distribution + .value_at(self.now_ms) + .cloned(), + hydra_shared_grid_collision_bound: evidence.hydra_shared_grid_collision_bound, + hydra_shared_grid_failure_probability: evidence + .hydra_shared_grid_failure_probability, ..Default::default() - }, - _ => PropagationStats::default(), + }); + if matches!(op, CompositionOperator::TopKSelection) { + if let Some(evidence) = self + .scoped + .filter(|e| e.topk_selected_lower_bound.is_some()) + { + stats.topk_selected_lower_bound = evidence.topk_selected_lower_bound; + stats.topk_excluded_upper_bound = evidence.topk_excluded_upper_bound; + stats.topk_interval_failure_probability = + evidence.topk_interval_failure_probability; + } else if let Some(evidence) = self.topk { + stats.topk_selected_lower_bound = Some(evidence.selected_lower_bound); + stats.topk_excluded_upper_bound = Some(evidence.excluded_upper_bound); + stats.topk_interval_failure_probability = + Some(evidence.interval_failure_probability); + } } + stats } } @@ -546,23 +731,16 @@ impl BackendLocalPlanningInput { self, frontend: QueryFrontend, ) -> Result { - let evidence = self.workload_cost_evidence.clone().ok_or_else(|| { - CompileError::Snapshot( - "deployment requires complete workload cost evidence; export candidates and price them before compiling".into(), - ) - })?; + let evidence = self.workload_cost_evidence.clone(); let (request, environment) = self.into_physical_compilation_request()?; let candidates = super::workload_cost::enumerate_exact_and_materialized_candidates(request)?; - if frontend == QueryFrontend::MetricsQl { - super::workload_cost::select_lowest_cost_metricsql_candidate( - candidates, - environment, - &evidence, - ) - } else { - super::workload_cost::select_lowest_cost_candidate(candidates, environment, &evidence) - } + super::workload_cost::select_candidates( + candidates, + environment, + evidence.as_ref(), + frontend, + ) } /// Build Planner-authorized candidates for evidence collection without publishing. @@ -582,6 +760,29 @@ impl BackendLocalPlanningInput { } let workload = self.query_workload; let mut data_workload = self.data_workload.clone(); + let scoped_snapshot_id = self + .physical_inputs + .data_snapshot_id + .as_deref() + .or_else(|| { + self.workload_cost_evidence + .as_ref() + .map(|evidence| evidence.data_snapshot_id.as_str()) + }); + if self + .physical_inputs + .data_snapshot_id + .as_ref() + .is_some_and(|id| { + self.workload_cost_evidence + .as_ref() + .is_some_and(|evidence| evidence.data_snapshot_id != *id) + }) + { + return Err(CompileError::Snapshot( + "accuracy evidence data snapshot differs from workload cost evidence".into(), + )); + } if data_workload.data_ingestion_interval.value.is_none() && self.physical_inputs.scrape_interval_ms > 0 { @@ -627,6 +828,7 @@ impl BackendLocalPlanningInput { let mut queries = Vec::with_capacity(entries.len()); let mut canonical_roots = Vec::with_capacity(entries.len()); let mut topk_evidence_by_id = HashMap::new(); + let mut scoped_evidence_by_id = HashMap::new(); for (index, (entry, parsed)) in entries.into_iter().zip(canonical_queries).enumerate() { let evaluation_interval_ms = match entry.recurrence { QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { @@ -638,14 +840,9 @@ impl BackendLocalPlanningInput { ))) } }; - if self.physical_inputs.scrape_interval_ms == 0 - || !self - .physical_inputs - .scrape_interval_ms - .is_multiple_of(1_000) - { + if self.physical_inputs.scrape_interval_ms == 0 { return Err(CompileError::Snapshot( - "scrape_interval_ms must be a positive whole number of seconds".into(), + "scrape_interval_ms must be positive".into(), )); } let accuracy = entry.requirements.accuracy.target(); @@ -673,6 +870,17 @@ impl BackendLocalPlanningInput { if let Some(evidence) = self.physical_inputs.topk_evidence.get(&query_string) { topk_evidence_by_id.insert(query_id.clone(), evidence.clone()); } + if let Some(evidence) = self.physical_inputs.accuracy_evidence.get(&query_string) { + evidence.validate( + &query_id, + &query_string, + &data_workload, + scoped_snapshot_id, + self.environment.observed_at_unix_ms, + self.environment.max_evidence_age_ms, + )?; + scoped_evidence_by_id.insert(query_id.clone(), evidence.clone()); + } queries.push(QueryCompilationInput { query_id, query_string: query_string.clone(), @@ -680,7 +888,7 @@ impl BackendLocalPlanningInput { legacy_query_source: Source::TimeSeries { metric: source_hint, }, - query_lookback_seconds: lookback_ms / 1_000, + query_lookback_ms: lookback_ms, group_by_labels: metadata.group_by_labels, accuracy_target: accuracy, summary_lifecycle_inputs: lifecycle, @@ -688,6 +896,16 @@ impl BackendLocalPlanningInput { materialization_runtime_policy: RuntimeRulePolicy::default(), }); } + if self + .physical_inputs + .accuracy_evidence + .keys() + .any(|query| !queries.iter().any(|entry| &entry.query_string == query)) + { + return Err(CompileError::Snapshot( + "accuracy evidence names a query absent from this workload".into(), + )); + } let mut exact_costs_by_id = HashMap::new(); for (index, entry) in workload.entries().enumerate() { if let Some(rows) = self @@ -709,12 +927,14 @@ impl BackendLocalPlanningInput { exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); } } - let planner_selection_trace = select_logical_roots_with_trace( + let planner_selection_trace = select_logical_roots_with_scoped_evidence_and_trace( &mut queries, canonical_roots.clone(), &topk_evidence_by_id, + &scoped_evidence_by_id, &exact_costs_by_id, self.physical_inputs.erp.as_ref(), + self.environment.observed_at_unix_ms, )?; for query in &mut queries { prepare_window_implementations( @@ -1083,7 +1303,7 @@ impl DeploymentPlanCompiler { validate_evidence(&query.query_id, e, &environment)?; } let node = query.selected_plan_root.clone(); - reject_uncertified_readouts(&query.query_id, &node)?; + reject_uncertified_readouts(&query.query_id, &node, environment.target)?; let selected = collect_selected_materializations( &node, request.allow_mixed_summary_and_exact_execution, @@ -1092,6 +1312,9 @@ impl DeploymentPlanCompiler { query_id: query.query_id.clone(), reason, })?; + if !selected.is_empty() { + reject_uncertified_readouts(&query.query_id, &node, environment.target)?; + } let selected = selected .into_iter() .filter(|state| { @@ -1210,8 +1433,10 @@ impl DeploymentPlanCompiler { let cohort_nodes = windows::cohort_nodes(&selected); for (ordinal, selected) in selected.into_iter().enumerate() { let mut branch_query = query.clone(); - branch_query.query_lookback_seconds = - selected.window_secs.unwrap_or(query.query_lookback_seconds); + branch_query.query_lookback_ms = selected + .window_secs + .map(|seconds| seconds.saturating_mul(1_000)) + .unwrap_or(query.query_lookback_ms); branch_query.group_by_labels = selected .group_by .clone() @@ -1219,7 +1444,8 @@ impl DeploymentPlanCompiler { branch_query .window_realization_candidates .retain(|candidate| { - candidate.window_secs == branch_query.query_lookback_seconds + candidate.window_secs.saturating_mul(1_000) + == branch_query.query_lookback_ms && if cohort_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { windows::is_full_cohort(candidate) } else { @@ -1306,7 +1532,7 @@ impl DeploymentPlanCompiler { let cadence = u64::from( query.summary_lifecycle_inputs.evaluation_interval_ms, ); - let window = query.query_lookback_seconds.saturating_mul(1_000); + let window = query.query_lookback_ms; a % cadence == b % cadence && a % window == b % window } _ => index == query_index, @@ -1697,7 +1923,7 @@ impl DeploymentPlanCompiler { let window_ms = materialization.window_size.saturating_mul(1_000); if materialization_family != *node_family || window_ms == 0 - || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) + || source_window.map(|seconds| seconds.saturating_mul(1_000)).unwrap_or(query.query_lookback_ms) % window_ms != 0 { return Err(crate::query_plan::QueryPlanError::Invalid(format!( @@ -1720,7 +1946,7 @@ impl DeploymentPlanCompiler { }) }; let instant = InstantExecution { - lookback_ms: query.query_lookback_seconds.saturating_mul(1_000), + lookback_ms: query.query_lookback_ms, full_history: false, cumulative_readout: true, }; @@ -2101,7 +2327,7 @@ pub fn select_logical_roots_for_queries( select_logical_roots_with_error_resource_profiles(queries, roots, evidence, exact_costs, None) } -fn observed_population_matches_root( +pub(crate) fn observed_population_matches_root( policy: &super::erp::ErpPlanningInput, root: &QueryExpr, ) -> bool { @@ -2164,6 +2390,26 @@ pub fn select_logical_roots_with_trace( evidence: &HashMap, exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, +) -> Result, CompileError> { + select_logical_roots_with_scoped_evidence_and_trace( + queries, + roots, + evidence, + &HashMap::new(), + exact_costs, + erp, + 0, + ) +} + +pub fn select_logical_roots_with_scoped_evidence_and_trace( + queries: &mut [QueryCompilationInput], + roots: Vec>, + evidence: &HashMap, + scoped_evidence: &HashMap, + exact_costs: &HashMap>, + erp: Option<&super::erp::ErpPlanningInput>, + now_ms: u64, ) -> Result, CompileError> { let mut traces = Vec::new(); if roots.len() != queries.len() { @@ -2177,6 +2423,7 @@ pub fn select_logical_roots_with_trace( for (index, root) in roots.into_iter().enumerate() { let accuracy = &queries[index].accuracy_target; let certificate_scope = (evidence.contains_key(&queries[index].query_id) + || scoped_evidence.contains_key(&queries[index].query_id) || exact_costs.contains_key(&queries[index].query_id)) .then(|| queries[index].query_id.clone()); if let Some((_, _, roots)) = cohorts @@ -2189,13 +2436,8 @@ pub fn select_logical_roots_with_trace( } } for (accuracy, scope, roots) in cohorts { - // ERP v1 has no calibrated failure probability. Preserve explicit - // confidence requirements through theoretical/exact fallback. let scoped_erp = erp.map(|policy| { let mut policy = policy.clone(); - if !matches!(accuracy, AccuracyTarget::Epsilon(_)) { - policy.artifact.records.clear(); - } if policy.observed_populations.is_some() && !roots .iter() @@ -2218,7 +2460,15 @@ pub fn select_logical_roots_with_trace( }); policy }); - let erp = scoped_erp.as_ref(); + // Confidence limitations invalidate an empirical accuracy decision, + // not the independently matched resource measurements. + let accuracy_erp = scoped_erp.clone().map(|mut policy| { + if !matches!(accuracy, AccuracyTarget::Epsilon(_)) { + policy.artifact.records.clear(); + } + policy + }); + let erp = accuracy_erp.as_ref(); let mut model = ControlPlaneCostModel::new(accuracy.clone()).with_exact_composition_costs( scope .as_ref() @@ -2229,8 +2479,17 @@ pub fn select_logical_roots_with_trace( if let Some(erp) = erp { model = model.with_erp(erp.clone()); } + if let Some(costs) = &scoped_erp { + model = model.with_erp_costs(costs.clone()); + } let certificate = scope.as_ref().and_then(|id| evidence.get(id)); + let scoped_certificate = scope.as_ref().and_then(|id| scoped_evidence.get(id)); + let hll = scoped_certificate.and_then(|evidence| evidence.hll.as_ref()); + if let Some(contract) = hll { + model = model.with_hll_confidence(contract.clone()); + } let accuracy_model = super::erp::ErpAccuracyModel { + hll, policy: erp, max_error: match accuracy { AccuracyTarget::Epsilon(e) | AccuracyTarget::EpsilonDelta { epsilon: e, .. } => e, @@ -2242,10 +2501,23 @@ pub fn select_logical_roots_with_trace( roots, accuracy, &model, - &QueryEvidence(certificate), + &QueryEvidence { + topk: certificate, + scoped: scoped_certificate, + now_ms, + }, &accuracy_model, ) .map_err(|error| CompileError::Snapshot(error.to_string()))?; + if let Some(evidence) = scoped_certificate { + trace["accuracy_evidence_scope"] = serde_json::json!({ + "query_id": scope, + "source": evidence.source, + "data_snapshot_id": evidence.data_snapshot_id, + "observed_at_unix_ms": evidence.observed_at_unix_ms, + "hll": evidence.hll, + }); + } trace["deployment_overrides"] = serde_json::json!([]); let selected_indices = selected.iter().map(|(index, _)| *index).collect::>(); for (index, node) in selected { @@ -2385,7 +2657,11 @@ pub fn select_post_asap( expr, &model, &DefaultAccuracyModel, - &QueryEvidence(evidence), + &QueryEvidence { + topk: evidence, + scoped: None, + now_ms: 0, + }, ) } @@ -2545,10 +2821,9 @@ pub(super) fn derived_window_cost( /// example, `a[1m] offset 1h` selects `(t - 61m, t - 60m]`. fn query_history_window_ms(expr: &QueryExpr, scrape_interval_ms: u64) -> Result { fn duration_ms(duration: std::time::Duration) -> Result { - if duration.subsec_nanos() != 0 { + if !duration.subsec_nanos().is_multiple_of(1_000_000) { return Err(CompileError::Snapshot( - "PromQL ranges must be a whole number of seconds in the backend-local profile" - .into(), + "PromQL ranges must have millisecond precision".into(), )); } u64::try_from(duration.as_millis()) @@ -2577,17 +2852,10 @@ fn query_history_window_ms(expr: &QueryExpr, scrape_interval_ms: u64) -> Result< duration_ms(*range)?, visit(child, scrape_interval_ms, true)?, ), - QueryExpr::TimeShift { shift, child } => { - if !shift.offset_ms.unsigned_abs().is_multiple_of(1_000) { - return Err(CompileError::Snapshot( - "PromQL offsets must be a whole number of seconds in the backend-local profile".into(), - )); - } - add( - shift.offset_ms.max(0) as u64, - visit(child, scrape_interval_ms, in_range)?, - ) - } + QueryExpr::TimeShift { shift, child } => add( + shift.offset_ms.max(0) as u64, + visit(child, scrape_interval_ms, in_range)?, + ), QueryExpr::PromqlScalarBridge(child) | QueryExpr::PromqlVectorFromScalar(child) | QueryExpr::PromqlScalarFromVector(child) @@ -2733,7 +3001,7 @@ pub(super) fn validate_window_implementations( && evidence.cpu_cost >= 0.0 && evidence.weighted_cost.is_finite() && evidence.weighted_cost >= 0.0 - && candidate.window_secs == query.query_lookback_seconds + && candidate.window_secs.saturating_mul(1_000) == query.query_lookback_ms && candidate .layout .validate(candidate.window_secs, candidate.slide_secs) @@ -2796,8 +3064,8 @@ fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) ) } -pub(super) fn estimated_state_bytes( - aggregation: &asap_types::AggregationType, +pub(crate) fn estimated_state_bytes( + aggregation_type: &asap_types::AggregationType, parameters: &HashMap, ) -> u128 { use asap_types::AggregationType as A; @@ -2808,7 +3076,7 @@ pub(super) fn estimated_state_bytes( .find_map(|name| parameters.get(*name).and_then(Value::as_u64)) .unwrap_or(fallback) as u128 }; - match aggregation { + match aggregation_type { A::CountMinSketch | A::CountSketch => { parameter(&["width", "w", "col_num", "col"], 1) * parameter(&["depth", "d", "row_num", "row"], 1) @@ -2840,7 +3108,7 @@ pub(super) fn estimated_state_bytes( } } -fn retained_partition_count( +pub(crate) fn retained_partition_count( materialization: &asap_types::PrecomputeMaterialization, input_cardinality: Option, ) -> u128 { @@ -2945,8 +3213,8 @@ fn select_lifecycle( raw_materialization_input_contract(node) .ok() .and_then(|(_, window, _)| window) - .unwrap_or(query.query_lookback_seconds) - .saturating_mul(1_000), + .map(|seconds| seconds.saturating_mul(1_000)) + .unwrap_or(query.query_lookback_ms), )), as_of: None, }, @@ -3261,7 +3529,11 @@ fn validate_executable_subdag(node: &Rc) -> Result<(), String> { Ok(()) } -fn reject_uncertified_readouts(query_id: &str, root: &Rc) -> Result<(), CompileError> { +fn reject_uncertified_readouts( + query_id: &str, + root: &Rc, + target: PhysicalDeploymentTarget, +) -> Result<(), CompileError> { let dag = planner_types::post_asap::compile_executable_dag(root).map_err(|error| { CompileError::Query { query_id: query_id.into(), @@ -3269,6 +3541,21 @@ fn reject_uncertified_readouts(query_id: &str, root: &Rc) -> Result } })?; for node in &dag.nodes { + if target != PhysicalDeploymentTarget::BackendLocalRemoteWrite + && node.guarantee.as_ref().is_some_and(|guarantee| { + guarantee.provenance.iter().any(|source| { + matches!(source, + planner_types::post_asap::GuaranteeSource::SketchReadout { contract, .. } + if contract == "classic_hll_linear_counting_collision_bound_v1") + }) + }) + { + return Err(CompileError::Query { + query_id: query_id.into(), + reason: "classic HLL confidence is bound to the backend-local Regular estimator" + .into(), + }); + } if matches!( node.payload, planner_types::post_asap::ExecutableOperatorPayload::SummaryEstimate { .. } @@ -3296,7 +3583,9 @@ fn physical_aggregation( aggregation_id, metric_name: selected.metric.clone(), family: selected.family.clone(), - window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), + window_secs: selected + .window_secs + .unwrap_or(query.query_lookback_ms.div_ceil(1_000)), spatial_filter: selected.spatial_filter.clone(), grouping: selected .group_by @@ -3691,7 +3980,7 @@ fn collect_selected_materializations( Ok(selected) } -pub(super) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { +pub(crate) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { P::UnivMon { @@ -4620,7 +4909,7 @@ pub(crate) mod tests { query_string: promql.into(), selected_plan_root: post_asap, legacy_query_source: Source::TimeSeries { metric: "m".into() }, - query_lookback_seconds: 60, + query_lookback_ms: 60_000, group_by_labels: vec![], accuracy_target: accuracy, summary_lifecycle_inputs: lifecycle, @@ -4873,6 +5162,273 @@ pub(crate) mod tests { assert!(result.unwrap().precompute_plan.materializations.is_empty()); } + fn bounded_hll_snapshot_wire() -> serde_json::Value { + let mut wire = serde_json::to_value(planning_snapshot()).unwrap(); + let query = "distinct_over_time(data[5s])"; + wire["query_workload"]["repeating_queries"][0]["query"] = query.into(); + wire["query_workload"]["repeating_queries"][0]["requirements"]["accuracy"] = + serde_json::json!({"explicit":{"EpsilonDelta":{"epsilon":0.05,"delta":0.01}}}); + wire["implementation"]["data_snapshot_id"] = "hll-bounded-population".into(); + let now = wire["environment"]["observed_at_unix_ms"].clone(); + wire["implementation"]["accuracy_evidence"] = serde_json::json!({query: { + "query_string": query, "data_snapshot_id": "hll-bounded-population", + "data_workload": wire["data_workload"], "source": "enforced-distinct-domain-v1", + "observed_at_unix_ms": now, "valid_for_ms": 60000, + "hll": {"model":"asap-classic64-uniform-hash-linear-counting-v1", + "max_distinct_per_readout": 128} + }}); + wire + } + + /// An applicable classic-HLL confidence contract restores normal selection. + #[test] + fn bounded_hll_confidence_selects_and_binds_a_materialization() { + let wire = bounded_hll_snapshot_wire(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request, environment) + .unwrap(); + assert_eq!(plan.precompute_plan.materializations.len(), 1, "{plan:#?}"); + assert_eq!( + plan.precompute_plan.materializations[0].aggregation_type, + asap_types::AggregationType::HLL + ); + } + + /// A backend-local estimator proof cannot certify an unverified collector implementation. + #[test] + fn hll_confidence_cannot_be_rebound_to_collectors() { + let snapshot: BackendLocalPlanningInput = + serde_json::from_value(bounded_hll_snapshot_wire()).unwrap(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); + let error = reject_uncertified_readouts( + "q", + &request.queries[0].selected_plan_root, + PhysicalDeploymentTarget::DistributedCollectors, + ) + .unwrap_err(); + assert!(error + .to_string() + .contains("backend-local Regular estimator")); + } + + /// Missing contracts and unattainable targets cannot acquire a confidence proof. + #[test] + fn hll_confidence_keeps_exact_when_absent_or_insufficient() { + for missing in [true, false] { + let mut wire = bounded_hll_snapshot_wire(); + if missing { + wire["implementation"]["accuracy_evidence"] = serde_json::json!({}); + } else { + wire["query_workload"]["repeating_queries"][0]["requirements"]["accuracy"] = + serde_json::json!({"explicit":{"EpsilonDelta":{"epsilon":0.05,"delta":1e-12}}}); + } + let snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request, environment) + .unwrap(); + assert!( + plan.precompute_plan.materializations.is_empty(), + "{plan:#?}" + ); + } + } + + /// An estimator mismatch or unsupported population must be rejected at admission. + #[test] + fn hll_confidence_rejects_invalid_source_contracts() { + for contract in [ + serde_json::json!({"model":"asap-classic64-uniform-hash-linear-counting-v1","max_distinct_per_readout":0}), + serde_json::json!({"model":"asap-classic64-uniform-hash-linear-counting-v1","max_distinct_per_readout":4097}), + serde_json::json!({"model":"hip-rse","max_distinct_per_readout":128}), + ] { + let mut wire = bounded_hll_snapshot_wire(); + wire["implementation"]["accuracy_evidence"]["distinct_over_time(data[5s])"]["hll"] = + contract; + let snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + assert!(snapshot.into_physical_compilation_request().is_err()); + } + } + + /// Accuracy facts must belong to the same query, workload, snapshot and + /// evidence window before they enter Planner. + #[test] + fn scoped_accuracy_evidence_rejects_wrong_or_stale_facts() { + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let evidence = ScopedAccuracyEvidence { + query_string: "quantile_over_time(0.9,m[1m])".into(), + data_snapshot_id: "snapshot-a".into(), + data_workload: snapshot.data_workload.clone(), + source: "enforced-source-contract".into(), + observed_at_unix_ms: 9_000, + valid_for_ms: 2_000, + hll: None, + quantile_operand_domains: vec![], + values_non_negative: Some(true), + input_row_count: Some(10), + hydra_shared_grid_collision_bound: None, + hydra_shared_grid_failure_probability: None, + topk_selected_lower_bound: None, + topk_excluded_upper_bound: None, + topk_interval_failure_probability: None, + }; + let validate = |evidence: &ScopedAccuracyEvidence, query: &str, snapshot_id: &str, now| { + evidence.validate( + "q", + query, + &snapshot.data_workload, + Some(snapshot_id), + now, + 2_000, + ) + }; + assert!(validate(&evidence, &evidence.query_string, "snapshot-a", 10_000).is_ok()); + assert!(validate(&evidence, "another query", "snapshot-a", 10_000).is_err()); + assert!(validate(&evidence, &evidence.query_string, "snapshot-b", 10_000).is_err()); + assert!(validate(&evidence, &evidence.query_string, "snapshot-a", 12_000).is_err()); + let mut invalid = evidence.clone(); + invalid.hydra_shared_grid_failure_probability = Some(1.5); + assert!(validate(&invalid, &invalid.query_string, "snapshot-a", 10_000).is_err()); + let mut partial_topk = evidence.clone(); + partial_topk.topk_selected_lower_bound = Some(10.0); + assert!(validate( + &partial_topk, + &partial_topk.query_string, + "snapshot-a", + 10_000 + ) + .is_err()); + let mut valid_topk = partial_topk; + valid_topk.topk_excluded_upper_bound = Some(8.0); + valid_topk.topk_interval_failure_probability = Some(0.01); + assert!(validate(&valid_topk, &valid_topk.query_string, "snapshot-a", 10_000).is_ok()); + let stats = QueryEvidence { + topk: None, + scoped: Some(&valid_topk), + now_ms: 10_000, + } + .propagation_stats( + &CompositionOperator::TopKSelection, + &SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + planner_types::post_asap::ExactParams::Count, + ), + None, + ); + assert_eq!(stats.topk_selected_lower_bound, Some(10.0)); + assert_eq!(stats.topk_excluded_upper_bound, Some(8.0)); + } + + /// Quantile domain proof applies only to the operand named by its AST, + /// even when both operands read the same metric. + #[test] + fn scoped_quantile_domain_matches_one_exact_operand() { + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let accuracy = AccuracyTarget::EpsilonDelta { + epsilon: 0.05, + delta: 0.01, + }; + let root = crate::query_parser::parse_query_expr_canonical( + "quantile_over_time(0.9,m[5m]) / quantile_over_time(0.5,m[5m])", + accuracy.clone(), + ) + .unwrap(); + let QueryExpr::BinaryOp { lhs, rhs, .. } = &root else { + panic!("expected ratio expression") + }; + let scoped = ScopedAccuracyEvidence { + query_string: "quantile_over_time(0.9,m[5m]) / quantile_over_time(0.5,m[5m])".into(), + data_snapshot_id: "snapshot-a".into(), + data_workload: snapshot.data_workload, + source: "enforced-source-contract".into(), + observed_at_unix_ms: 9_000, + valid_for_ms: 2_000, + hll: None, + quantile_operand_domains: vec![QuantileOperandDomainEvidence { + operand: serde_json::to_value(lhs.as_ref()).unwrap(), + lower: 1.0, + upper: 100.0, + max_samples: 10_000, + contract: "source-schema-v1".into(), + }], + values_non_negative: None, + input_row_count: None, + hydra_shared_grid_collision_bound: None, + hydra_shared_grid_failure_probability: None, + topk_selected_lower_bound: None, + topk_excluded_upper_bound: None, + topk_interval_failure_probability: None, + }; + let provider = QueryEvidence { + topk: None, + scoped: Some(&scoped), + now_ms: 10_000, + }; + assert!(provider.quantile_input_domain(lhs).is_some()); + assert!(provider.quantile_input_domain(rhs).is_none()); + let inspect = |provider: &dyn AccuracyEvidenceProvider| { + let (_, trace) = + crate::planner_selection::select_workload_with_accuracy_model_and_trace( + vec![(0, Rc::new(root.clone()))], + accuracy.clone(), + &ControlPlaneCostModel::new(accuracy.clone()), + provider, + &DefaultAccuracyModel, + ) + .unwrap(); + trace + }; + let unknown = inspect(&provider); + assert!(unknown["groups"].as_array().unwrap().iter().any(|group| { + group["candidates"] + .as_array() + .unwrap() + .iter() + .any(|candidate| { + candidate["accuracy_status"] == "unknown" && candidate["selected"] == false + }) + })); + let mut complete = scoped.clone(); + complete + .quantile_operand_domains + .push(QuantileOperandDomainEvidence { + operand: serde_json::to_value(rhs.as_ref()).unwrap(), + lower: 1.0, + upper: 100.0, + max_samples: 10_000, + contract: "source-schema-v1".into(), + }); + let complete_provider = QueryEvidence { + topk: None, + scoped: Some(&complete), + now_ms: 10_000, + }; + assert!(complete_provider.quantile_input_domain(rhs).is_some()); + let complete_trace = inspect(&complete_provider); + assert!(complete_trace["groups"] + .as_array() + .unwrap() + .iter() + .any(|group| { + group["candidates"] + .as_array() + .unwrap() + .iter() + .any(|candidate| { + candidate["accuracy_status"] == "known" + && candidate["replacement_kind"] == "summary" + }) + })); + } + #[test] fn snapshot_metricsql_entry_uses_the_shared_serving_language_contract() { let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( @@ -6050,6 +6606,29 @@ pub(crate) mod tests { .unwrap() } + /// ERP memory remains usable under an explicit confidence target, even + /// though ERP v1 error observations cannot certify that target. + #[test] + fn erp_resource_costs_survive_confidence_target() { + let mut snapshot = planning_snapshot(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("quantile_over_time(0.9,m[1m])".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { + epsilon: 0.1, + delta: 0.01, + }); + snapshot.physical_inputs.erp = + Some(crate::physical::post_asap::cost_model::tests::erp_cost_fixture()); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); + assert!(request + .planner_selection_trace + .iter() + .flat_map(|trace| trace["groups"].as_array().unwrap()) + .flat_map(|group| group["candidates"].as_array().unwrap()) + .any(|candidate| candidate["cost_estimate"]["source"] == "erp" + && candidate["estimated_cost"] == 7777.0)); + } + fn derived_query_window_secs(query: &str) -> u64 { let mut snapshot = planning_snapshot(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; @@ -6060,7 +6639,8 @@ pub(crate) mod tests { .unwrap() .0 .queries[0] - .query_lookback_seconds + .query_lookback_ms + / 1_000 } fn set_data_ingestion_interval( @@ -6085,7 +6665,7 @@ pub(crate) mod tests { set_data_ingestion_interval(&mut snapshot, Some(1_000)); let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert_eq!(request.queries[0].query_lookback_seconds, 1); + assert_eq!(request.queries[0].query_lookback_ms, 1_000); } // Snapshot lowering must use the environment clock for cadence freshness. @@ -6125,7 +6705,7 @@ pub(crate) mod tests { request.data_workload.unwrap().data_ingestion_interval.value, Some(DurationMs(5_000)) ); - assert_eq!(request.queries[0].query_lookback_seconds, 5); + assert_eq!(request.queries[0].query_lookback_ms, 5_000); } // An explicitly invalid interval must not be replaced by the migration value. @@ -6160,31 +6740,50 @@ pub(crate) mod tests { assert_eq!(derived_query_window_secs("sum_over_time(a[1s])"), 1); } - // The seconds-based backend must fail explicitly instead of shrinking history. + /// Fractional ranges, subqueries and offsets preserve their exact history. #[test] - fn derived_history_rejects_fractional_seconds() { - for query in [ - "sum_over_time(a[1500ms])", - "sum_over_time(a[500ms])", - "sum_over_time(a[1500ms] offset 500ms)", - "sum_over_time(a[1s]) + sum(sum_over_time(b[500ms]))", - "avg_over_time((sum(a))[1500ms:])", - "sum(a offset 500ms)", + fn derived_history_preserves_milliseconds() { + for (query, expected) in [ + ("sum_over_time(a[1500ms])", 1500), + ("sum_over_time(a[500ms])", 500), + ("sum_over_time(a[1500ms] offset 500ms)", 2000), + ("sum_over_time(a[1s]) + sum(sum_over_time(b[500ms]))", 1000), + ("avg_over_time((sum(a))[1500ms:])", 6500), + ("sum(a offset 500ms)", 5500), ] { let mut snapshot = planning_snapshot(); snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query(query.into()); - let error = snapshot - .into_physical_compilation_request() - .expect_err(query) - .to_string(); - assert!( - error.contains("whole number of seconds"), - "{query}: {error}" - ); + let (request, _) = snapshot.into_physical_compilation_request().expect(query); + assert_eq!(request.queries[0].query_lookback_ms, expected, "{query}"); } } + /// A real 100 ms source stays at 100 ms through lowering and query publication. + #[test] + fn hundred_millisecond_source_compiles_without_rounding() { + let mut snapshot = planning_snapshot(); + snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = + Query("sum(data)".into()); + snapshot.physical_inputs.scrape_interval_ms = 100; + set_data_ingestion_interval(&mut snapshot, Some(100)); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + assert_eq!(request.queries[0].query_lookback_ms, 100); + let plan = DeploymentPlanCompiler + .compile_promql(request, environment) + .unwrap(); + assert_eq!( + plan.query_plan + .entries + .values() + .next() + .unwrap() + .instant + .lookback_ms, + 100 + ); + } + #[test] fn snapshot_rejects_manual_lookback() { let mut wire = serde_json::to_value(planning_snapshot()).unwrap(); @@ -6343,7 +6942,7 @@ pub(crate) mod tests { (60_000, 90_000), ] { let mut query = request.queries[0].clone(); - query.query_lookback_seconds = lookback_ms / 1_000; + query.query_lookback_ms = lookback_ms; let expr = crate::query_parser::parse_query_expr_canonical( &format!("quantile_over_time(0.5, data[{}s])", lookback_ms / 1_000), AccuracyTarget::Exact, @@ -7227,6 +7826,8 @@ pub(crate) mod tests { query_retention_margin_ms: 0, retained_summary_memory_budget_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, topk_evidence: HashMap::new(), + data_snapshot_id: None, + accuracy_evidence: HashMap::new(), exact_composition_costs: HashMap::new(), erp: None, }, @@ -7358,8 +7959,13 @@ pub(crate) mod tests { assert_eq!(encoded, fixture); assert!( - snapshot.clone().compile_promql().is_err(), - "discovery fixtures must be priced before deployment" + snapshot + .clone() + .compile_promql() + .unwrap() + .cost_comparison + .is_some(), + "deployment computes complete workload costs automatically" ); let (local, env) = snapshot .clone() @@ -7398,8 +8004,13 @@ pub(crate) mod tests { let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict compatibility demo fixture"); assert!( - snapshot.clone().compile_promql().is_err(), - "discovery fixtures must be priced before deployment" + snapshot + .clone() + .compile_promql() + .unwrap() + .cost_comparison + .is_some(), + "deployment computes complete workload costs automatically" ); let (local, env) = snapshot .clone() @@ -7663,9 +8274,9 @@ pub(crate) mod tests { let mut second = request("q40", "sum(sum_over_time(m[40s]))") .queries .remove(0); - workload.queries[0].query_lookback_seconds = 20; + workload.queries[0].query_lookback_ms = 20_000; workload.queries[0].window_realization_candidates[0].window_secs = 20; - second.query_lookback_seconds = 40; + second.query_lookback_ms = 40_000; second.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; second.window_realization_candidates[0].window_secs = 40; workload.queries.push(second); diff --git a/control_plane/src/physical/compiler/windows.rs b/control_plane/src/physical/compiler/windows.rs index 072ee7c2e..709273322 100644 --- a/control_plane/src/physical/compiler/windows.rs +++ b/control_plane/src/physical/compiler/windows.rs @@ -157,7 +157,9 @@ pub fn prepare_window_implementations( .iter() .map(|state| { ( - state.window_secs.unwrap_or(query.query_lookback_seconds), + state + .window_secs + .unwrap_or(query.query_lookback_ms.div_ceil(1_000)), cohorts.contains(&(Rc::as_ptr(&state.node) as usize)), ) }) diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 26bf26af2..97d6e9ddd 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -204,10 +204,44 @@ impl ReadoutEvidence { } } -/// ERP v1 measures error magnitudes, not tail probabilities. Only an explicit -/// epsilon-only request may use these observations as its accuracy contract. +/// Trusted source contract for every HLL readout population of one scoped query. +/// The upper bound covers the union of all merged panes, not each pane separately. +/// This is not derived from observed series count or a sampled distinct count. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct HllConfidenceContract { + /// Identifies classic 64-bit HLL with its linear-counting correction, under + /// independent uniform bucket hashing. Other estimators are not certified. + pub model: String, + pub max_distinct_per_readout: u32, +} + +impl HllConfidenceContract { + pub(crate) fn valid(&self) -> bool { + self.model == "asap-classic64-uniform-hash-linear-counting-v1" + && (1..=4096).contains(&self.max_distinct_per_readout) + } + + pub(crate) fn confidence( + &self, + epsilon: f64, + ) -> Option { + self.valid() + .then(|| { + asap_aware_mapping::hll_confidence::ClassicHllConfidence::new( + self.max_distinct_per_readout, + epsilon, + ) + }) + .flatten() + } +} + +/// ERP error magnitudes and estimator-specific confidence are separate inputs. +/// ERP v1 observed maxima do not establish a failure-probability guarantee. pub(crate) struct ErpAccuracyModel<'a> { pub policy: Option<&'a ErpPlanningInput>, + pub hll: Option<&'a HllConfidenceContract>, pub max_error: f64, } @@ -224,6 +258,20 @@ impl asap_aware_mapping::AccuracyModel for ErpAccuracyModel<'_> { query: &planner_types::post_asap::SketchQuery, ) -> Option { use planner_types::post_asap::*; + if let ( + Some(contract), + SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance), + SketchQuery::Cardinality, + ) = (self.hll, family, query) + { + if let (SketchAlgorithm::Hll, SketchParams::Hll { precision }) = + (kind.algorithm(), kind.params()) + { + // Resource ERP can still rank this state. Error maxima must not + // replace the independent, estimator-specific confidence bound. + return contract.confidence(self.max_error)?.guarantee(*precision); + } + } let theoretical = asap_aware_mapping::DefaultAccuracyModel.local_guarantee(family, query); let (Some(policy), SummaryFamilyType::Sketch(kind, _)) = (self.policy, family) else { return theoretical; @@ -575,6 +623,91 @@ impl ErpParameterDecision { } impl ErpPlanningInput { + /// Match ERP resource measurements for an already-sized candidate. Error + /// magnitudes only identify existing records here: no accuracy target is + /// certified by this cost lookup. Reuse ERP's implementation, population, + /// shape, minimum-trial and runtime checks rather than a second matcher. + pub(crate) fn candidate_resources( + &self, + algorithm: &SketchAlgorithm, + params: &SketchParams, + ) -> Option<(asap_aware_mapping::erp::ErpResourceProfile, Vec)> { + self.artifact.validate().ok()?; + let metrics = self + .artifact + .records + .iter() + .flat_map(|row| row.error_metrics.keys().cloned()) + .collect::>(); + metrics + .into_iter() + .filter_map(|metric| { + let ErpParameterDecision::Empirical { + record_id, + params: selected, + .. + } = self.select_metric( + algorithm.clone(), + &metric, + f64::MAX, + params.clone(), + Some(params), + ) + else { + return None; + }; + if &selected != params { + return None; + } + let mut ids = if self.artifact.records.iter().any(|row| row.id == record_id) { + vec![record_id] + } else { + serde_json::from_str::>(&record_id).ok()? + }; + if ids.is_empty() { + return None; + } + // The logical proxy is per partition. For population-specific + // profiles use the largest matched partition, not a pooled sketch. + let mut resources = asap_aware_mapping::erp::ErpResourceProfile { + memory_bytes: 0.0, + update_cpu_seconds: 0.0, + merge_cpu_seconds: 0.0, + query_cpu_seconds: 0.0, + }; + for id in &ids { + let row = self.artifact.records.iter().find(|row| &row.id == id)?; + resources.memory_bytes = resources.memory_bytes.max(row.resources.memory_bytes); + resources.update_cpu_seconds = resources + .update_cpu_seconds + .max(row.resources.update_cpu_seconds); + resources.merge_cpu_seconds = resources + .merge_cpu_seconds + .max(row.resources.merge_cpu_seconds); + resources.query_cpu_seconds = resources + .query_cpu_seconds + .max(row.resources.query_cpu_seconds); + } + ids.sort(); + ids.dedup(); + Some((resources, ids)) + }) + .min_by(|a, b| { + a.0.memory_bytes + .total_cmp(&b.0.memory_bytes) + .then_with(|| a.1.cmp(&b.1)) + }) + } + + pub(crate) fn candidate_memory_bytes( + &self, + algorithm: &SketchAlgorithm, + params: &SketchParams, + ) -> Option<(f64, Vec)> { + self.candidate_resources(algorithm, params) + .map(|(resources, ids)| (resources.memory_bytes, ids)) + } + pub fn select( &self, algorithm: SketchAlgorithm, @@ -873,7 +1006,10 @@ fn u32_param(parameters: &Value, names: &[&str]) -> Option { .then_some(value as u32) } -fn parse_params(algorithm: &SketchAlgorithm, parameters: &Value) -> Option { +pub(crate) fn parse_params( + algorithm: &SketchAlgorithm, + parameters: &Value, +) -> Option { let width = || u32_param(parameters, &["width", "cols"]); let depth = || u32_param(parameters, &["depth", "rows"]); Some(match algorithm { @@ -1080,6 +1216,7 @@ mod tests { GroupingStrategy::PerSubpopulationInstance, ); let model = ErpAccuracyModel { + hll: None, policy: Some(&policy), max_error: 0.2, }; @@ -1093,6 +1230,7 @@ mod tests { )); policy.artifact.records[1].error_metrics.clear(); assert!(ErpAccuracyModel { + hll: None, policy: Some(&policy), max_error: 0.2 } @@ -1114,6 +1252,7 @@ mod tests { GroupingStrategy::PerSubpopulationInstance, ); let model = ErpAccuracyModel { + hll: None, policy: Some(&policy), max_error: 0.05, }; @@ -1160,6 +1299,7 @@ mod tests { GroupingStrategy::PerSubpopulationInstance, ); let model = ErpAccuracyModel { + hll: None, policy: Some(&policy), max_error: 0.2, }; @@ -1191,6 +1331,7 @@ mod tests { .error_metrics .remove("max_frequency_entropy_absolute_bits_error"); let model = ErpAccuracyModel { + hll: None, policy: Some(&policy), max_error: 0.2, }; @@ -1444,6 +1585,15 @@ mod tests { assert!(invalid.populations.is_empty()); assert!(invalid.invalid_reason.as_deref().unwrap().contains("stale")); assert!(policy.observed_shape.is_none()); + assert!(policy + .candidate_memory_bytes( + &SketchAlgorithm::Cms, + &SketchParams::Cms { + width: 512, + depth: 3 + } + ) + .is_none()); } /// Only the activated catalog may supply a candidate's data contract. @@ -1604,6 +1754,16 @@ mod tests { minimum_confidence: 0.9, minimum_confidence_margin: 0.05, }); + assert_eq!( + policy.candidate_memory_bytes( + &SketchAlgorithm::Cms, + &SketchParams::Cms { + width: 512, + depth: 3 + } + ), + Some((12_288.0, vec!["cms-512".into()])) + ); assert!(matches!( policy.select( SketchAlgorithm::Cms, @@ -1647,6 +1807,15 @@ mod tests { policy.select(SketchAlgorithm::Cms, 0.01, theory.clone()), ErpParameterDecision::TheoreticalFallback { params, .. } if params == theory )); + assert!(policy + .candidate_memory_bytes( + &SketchAlgorithm::Cms, + &SketchParams::Cms { + width: 512, + depth: 3 + } + ) + .is_none()); } #[test] diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 9610ed0a1..8e0a4a3b5 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -167,6 +167,8 @@ pub struct ControlPlaneCostModel { offline_frequency_comparison: Option<(OfflineComparisonEvidence, OfflineComparisonRequest)>, exact_composition_costs: Vec, erp: Option, + erp_costs: Option, + hll_confidence: Option, } impl ControlPlaneCostModel { @@ -193,6 +195,8 @@ impl ControlPlaneCostModel { let dag = planner_types::post_asap::compile_executable_dag(root).ok()?; let mut value = 0.0; let mut states = 0; + let mut analytical = false; + let mut erp_record_ids = Vec::new(); for node in &dag.nodes { let family = match &node.payload { ExecutableOperatorPayload::SummaryAgg { family, .. } @@ -200,17 +204,38 @@ impl ControlPlaneCostModel { _ => continue, }; states += 1; - value += analytical_state_bytes(family)?; + let measurement = match family { + SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => self + .erp_costs + .as_ref() + .and_then(|erp| erp.candidate_memory_bytes(kind.algorithm(), kind.params())), + _ => None, + }; + if let Some((bytes, ids)) = measurement { + value += bytes; + erp_record_ids.extend(ids); + } else { + value += analytical_state_bytes(family)?; + analytical = true; + } } if states == 0 || !value.is_finite() { return None; } + erp_record_ids.sort(); + erp_record_ids.dedup(); Some(CandidateCostEstimate { value, unit: "bytes_per_state_partition", model: "backend_state_footprint_v1", - source: "analytical", - erp_record_ids: vec![], + source: if erp_record_ids.is_empty() { + "analytical" + } else if analytical { + "mixed" + } else { + "erp" + }, + erp_record_ids, }) } @@ -224,6 +249,8 @@ impl ControlPlaneCostModel { offline_frequency_comparison: None, exact_composition_costs: Vec::new(), erp: None, + erp_costs: None, + hll_confidence: None, } } @@ -235,11 +262,27 @@ impl ControlPlaneCostModel { self } + pub fn with_hll_confidence( + mut self, + contract: super::super::erp::HllConfidenceContract, + ) -> Self { + self.hll_confidence = Some(contract); + self + } + pub fn with_erp(mut self, erp: ErpPlanningInput) -> Self { + self.erp_costs = Some(erp.clone()); self.erp = Some(erp); self } + /// Resource evidence can remain usable when ERP's error observations + /// cannot establish the query's requested confidence guarantee. + pub fn with_erp_costs(mut self, erp: ErpPlanningInput) -> Self { + self.erp_costs = Some(erp); + self + } + pub fn erp_parameter_decision( &self, algorithm: SketchAlgorithm, @@ -384,6 +427,44 @@ impl ControlPlaneCostModel { costs.into_iter().map(|(algorithm, _)| algorithm).collect() } + fn rank_with_erp_costs( + &self, + intent: &AggIntent, + defaults: Vec, + ) -> Vec { + let Some(erp) = &self.erp_costs else { + return defaults; + }; + let (eps, delta) = + asap_aware_mapping::replacement::accuracy_budget(&intent_accuracy(intent)); + let mut has_measurement = false; + let mut costs = defaults + .iter() + .map(|algorithm| { + let params = self.size_params(algorithm.clone(), intent, eps, delta); + let measured = erp.candidate_memory_bytes(algorithm, ¶ms); + has_measurement |= measured.is_some(); + let cost = measured.map(|(bytes, _)| bytes).or_else(|| { + analytical_state_bytes(&SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new(algorithm.clone(), params), + GroupingStrategy::PerSubpopulationInstance, + )) + }); + (algorithm.clone(), cost) + }) + .collect::>(); + if !has_measurement { + return defaults; + } + costs.sort_by(|(_, a), (_, b)| match (a, b) { + (Some(a), Some(b)) => a.total_cmp(b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }); + costs.into_iter().map(|(algorithm, _)| algorithm).collect() + } + /// Keep concrete physical identities in Planner's complete-candidate estimate, /// including distinct pane sizes using the same abstract window framework. pub fn with_window_implementation_costs( @@ -693,7 +774,7 @@ impl CostModel for ControlPlaneCostModel { // puts it first (`summary_candidates`), nothing to reorder. _ => candidates.to_vec(), }; - self.rank_with_offline_evidence(intent, defaults) + self.rank_with_erp_costs(intent, self.rank_with_offline_evidence(intent, defaults)) } fn size_params( @@ -703,6 +784,19 @@ impl CostModel for ControlPlaneCostModel { eps: f64, delta: f64, ) -> SketchParams { + if kind == SketchAlgorithm::Hll && matches!(intent, AggIntent::Cardinality { .. }) { + if let Some(contract) = &self.hll_confidence { + if let Some((eps, delta)) = self.combined_eps_delta(&intent_accuracy(intent)) { + // If no supported precision meets the target, keep the tightest + // candidate for explain; its guarantee still fails selection. + let precision = contract + .confidence(eps) + .and_then(|model| model.precision(delta)) + .unwrap_or(18); + return SketchParams::Hll { precision }; + } + } + } let (max_error, theoretical) = match intent { AggIntent::TopK { k, .. } => { let (eps, delta) = self.topk_eps_delta(&intent_accuracy(intent)); @@ -998,7 +1092,7 @@ fn hll_precision_for_eps(eps: f64) -> u8 { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use planner_types::pre_asap::{default_cardinality, default_quantile}; @@ -1006,6 +1100,153 @@ mod tests { AccuracyTarget::Epsilon(e) } + /// Logical summary selection has an explicit Planner estimate before + /// deployment pricing, including when a family is forced. + #[test] + fn summary_candidates_have_explicit_logical_costs() { + use asap_aware_mapping::{ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG}; + use std::rc::Rc; + + let accuracy = eps(0.1); + let root = Rc::new( + crate::query_parser::parse_query_expr_canonical( + "quantile_over_time(0.9,m[1m])", + accuracy.clone(), + ) + .unwrap(), + ); + let target = TargetSubDAG::new(&root); + let model = ControlPlaneCostModel::new(accuracy.clone()); + let forced = ForcedFamilyCostModel::new(accuracy, SketchAlgorithm::DDSketch); + let candidates = SketchAlgorithmStrategy::new(&model).replacements(&target); + assert!(!candidates.is_empty()); + for candidate in &candidates { + let estimate = model.candidate_cost_estimate(candidate).unwrap(); + assert!(estimate.value.is_finite() && estimate.value > 0.0); + assert_eq!(estimate.source, "analytical"); + assert_eq!(estimate.unit, "bytes_per_state_partition"); + assert_eq!( + model.candidate_cost(candidate, &target), + Some(Cost(estimate.value)) + ); + assert_eq!( + forced.candidate_cost(candidate, &target), + Some(Cost(estimate.value)) + ); + } + } + + /// Analytical estimates scale with configured state size; unsupported + /// families remain unavailable instead of receiving a made-up zero. + #[test] + fn analytical_footprint_scales_with_parameters() { + use planner_types::post_asap::SketchKind; + let kll = |k| { + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + assert_eq!(analytical_state_bytes(&kll(200)), Some(6400.0)); + assert_eq!(analytical_state_bytes(&kll(400)), Some(12800.0)); + let unsupported = SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Theta, SketchParams::Theta { k: 1024 }), + GroupingStrategy::PerSubpopulationInstance, + ); + assert_eq!(analytical_state_bytes(&unsupported), None); + } + + pub(crate) fn erp_cost_fixture() -> ErpPlanningInput { + serde_json::from_value(serde_json::json!({ + "artifact": {"schema_version": 1, "producer_version": "synthetic-test-fixture", + "records": [{"id": "kll-200", "sketch": "kll-percall", "implementation": "lib", + "parameters": {"k": 200}, "distribution": {"test": "population-a"}, "trials": 20, + "error_metrics": {"max_rank_err": 0.9}, + "resources": {"memory_bytes": 7777.0, "update_cpu_seconds": 1e-7, + "merge_cpu_seconds": 1e-5, "query_cpu_seconds": 1e-6}}]}, + "distribution": {"test": "population-a"}, "implementation": "lib", + "error_metric": "max_rank_err", "min_trials": 10, + "expected_updates": 1000.0, "expected_queries": 100.0, "expected_merges": 1.0, + "retention_seconds": 60.0, "cpu_weight": 1.0, "byte_second_weight": 1e-9, + "mode": "hybrid" + })).unwrap() + } + + /// ERP memory overrides analytical memory for the exact configuration, + /// even when it is larger. Mismatched profiles fall back to analytical. + #[test] + fn erp_cost_precedes_analytical_without_certifying_accuracy() { + use asap_aware_mapping::{ReplacementStrategy, SketchAlgorithmStrategy}; + use std::rc::Rc; + let policy = erp_cost_fixture(); + let root = Rc::new( + crate::query_parser::parse_query_expr_canonical( + "quantile_over_time(0.9,m[1m])", + eps(0.1), + ) + .unwrap(), + ); + let model = ControlPlaneCostModel::new(eps(0.1)); + let candidates = + SketchAlgorithmStrategy::new(&model).replacements(&TargetSubDAG::new(&root)); + let candidate = candidates + .iter() + .find(|candidate| { + model + .candidate_cost_estimate(candidate) + .is_some_and(|cost| cost.value == 6400.0) + }) + .expect("KLL candidate"); + let estimate = |policy: ErpPlanningInput| { + ControlPlaneCostModel::new(eps(0.1)) + .with_erp_costs(policy) + .candidate_cost_estimate(candidate) + .unwrap() + }; + let matched = estimate(policy.clone()); + assert_eq!(matched.value, 7777.0); + assert_eq!(matched.source, "erp"); + assert_eq!(matched.erp_record_ids, ["kll-200"]); + let mut mismatch = policy.clone(); + mismatch.distribution = serde_json::json!({"test": "population-b"}); + assert_eq!(estimate(mismatch).source, "analytical"); + let mut mismatch = policy.clone(); + mismatch.implementation = Some("different-runtime".into()); + assert_eq!(estimate(mismatch).source, "analytical"); + let mut mismatch = policy.clone(); + mismatch.artifact.records[0].parameters = serde_json::json!({"k": 400}); + assert_eq!(estimate(mismatch).source, "analytical"); + let mut mismatch = policy.clone(); + mismatch.min_trials = 21; + assert_eq!(estimate(mismatch).source, "analytical"); + let mut mismatch = policy.clone(); + mismatch.artifact.records[0].resources.memory_bytes = f64::NAN; + assert_eq!(estimate(mismatch).source, "analytical"); + let model = ControlPlaneCostModel::new(eps(0.1)).with_erp_costs(policy); + assert_eq!( + model.rank_candidates( + &default_quantile(0.9), + &[SketchAlgorithm::DDSketch, SketchAlgorithm::Kll] + )[0], + SketchAlgorithm::Kll + ); + let (_, trace) = crate::planner_selection::select_workload_with_accuracy_model_and_trace( + vec![(0, root)], + eps(0.1), + &model, + &asap_aware_mapping::NoAccuracyEvidence, + &asap_aware_mapping::DefaultAccuracyModel, + ) + .unwrap(); + assert!(trace["groups"] + .as_array() + .unwrap() + .iter() + .flat_map(|group| group["candidates"].as_array().unwrap()) + .any(|candidate| candidate["cost_estimate"]["source"] == "erp" + && candidate["cost_estimate"]["unit"] == "bytes_per_state_partition")); + } + #[test] fn quantile_always_prefers_ddsketch_over_kll() { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 963ebccac..338dc5b0b 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -4,7 +4,9 @@ //! Planner owns semantic legality. A manifest describes the exact physical //! demand to price; provider quotes and candidate evaluations are separate. +mod automatic; mod materialization_candidates; +pub use automatic::{AutomaticCostReport, ComponentResources}; mod status; pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; @@ -90,6 +92,8 @@ pub struct CandidatePlanEvaluation { pub status: CandidateEvaluationStatus, pub plan_id: Option, pub total_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub automatic_cost: Option, pub unavailable_reason: Option, } @@ -500,6 +504,7 @@ fn candidate_description(candidate: &PhysicalCompilationRequest) -> CandidatePla status: CandidateEvaluationStatus::CompilationFailed, plan_id: None, total_cost: None, + automatic_cost: None, unavailable_reason: None, } } @@ -612,7 +617,7 @@ pub fn select_lowest_cost_candidate( select_candidates( candidates, env, - evidence, + Some(evidence), super::compiler::QueryFrontend::PromQl, ) } @@ -625,18 +630,20 @@ pub fn select_lowest_cost_metricsql_candidate( select_candidates( candidates, env, - evidence, + Some(evidence), super::compiler::QueryFrontend::MetricsQl, ) } -fn select_candidates( +pub fn select_candidates( candidates: Vec, env: PhysicalDeploymentContext, - evidence: &WorkloadCostEvidence, + evidence: Option<&WorkloadCostEvidence>, frontend: super::compiler::QueryFrontend, ) -> Result { - evidence.validate(&env)?; + if let Some(evidence) = evidence { + evidence.validate(&env)?; + } if candidates.is_empty() || candidates.len() > 64 { return Err(invalid( "candidate inventory must contain 1..=64 candidates", @@ -661,6 +668,7 @@ fn select_candidates( }); let planner_selection_trace = candidates[0].planner_selection_trace.clone(); let mut comparison_workload = None; + let mut comparison_inputs = None; let mut candidate_evaluations = Vec::new(); let mut best_index = 0; let mut best: Option<( @@ -670,6 +678,19 @@ fn select_candidates( BTreeMap, )> = None; for candidate in candidates { + if evidence.is_none() { + let inputs = json!({"data":candidate.data_workload,"erp":candidate.erp}); + if comparison_inputs + .as_ref() + .is_some_and(|previous| previous != &inputs) + { + return Err(invalid( + "candidates describe different data/ERP cost inputs", + )); + } + comparison_inputs = Some(inputs); + } + let cost_request = candidate.clone(); let (plan, manifest, mut description) = match compile_candidate_for_pricing(candidate, env.clone(), frontend) { Ok(bound) => bound, @@ -686,11 +707,32 @@ fn select_candidates( return Err(invalid("candidates describe different workloads/horizons")); } comparison_workload = Some(scope); - match super::realization::RealizationProvider::price( - &super::realization::ExistingRealizations, - evidence, - &manifest, - ) { + let priced = if let Some(evidence) = evidence { + super::realization::RealizationProvider::price( + &super::realization::ExistingRealizations, + evidence, + &manifest, + ) + } else { + automatic::estimate(&cost_request, &env, &plan, &manifest) + .map(|report| { + let costs = report + .components + .iter() + .map(|(id, r)| (id.clone(), r.weighted_cost())) + .collect::>(); + let total = Cost(costs.values().sum()); + description.automatic_cost = Some(report); + (total, costs) + }) + .map_err(|error| { + ( + CandidateEvaluationStatus::EvidenceMissing, + error.to_string(), + ) + }) + }; + match priced { Ok((cost, components)) => { description.status = CandidateEvaluationStatus::Unselected; description.total_cost = Some(cost.0); @@ -718,8 +760,12 @@ fn select_candidates( plan.cost_comparison = Some(CandidatePlanSelectionReport { planner_selection_trace, materialization_search_coverage, - data_snapshot_id: evidence.data_snapshot_id.clone(), - model_version: evidence.model_version.clone(), + data_snapshot_id: evidence + .map(|e| e.data_snapshot_id.clone()) + .unwrap_or_else(|| "request-scoped-data-workload".into()), + model_version: evidence + .map(|e| e.model_version.clone()) + .unwrap_or_else(|| automatic::MODEL_VERSION.into()), selected_plan_id: plan.envelope.plan_id, selected_manifest, component_costs, @@ -868,6 +914,153 @@ mod tests { use super::super::compiler::BackendLocalPlanningInput; use super::*; + /// Deployment computes and compares complete costs without external quotes. + #[test] + fn deployment_automatically_prices_workload() { + let mut input = fixture(); + input.workload_cost_evidence = None; + let plan = input.compile_promql().unwrap(); + let report = plan.cost_comparison.unwrap(); + let selected = report + .candidate_evaluations + .iter() + .find(|c| c.status == CandidateEvaluationStatus::Selected) + .unwrap(); + assert_eq!( + selected.total_cost.unwrap(), + report.component_costs.values().sum::() + ); + assert!(report + .candidate_evaluations + .iter() + .filter_map(|c| c.total_cost) + .all(|cost| cost >= selected.total_cost.unwrap())); + assert!(!report.component_costs.is_empty()); + assert_eq!( + report.component_costs.len(), + report.selected_manifest.components.len() + ); + assert!( + report + .candidate_evaluations + .iter() + .filter(|c| c.total_cost.is_some()) + .count() + > 1 + ); + } + + fn automatic_report( + request: &PhysicalCompilationRequest, + env: &PhysicalDeploymentContext, + plan: &CompiledPhysicalPlan, + ) -> AutomaticCostReport { + automatic::estimate( + request, + env, + plan, + &manifest(plan, &request.queries).unwrap(), + ) + .unwrap() + } + + /// Demand changes recurring work, while shared maintenance stays once per location. + #[test] + fn automatic_cost_scales_demand_data_and_shared_consumers() { + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request.clone(), env.clone()) + .unwrap(); + let before = automatic_report(&request, &env, &plan); + request.queries[0] + .summary_lifecycle_inputs + .evaluation_interval_ms /= 2; + let frequent = automatic_report(&request, &env, &plan); + for (id, cost) in &before.components { + let factor = if id.starts_with("query:") || id.starts_with("result:") { + 2.0 + } else { + 1.0 + }; + assert!( + (frequent.components[id].weighted_cost() - factor * cost.weighted_cost()).abs() + < 1e-10, + "{id}" + ); + } + request + .data_workload + .as_mut() + .unwrap() + .ingestion_rate + .value + .as_mut() + .unwrap() + .0 *= 2.0; + let larger = automatic_report(&request, &env, &plan); + assert!( + larger + .components + .iter() + .filter(|(id, _)| id.ends_with(":update")) + .map(|(_, c)| c.cpu_seconds) + .sum::() + > frequent + .components + .iter() + .filter(|(id, _)| id.ends_with(":update")) + .map(|(_, c)| c.cpu_seconds) + .sum::() + ); + let mut shared = plan.clone(); + let mut query = request.queries[0].clone(); + query.query_id = "second-consumer".into(); + let mut entry = shared.query_plan.entries.values().next().unwrap().clone(); + entry.query_id = query.query_id.clone(); + shared + .query_plan + .entries + .insert(query.query_id.clone(), entry); + request.queries.push(query); + let twice = automatic_report(&request, &env, &shared); + for (id, cost) in &larger.components { + assert_eq!(&twice.components[id], cost); + } + assert_eq!( + twice + .components + .keys() + .filter(|id| id.starts_with("source:") || id.starts_with("state:")) + .count(), + larger + .components + .keys() + .filter(|id| id.starts_with("source:") || id.starts_with("state:")) + .count() + ); + } + + /// Expired facts and arithmetic overflow cannot silently become a zero quote. + #[test] + fn automatic_cost_rejects_stale_and_overflowing_data() { + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request.clone(), env.clone()) + .unwrap(); + let manifest = manifest(&plan, &request.queries).unwrap(); + let rate = &mut request.data_workload.as_mut().unwrap().ingestion_rate; + rate.observed_at_ms = Some(1); + rate.valid_for_ms = Some(1); + assert!(automatic::estimate(&request, &env, &plan, &manifest) + .unwrap_err() + .to_string() + .contains("fresh ingestion rate")); + let rate = &mut request.data_workload.as_mut().unwrap().ingestion_rate; + rate.valid_for_ms = None; + rate.value.as_mut().unwrap().0 = f64::MAX; + assert!(automatic::estimate(&request, &env, &plan, &manifest).is_err()); + } + fn fixture() -> BackendLocalPlanningInput { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" @@ -1383,9 +1576,14 @@ mod tests { } #[test] - fn snapshot_requires_quotes_and_roundtrips_selection() { + fn snapshot_supports_automatic_cost_and_roundtrips_quote_override() { let mut snapshot = fixture(); - assert!(snapshot.clone().compile_promql().is_err()); + assert!(snapshot + .clone() + .compile_promql() + .unwrap() + .cost_comparison + .is_some()); let (_, _, evidence) = quoted(); snapshot.workload_cost_evidence = Some(evidence); let snapshot: BackendLocalPlanningInput = diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs new file mode 100644 index 000000000..127b90d18 --- /dev/null +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -0,0 +1,667 @@ +//! Versioned reference resource model. These coefficients are analytical +//! assumptions, not measurements or currency prices. ERP replaces only the +//! resource dimensions it measures; plan demand always comes from the backend. +use super::*; +use crate::query_plan::{residual::ResidualQueryOperator as Op, QueryPlanNode as Node}; +use asap_aware_mapping::erp::ErpResourceProfile; +use asap_types::{ + AggregationType as A, PrecomputeMaterialization, WindowMaterializationLayout as Layout, +}; +use planner_types::post_asap::SketchAlgorithm; + +pub const MODEL_VERSION: &str = "backend-workload-resources-v1"; +const CPU_PER_ITEM: f64 = 1e-7; +const CPU_PER_BYTE: f64 = 1e-9; +const SAMPLE_BYTES: f64 = 24.0; +const SERIES_BYTES: f64 = 256.0; +const MEMORY_WEIGHT: f64 = 1e-9; +const NETWORK_WEIGHT: f64 = 1e-8; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ComponentResources { + pub cpu_seconds: f64, + pub memory_byte_seconds: f64, + pub network_bytes: f64, + pub source: String, + pub erp_record_ids: Vec, + pub calculation: Value, +} +impl ComponentResources { + pub(super) fn weighted_cost(&self) -> f64 { + self.cpu_seconds + + self.memory_byte_seconds * MEMORY_WEIGHT + + self.network_bytes * NETWORK_WEIGHT + } +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AutomaticCostReport { + pub model_version: String, + pub weights: Value, + pub inputs: Value, + pub assumptions: Vec, + pub components: BTreeMap, +} +#[derive(Clone)] +struct State { + profile: ErpResourceProfile, + ids: Vec, + partitions: f64, + retained: f64, + allocations_per_second: f64, + overlap: f64, + rollup_merges: f64, +} +fn resources( + cpu: f64, + memory: f64, + network: f64, + ids: Vec, + calculation: Value, +) -> ComponentResources { + ComponentResources { + cpu_seconds: cpu, + memory_byte_seconds: memory, + network_bytes: network, + source: if ids.is_empty() { + "analytical" + } else { + "erp+analytical" + } + .into(), + erp_record_ids: ids, + calculation, + } +} +fn state( + m: &PrecomputeMaterialization, + request: &PhysicalCompilationRequest, + cardinality: u64, +) -> Result { + let bytes = + super::super::compiler::estimated_state_bytes(&m.aggregation_type, &m.parameters) as f64; + let mut profile = ErpResourceProfile { + memory_bytes: bytes, + update_cpu_seconds: CPU_PER_ITEM * (bytes / 16.0).max(2.0).log2(), + merge_cpu_seconds: CPU_PER_BYTE * bytes, + query_cpu_seconds: CPU_PER_BYTE * bytes + CPU_PER_ITEM, + }; + let mut ids = Vec::new(); + let algorithm = match m.aggregation_type { + A::DatasketchesKLL => Some(SketchAlgorithm::Kll), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let (Some(policy), Some(algorithm)) = (&request.erp, algorithm) { + let mut policy = policy.clone(); + // Use the same deployed implementation and population checks as the + // logical cost path. Unknown implementations fall back to this model. + policy.artifact.records.retain(|row| { + (row.sketch == "kll-percall" && row.implementation == "lib") + || (row.sketch == "hll" && row.implementation == "asap-sketchlib-hll-regular-v1") + || (row.sketch == "univmon" + && row.implementation == "asap-sketchlib-univmon-standard-v1") + }); + let population_matches = policy.observed_populations.is_none() + || (!request.canonical_roots.is_empty() + && request.canonical_roots.iter().all(|root| { + super::super::compiler::observed_population_matches_root(&policy, root) + })); + if population_matches { + if let Some(params) = super::super::erp::parse_params(&algorithm, &json!(m.parameters)) + { + if let Some((measured, records)) = policy.candidate_resources(&algorithm, ¶ms) { + profile = measured; + ids = records; + } + } + } + } + let (period, overlap, rollup_merges) = match &m.window_layout { + Layout::Pane { pane_secs } => (*pane_secs as f64, 1.0, 0.0), + Layout::FullWindow => ( + m.slide_interval as f64, + (m.window_size as f64 / m.slide_interval as f64).ceil(), + 0.0, + ), + Layout::HierarchicalRollup { + base_pane_secs, + levels_secs, + } => { + // Every closed child state is folded into its parent once. + let periods = std::iter::once(base_pane_secs) + .chain(levels_secs.iter().take(levels_secs.len().saturating_sub(1))); + ( + *base_pane_secs as f64, + 1.0, + periods.map(|p| 1.0 / *p as f64).sum(), + ) + } + }; + let retained = m + .num_aggregates_to_retain + .ok_or_else(|| invalid("state retention count is unknown"))? as f64; + if period <= 0.0 || retained <= 0.0 { + return Err(invalid("invalid state layout")); + } + Ok(State { + profile, + ids, + partitions: super::super::compiler::retained_partition_count(m, Some(cardinality)) as f64, + retained, + allocations_per_second: 1.0 / period + + match &m.window_layout { + Layout::HierarchicalRollup { levels_secs, .. } => { + levels_secs.iter().map(|p| 1.0 / *p as f64).sum::() + } + _ => 0.0, + }, + overlap, + rollup_merges, + }) +} + +pub(super) fn estimate( + request: &PhysicalCompilationRequest, + env: &PhysicalDeploymentContext, + plan: &CompiledPhysicalPlan, + manifest: &WorkloadCostManifest, +) -> Result { + let data = request + .data_workload + .as_ref() + .ok_or_else(|| invalid("automatic costing needs data_workload"))?; + let now = env.observed_at_unix_ms; + let horizon = manifest.horizon_seconds; + let rate = data + .ingestion_rate + .value_at(now) + .map(|r| r.0) + .ok_or_else(|| invalid("automatic costing needs a fresh ingestion rate"))?; + let cadence = data + .data_ingestion_interval + .value_at(now) + .map(|d| d.0 as f64 / 1000.0) + .ok_or_else(|| invalid("automatic costing needs fresh source cadence"))?; + if !rate.is_finite() || rate < 0.0 || !cadence.is_finite() || cadence <= 0.0 { + return Err(invalid("invalid source rate/cadence")); + } + let mut assumptions = vec![ + "Reference analytical coefficients are estimates, not calibrated benchmarks or currency prices.".into(), + "Absent per-source/selectivity facts, each source and collector uses the whole workload rate and cardinality (conservative replication).".into(), + "Grouped states use input cardinality as an upper bound on group count; filtered inputs receive no selectivity discount.".into(), + "Transport uses full state size even for deltas; memory footprint approximates encoded size; result rows include a 256-byte label allowance.".into(), + ]; + let cardinality = match data.input_cardinality.value_at(now) { + Some(value) => *value, + None => { + if rate == 0.0 { + return Err(invalid( + "zero arrival rate cannot establish active cardinality", + )); + } + assumptions.push("Unknown cardinality estimated as ceil(ingestion_rate * source_cadence), assuming one sample per active series per scrape.".into()); + let n = (rate * cadence).ceil(); + if !n.is_finite() || n >= u64::MAX as f64 { + return Err(invalid("cardinality estimate overflow")); + } + n as u64 + } + }; + if cardinality == 0 && rate > 0.0 { + return Err(invalid("positive ingestion rate with zero cardinality")); + } + let cardinality = cardinality as f64; + let mut states = BTreeMap::new(); + for schema in &plan.precompute_plan.schemas { + let m = plan + .precompute_plan + .materializations + .iter() + .find(|m| m.policy_fingerprint() == schema.materialization.fingerprint()) + .ok_or_else(|| invalid("missing materialization"))?; + states.insert( + schema.materialization.0, + state(m, request, cardinality as u64)?, + ); + } + let mut components = BTreeMap::new(); + let mut insert = |id: String, value: ComponentResources| -> Result<(), CompileError> { + if !manifest.components.contains_key(&id) { + return Err(invalid(format!("unmanifested cost {id}"))); + } + if [ + value.cpu_seconds, + value.memory_byte_seconds, + value.network_bytes, + value.weighted_cost(), + ] + .iter() + .any(|v| !v.is_finite() || *v < 0.0) + { + return Err(invalid(format!("non-finite/negative resource cost {id}"))); + } + if components.insert(id.clone(), value).is_some() { + return Err(invalid(format!("duplicate cost {id}"))); + } + Ok(()) + }; + let updates = rate * horizon; + let max_lookback = plan + .query_plan + .entries + .values() + .map(|e| e.instant.lookback_ms as f64 / 1000.0) + .fold(cadence, f64::max); + let needs_volume = matches!(data.arrival, planner_types::workload::DataArrival::AtRest) + || plan + .query_plan + .entries + .values() + .any(|e| e.instant.full_history); + let source_samples = + if needs_volume { + *data.ingestion_volume.value_at(now).ok_or_else(|| { + invalid("at-rest/full-history costing needs fresh ingestion_volume") + })? as f64 + + updates + } else { + rate * max_lookback + }; + for (id, demand) in &manifest.components { + if id.starts_with("source:") { + let location = demand.implementation["location"].as_str().unwrap_or(""); + let remote_backend = location == "backend" && !plan.collector_plans.is_empty(); + let items = if remote_backend { 0.0 } else { updates }; + let raw_retained = if location == "exact_backend" { + source_samples * SAMPLE_BYTES + cardinality * SERIES_BYTES + } else { + 0.0 + }; + insert( + id.clone(), + resources( + items * CPU_PER_ITEM, + raw_retained * horizon, + items * SAMPLE_BYTES, + vec![], + json!({"input_samples": items, "retained_raw_bytes": raw_retained, "cpu_seconds_per_sample": CPU_PER_ITEM, + "backend_summary_decode_charged_in_transport": remote_backend}), + ), + )?; + } else if id.starts_with("state:") { + let binding = &demand.implementation["binding"]; + let materialization: asap_types::PolicyFingerprint = + serde_json::from_value(binding["schema"]["materialization"].clone()) + .map_err(|e| invalid(e.to_string()))?; + let s = states + .get(&materialization) + .ok_or_else(|| invalid("unknown state"))?; + let operation = demand.implementation["operation"].as_str().unwrap_or(""); + let builds = s.partitions * (s.retained + (horizon * s.allocations_per_second).ceil()); + let remote_backend = + binding["location"] == "backend" && !plan.collector_plans.is_empty(); + let update_count = if remote_backend { + 0.0 + } else { + updates * s.overlap + }; + let merges = s.partitions * horizon * s.rollup_merges; + let (cpu, memory) = match operation { + "build" => (builds * s.profile.memory_bytes * CPU_PER_BYTE, 0.0), + "update" => ( + update_count * s.profile.update_cpu_seconds + + merges * s.profile.merge_cpu_seconds, + 0.0, + ), + "residency" => ( + 0.0, + s.partitions * s.retained * s.profile.memory_bytes * horizon, + ), + "retire" => (builds * CPU_PER_ITEM, 0.0), + _ => return Err(invalid("unsupported state phase")), + }; + insert( + id.clone(), + resources( + cpu, + memory, + 0.0, + s.ids.clone(), + json!({"operation":operation, + "partitions":s.partitions, "retained_states_per_partition":s.retained, "builds_and_retirements":builds, + "updates":update_count,"rollup_merges":merges,"unit_resources":s.profile, + "allocation_cpu_seconds_per_byte":CPU_PER_BYTE,"retirement_cpu_seconds_per_state":CPU_PER_ITEM}), + ), + )?; + } else if id.starts_with("current-series:") { + let phase = demand.implementation["phase"].as_str().unwrap_or(""); + let bytes = cardinality * SERIES_BYTES; + let retention_ms = demand.implementation["population"]["history_retention_ms"] + .as_u64() + .unwrap_or(0); + let versions = if retention_ms == 0 { + 0.0 + } else { + (retention_ms as f64 / (cadence * 1000.)).ceil() + 1.0 + }; + let retained_bytes = bytes + versions * (bytes + 1024.0); + let snapshots = if versions == 0.0 { + 0.0 + } else { + horizon / cadence + }; + let (cpu, memory) = match phase { + "build" => (bytes * CPU_PER_BYTE, 0.0), + "update" => ( + updates * CPU_PER_ITEM * cardinality.max(2.0).log2() + + snapshots * bytes * CPU_PER_BYTE, + 0.0, + ), + "residency" => (0.0, retained_bytes * horizon), + "retire" => ((cardinality + snapshots) * CPU_PER_ITEM, 0.0), + _ => return Err(invalid("unsupported current-series phase")), + }; + insert( + id.clone(), + resources( + cpu, + memory, + 0.0, + vec![], + json!({"phase":phase,"series":cardinality,"updates":updates,"bytes_per_series":SERIES_BYTES,"historical_versions":versions,"retained_bytes":retained_bytes,"snapshots":snapshots}), + ), + )?; + } + } + for rule in &plan.transmission_plan.rules { + if rule.emit_every_ms == 0 { + return Err(invalid("unknown transmission cadence")); + } + let s = states + .get(&rule.materialization.0) + .ok_or_else(|| invalid("unknown transport state"))?; + let checkpoints = rule + .full_checkpoint_every_ms + .map(|ms| { + if ms == 0 { + f64::INFINITY + } else { + horizon * 1000.0 / ms as f64 + } + }) + .unwrap_or(0.0); + let frames = (horizon * 1000.0 / rule.emit_every_ms as f64 + checkpoints).ceil() + * s.partitions + * s.retained; + let bytes = frames * s.profile.memory_bytes; + insert( + format!("transport:{}:{}", rule.producer_id, rule.materialization.0), + resources( + bytes * CPU_PER_BYTE * 2.0 + frames * s.profile.merge_cpu_seconds, + 0.0, + bytes, + s.ids.clone(), + json!({"frames":frames,"bytes_per_frame":s.profile.memory_bytes,"merge_cpu_seconds_per_frame":s.profile.merge_cpu_seconds, + "encode_decode_cpu_seconds_per_byte":2.0*CPU_PER_BYTE}), + ), + )?; + } + for entry in plan.query_plan.entries.values() { + let mut rows = BTreeMap::new(); + let mut profiles: BTreeMap<_, Vec<&State>> = BTreeMap::new(); + for node_id in entry + .topological_order() + .map_err(|e| invalid(e.to_string()))? + { + let node = &entry.nodes[&node_id]; + let id = format!("query:{}:{}", entry.query_id, node_id.0); + let evaluations = manifest.components[&id].occurrences_per_horizon; + let input_rows: f64 = node.inputs().iter().map(|id| rows[id]).sum(); + let mut output_rows = input_rows.max(1.0); + let mut used: Vec<&State> = node + .inputs() + .iter() + .flat_map(|id| profiles.get(id).into_iter().flatten().copied()) + .collect(); + let mut detail = json!({"input_rows":input_rows}); + let mut network = 0.0; + let cpu = match node { + Node::ReadMaterialization { binding } => { + let s = states + .get(&binding.materialization.0) + .ok_or_else(|| invalid("unknown read state"))?; + let panes = if binding.full_window_slide_ms.is_some() { + 1.0 + } else { + (binding + .readout_lookback_ms + .unwrap_or(entry.instant.lookback_ms) as f64 + / binding.window_ms as f64) + .ceil() + .max(1.0) + }; + output_rows = s.partitions; + used = vec![s]; + detail = json!({"partitions":s.partitions,"panes_per_read":panes,"unit_resources":s.profile}); + s.partitions + * (panes * s.profile.memory_bytes * CPU_PER_BYTE + + (panes - 1.0) * s.profile.merge_cpu_seconds) + } + Node::SummaryEstimate { .. } => used + .iter() + .map(|s| s.partitions * s.profile.query_cpu_seconds) + .sum(), + Node::SummaryMerge { .. } => used + .iter() + .map(|s| s.partitions * s.profile.merge_cpu_seconds) + .sum(), + Node::Scalar { .. } => { + output_rows = 1.0; + CPU_PER_ITEM + } + Node::ExactFallback { .. } + | Node::Logical { + operator: Op::ExactSubquery { .. } | Op::CandidateExactSubquery { .. }, + .. + } => { + let query = match node { + Node::Logical { + operator: + Op::ExactSubquery { query } | Op::CandidateExactSubquery { query, .. }, + .. + } => query.as_str(), + _ => entry.canonical_query.as_str(), + }; + let parsed = crate::query_parser::parse_query_expr_canonical( + query, + crate::types::AccuracyTarget::Exact, + ) + .map_err(|e| invalid(e.to_string()))?; + let sources = exact_source_metrics(&parsed)?.len() as f64; + let samples = if needs_volume { + *data.ingestion_volume.value_at(now).ok_or_else(|| { + invalid("full-history exact cost needs fresh ingestion_volume") + })? as f64 + } else { + (rate * (entry.instant.lookback_ms as f64 / 1000.0).max(cadence)) + .max(cardinality) + } * sources; + let operators = tree_size( + &serde_json::to_value(parsed).map_err(|e| invalid(e.to_string()))?, + ) as f64; + output_rows = cardinality.max(1.0); + network = output_rows * SERIES_BYTES + query.len() as f64; + detail = json!({"scanned_samples":samples,"syntax_objects":operators,"cpu_seconds_per_item":CPU_PER_ITEM, + "formula":"(samples + 1) * log2(max(samples, 2)) * syntax_objects * cpu_seconds_per_item"}); + (samples + 1.0) * samples.max(2.0).log2() * operators * CPU_PER_ITEM + } + Node::Logical { + operator: Op::CurrentSeries { .. }, + .. + } => { + output_rows = cardinality; + cardinality * CPU_PER_ITEM + } + Node::RelationalJoin { + inputs, + join_kind, + pred, + left_schema, + right_schema, + .. + } if matches!( + join_kind, + planner_types::pre_asap::JoinKind::Semi + | planner_types::pre_asap::JoinKind::Anti + ) => + { + output_rows = rows[&inputs[0]]; + let indexed = *join_kind == planner_types::pre_asap::JoinKind::Semi + && serde_json::from_value(pred.clone()).is_ok_and(|predicate| { + asap_physical_operators::dag::planner::equijoin_keys( + &predicate, + left_schema, + right_schema, + ) + .is_ok() + }); + detail = json!({"input_rows": input_rows, "output_rows_upper_bound": output_rows, "indexed_equality_semijoin": indexed}); + if indexed { + input_rows.max(1.0) * input_rows.max(2.0).log2() * CPU_PER_ITEM + } else { + rows[&inputs[0]] * rows[&inputs[1]] * CPU_PER_ITEM + } + } + Node::RelationalJoin { .. } => { + output_rows = input_rows.powi(2); + output_rows * CPU_PER_ITEM + } + Node::ExternalExact { .. } + | Node::Logical { + operator: Op::Scan { .. } | Op::Subquery { .. }, + .. + } => { + return Err(invalid("no automatic model for external exact/generic scan/nested subquery operator")); + } + Node::ReduceSum { grouping, .. } => { + if matches!(grouping, crate::query_plan::PhysicalGrouping::Reduce(keys) if keys.is_empty()) + { + output_rows = 1.0; + } + input_rows * CPU_PER_ITEM + } + Node::ExactReadout { .. } + | Node::Binary { .. } + | Node::Relational { .. } + | Node::Logical { .. } => { + input_rows.max(1.0) * input_rows.max(2.0).log2() * CPU_PER_ITEM + } + }; + let ids: BTreeSet<_> = if matches!( + node, + Node::ReadMaterialization { .. } + | Node::SummaryEstimate { .. } + | Node::SummaryMerge { .. } + ) { + used.iter().flat_map(|s| s.ids.iter().cloned()).collect() + } else { + BTreeSet::new() + }; + detail["evaluations"] = json!(evaluations); + detail["output_rows_bound"] = json!(output_rows); + insert( + id, + resources( + cpu * evaluations, + 0.0, + network * evaluations, + ids.into_iter().collect(), + detail, + ), + )?; + rows.insert(node_id, output_rows); + profiles.insert(node_id, used); + } + let id = format!("result:{}", entry.query_id); + let evaluations = manifest.components[&id].occurrences_per_horizon; + let bytes = rows[&entry.root] * SERIES_BYTES * evaluations; + insert( + id, + resources( + bytes * CPU_PER_BYTE, + 0.0, + bytes, + vec![], + json!({"rows_per_evaluation":rows[&entry.root],"evaluations":evaluations,"bytes_per_row":SERIES_BYTES}), + ), + )?; + } + if components.len() != manifest.components.len() { + return Err(invalid( + "automatic cost model did not cover every manifest component", + )); + } + if !components + .values() + .map(ComponentResources::weighted_cost) + .sum::() + .is_finite() + { + return Err(invalid("total workload cost overflow")); + } + Ok(AutomaticCostReport { + model_version: MODEL_VERSION.into(), + weights: json!({"unit":"weighted_resource_seconds","cpu_seconds":1.0,"memory_byte_seconds":MEMORY_WEIGHT,"network_bytes":NETWORK_WEIGHT}), + inputs: json!({"data_workload":data,"horizon_seconds":horizon,"ingestion_rate":rate,"source_cadence_seconds":cadence, + "input_cardinality":cardinality,"capability_snapshot_id":env.capability_snapshot_id,"observed_at_unix_ms":now}), + assumptions, + components, + }) +} +fn tree_size(value: &Value) -> usize { + match value { + Value::Object(fields) => 1 + fields.values().map(tree_size).sum::(), + Value::Array(items) => items.iter().map(tree_size).sum(), + _ => 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + /// A concrete implementation uses all ERP resources even when dearer; + /// a profile for a different implementation cannot replace its estimate. + #[test] + fn physical_state_resources_prefer_applicable_erp() { + let input: super::super::super::compiler::BackendLocalPlanningInput = serde_json::from_str( + include_str!("../../../../docs/examples/asapquery-planning-snapshot.json"), + ) + .unwrap(); + let (mut request, env) = input.into_physical_compilation_request().unwrap(); + let plan = super::super::super::compiler::DeploymentPlanCompiler + .compile_promql(request.clone(), env) + .unwrap(); + let mut m = plan.precompute_plan.materializations[0].clone(); + m.aggregation_type = A::DatasketchesKLL; + m.parameters = std::collections::HashMap::from([("k".into(), json!(200))]); + let baseline = state(&m, &request, 100).unwrap(); + let mut erp = crate::physical::post_asap::cost_model::tests::erp_cost_fixture(); + erp.artifact.records[0].resources = ErpResourceProfile { + memory_bytes: 1e8, + update_cpu_seconds: 0.1, + merge_cpu_seconds: 0.2, + query_cpu_seconds: 0.3, + }; + let expected = erp.artifact.records[0].resources.clone(); + request.erp = Some(erp); + let measured = state(&m, &request, 100).unwrap(); + assert!(!measured.ids.is_empty()); + assert_eq!(measured.profile, expected); + assert!(measured.profile.memory_bytes > baseline.profile.memory_bytes); + request.erp.as_mut().unwrap().artifact.records[0].implementation = "other-runtime".into(); + let fallback = state(&m, &request, 100).unwrap(); + assert!(fallback.ids.is_empty()); + assert_eq!(fallback.profile, baseline.profile); + } +} diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index 37e17fd7f..a97342bb1 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -1,8 +1,8 @@ -//! Deployment-owned selection at the latest ASAPPlanner boundary. +//! Supplies deployment capabilities and cost/accuracy evidence to ASAPPlanner. //! -//! ASAPPlanner enumerates a ranked candidate space and deliberately does not -//! commit to one deployment plan. The backend owns that decision because it -//! also owns placement, runtime capabilities, and the physical wire contract. +//! Planner constructs, evaluates, and selects computation candidates. This +//! adapter registers the supported strategies and retains selection evidence; +//! DeploymentPlanCompiler binds the resulting computation and lifecycle. use std::rc::Rc; diff --git a/control_plane/tests/discovery_snapshot.rs b/control_plane/tests/discovery_snapshot.rs index c6e4a69b0..ecedfd3fa 100644 --- a/control_plane/tests/discovery_snapshot.rs +++ b/control_plane/tests/discovery_snapshot.rs @@ -55,7 +55,7 @@ fn discovered_snapshot_plans_with_observed_cadence_and_promql_history() { .into_physical_compilation_request() .unwrap(); assert_eq!(request.scrape_interval_ms, Some(60_000)); - assert_eq!(request.queries[0].query_lookback_seconds, 3660); + assert_eq!(request.queries[0].query_lookback_ms, 3_660_000); let roundtrip: BackendLocalPlanningInput = serde_json::from_value(serde_json::to_value(&snapshot).unwrap()).unwrap(); assert_eq!(snapshot, roundtrip); diff --git a/data_plane/tests/support/distinct_planning_process.rs b/data_plane/tests/support/distinct_planning_process.rs index 72bb3dd74..242844ec5 100644 --- a/data_plane/tests/support/distinct_planning_process.rs +++ b/data_plane/tests/support/distinct_planning_process.rs @@ -1,4 +1,5 @@ use super::*; +use control_plane::physical::compiler::BackendLocalPlanningInput; /// Modeled HLL error without a confidence certificate cannot replace exact execution. #[tokio::test] @@ -14,3 +15,181 @@ async fn uncertified_distinct_uses_exact_process() { fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); assert_uncertified_exact_process(fixture, &[QUERY]).await; } + +/// The production compiler, ingest engine and query DAG preserve distinct populations. +#[tokio::test] +async fn bounded_classic_hll_executes_with_source_labels() { + const QUERY: &str = "distinct_over_time(distinct_values{job=\"api\"}[5s])"; + let fallback_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let fallback_url = format!("http://{}", fallback_listener.local_addr().unwrap()); + let fallback_task = tokio::spawn(async move { + axum::serve( + fallback_listener, + Router::new().route("/-/healthy", get(|| async { "healthy" })), + ) + .await + .unwrap(); + }); + let mut fixture: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut entry = fixture["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = QUERY.into(); + entry["requirements"]["accuracy"] = serde_json::json!({"explicit": {"Epsilon": 0.05}}); + fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + // The generated finite value domain enforces this bound over complete + // readout populations, including all panes. Series count is not the bound. + fixture["implementation"]["data_snapshot_id"] = "process-fixture".into(); + fixture["implementation"]["accuracy_evidence"] = serde_json::json!({QUERY: { + "query_string": QUERY, "data_snapshot_id": "process-fixture", + "data_workload": fixture["data_workload"], "source": "finite-generated-value-domain", + "observed_at_unix_ms": fixture["environment"]["observed_at_unix_ms"], + "valid_for_ms": 60000, + "hll": {"model":"asap-classic64-uniform-hash-linear-counting-v1", + "max_distinct_per_readout":128} + }}); + let plan = quote_snapshot_for_test( + serde_json::from_value::(fixture.clone()).unwrap(), + ) + .compile_promql() + .unwrap(); + assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!( + plan.precompute_plan.materializations[0].aggregation_type, + asap_types::AggregationType::HLL + ); + eprintln!( + "DISTINCT_PLANNED {}", + serde_json::json!({"materializations": plan.precompute_plan.materializations, "query_plan": plan.query_plan, "lifecycle_estimates": plan.lifecycle_estimates}) + ); + let output = tempfile::tempdir().unwrap(); + let path = output.path().join("planning.json"); + let priced = quote_snapshot_for_test(serde_json::from_value(fixture.clone()).unwrap()); + std::fs::write(&path, serde_json::to_vec(&priced).unwrap()).unwrap(); + let port = unused_port(); + let mut vm_port = unused_port(); + while vm_port == port { + vm_port = unused_port(); + } + let mut child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .args([ + "--forward-unsupported-queries", + "--prometheus-server", + &fallback_url, + "--profile", + "asapquery", + "--planning-snapshot", + ]) + .arg(&path) + .args(["--http-port", &port.to_string(), "--output-dir"]) + .arg(output.path()) + .args([ + "--victoriametrics-http-port", + &vm_port.to_string(), + "--victoriametrics-url", + &fallback_url, + ]) + .args([ + "--precompute-allowed-lateness-ms", + "0", + "--precompute-flush-interval-ms", + "25", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + let client = reqwest::Client::new(); + let backend = format!("http://127.0.0.1:{port}"); + wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; + // Source syntax uses the shared parser fork; serving semantics and exact + // routing belong to the MetricsQL adapter and its installed query entries. + let snapshot = serde_json::from_value::(fixture).unwrap(); + let mut snapshot = snapshot; + snapshot.environment.plan_version = 2; + let compiled = quote_snapshot_for_frontend_test(snapshot, true) + .compile_metricsql() + .unwrap(); + let identity = serde_json::json!({"plan_id": compiled.envelope.plan_id, "plan_version": compiled.envelope.plan_version}); + let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: compiled.summary_catalog, + collector_plans: compiled.collector_plans, + precompute_plan: compiled.precompute_plan, + transmission_plan: compiled.transmission_plan, + query_plan: compiled.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + }; + eprintln!( + "DISTINCT_INSTALLED {}", + serde_json::to_string(&install).unwrap() + ); + let response = client + .post(format!("{backend}/api/v1/physical-plan")) + .json(&install) + .send() + .await + .unwrap(); + assert!( + response.status().is_success(), + "{}", + response.text().await.unwrap() + ); + let response = client + .post(format!("{backend}/api/v1/physical-plan/activate")) + .json(&identity) + .send() + .await + .unwrap(); + assert!( + response.status().is_success(), + "{}", + response.text().await.unwrap() + ); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let base = now - now.rem_euclid(5000) - 20000; + let mut series = Vec::new(); + for (instance, distinct, job) in [("a", 5, "api"), ("b", 13, "api"), ("excluded", 23, "other")] + { + let mut samples: Vec<_> = (0..100) + .map(|i| (base + 1 + i, (i % distinct) as f64)) + .collect(); + samples.push((base + 15001, 1000.0)); + series.push(series_with_labels( + "distinct_values", + &[("instance", instance), ("job", job)], + &samples, + )); + } + assert_eq!( + remote_write(&client, &backend, &WriteRequest { timeseries: series }).await, + 204 + ); + drain_precompute(&client, &backend).await; + let result = wait_for_warm_instant( + &client, + &format!("http://127.0.0.1:{vm_port}"), + QUERY, + (base + 5000) as f64 / 1000.0, + &output.path().join("query_engine.log"), + ) + .await; + let rows = result["data"]["result"].as_array().unwrap(); + assert_eq!(rows.len(), 2, "{result}"); + for (instance, exact) in [("a", 5.0), ("b", 13.0)] { + let row = rows + .iter() + .find(|row| row["metric"]["instance"] == instance) + .unwrap(); + let estimate = row["value"][1].as_str().unwrap().parse::().unwrap(); + assert!((estimate - exact).abs() / exact <= 0.05, "{result}"); + } + eprintln!("DISTINCT_WARM {result}"); + fallback_task.abort(); +} diff --git a/docs/design_docs/README.md b/docs/design_docs/README.md index 9695976af..70c0687d2 100644 --- a/docs/design_docs/README.md +++ b/docs/design_docs/README.md @@ -26,6 +26,7 @@ under [developer docs](../developer_docs/README.md). Other designs and profiles: +- [Evidence-dependent candidate selection](evidence-dependent-candidates.md) defines evidence ownership, logical selection, physical admission, exact fallback, and current proof limits. - [ASAPQuery compatibility profile](asapquery-compatibility-profile.md) - [Shape-aware ERP](shape-aware-erp-v1.md) - [Empirical observability execution plan](empirical-o11y-execution-plan.md) diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index 6272905c0..2776fdb21 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -387,6 +387,16 @@ that output a `stored_output_id` and emits matching writer/reader bindings; see | Plan read/write bindings | State references, format, partition rules and writer ownership | | `SummaryStore` | Owns `summary_definitions` and `stored_summaries`; the latter holds committed metadata and payload together | +Costs include initialization, ingestion updates, overlapping and retained state, +transmission, storage, merges, readouts, recurring queries, and shared producer +construction once. Compare candidates over the same data and demand scope. +Missing evidence is not zero cost. The backend supplies candidate estimates +through `candidate_cost()`, preferring applicable ERP benchmarks and otherwise +using its analytical model. Unsupported estimates remain unavailable; +the backend combines unit resources, data size, recurrence, and the bound +physical DAG into complete workload costs before publication. See the +[calculation model](evidence-dependent-candidates.md#backend-owned-workload-calculation). + ## Compiler contract The compiler consumes: diff --git a/docs/design_docs/evidence-dependent-candidates.md b/docs/design_docs/evidence-dependent-candidates.md new file mode 100644 index 000000000..83c57b87b --- /dev/null +++ b/docs/design_docs/evidence-dependent-candidates.md @@ -0,0 +1,319 @@ +# Evidence-dependent candidate selection and deployment + +Status: evidence and workload costing implemented by backend PR #761, building +on the API adaptation in #768, against ASAPPlanner #455 +(`2ec3fc80`). This document defines the backend decision boundary for Planner +issue #454 and backend issue #752. It does not claim that every retained +candidate has a deployable implementation. + +## Decision and ownership + +Keep constructible candidates visible when external evidence is absent. +Separate candidate existence, logical selection, and deployment admission: +none implies the next. Otherwise the backend either loses a candidate it +could prove valid or deploys one whose guarantee has never been established. + +| Decision | Owner | Required behavior | +| --- | --- | --- | +| Construct semantic candidates | Planner | Preserve unknown guarantees; reject known-invalid evidence and impossible shapes | +| Supply external facts | Backend | Bind evidence to the query, data population, snapshot and validity period | +| Derive accuracy and select logical roots | Planner, under backend models and policy | Respect the root accuracy target; missing proof is not certification | +| Bind and admit a deployment | Backend | Verify concrete execution support and compute complete workload resource cost | + +The backend still invokes Planner's workload search and global selection. It +does not introduce a second semantic optimizer. Physical binding preserves the +selected DAG's operators, grouping, windows and dependencies; a semantic change +requires a new selection. Single-query and workload selection use the same +costed Planner search; the first-candidate helper has been removed. + +## Decision flow + +```mermaid +flowchart TD + Input[Queries, accuracy targets and optional backend evidence] --> Validate[Validate evidence scope and validity] + Validate -->|Invalid supplied evidence| Error[Reject request with reason] + Validate -->|Valid or absent evidence| Search[Planner search retains constructible candidates] + Search --> Inspect[Explain known and unknown candidate properties] + Search --> Select[Planner global selection under backend policy] + Select --> Exact[Explicit exact fallback when no certified summary is selected] + Select --> Bind[Bind selected logical DAG to concrete execution] + Bind --> Admit[Check guarantees, runtime support and computed workload cost] + Admit -->|Pass| Publish[Publish coherent physical plan] + Admit -->|Fail| Reject[Reject deployment] +``` + +Exact fallback is part of normal logical planning. A directly supplied summary +plan with missing or unknown readout guarantees fails physical compilation; +there is no promise that every compilation error retries another candidate. +An exact route must itself be available under the deployment's existing policy. + +## Evidence contract + +Evidence belongs to an exact query text, full data workload and data snapshot. +It also names its source, observation time and validity window. The backend +rejects mismatched, expired, future-dated or invalid records before selection. +Discovery without a workload quote needs an explicit data snapshot; when both +are supplied, their snapshot identities must agree. Queries with individual +certificates are isolated during selection so another query cannot borrow them. + +| Family | Facts that can justify a candidate | Missing-proof behavior | +| --- | --- | --- | +| DDSketch quantile ratio | Enforced input bounds and maximum sample count for each exact operand | Ratio remains visible with unknown propagated accuracy | +| Hydra | Shared-grid collision bound and failure probability | Missing bounds remain unknown | +| TopK | Selected lower bound, excluded upper bound and interval failure probability | No membership certificate is invented | +| Relative composition | Applicable non-negativity, cardinality and distribution facts | Unsupported propagation remains unknown | +| HLL / ERP readouts | A valid accuracy model including the required confidence guarantee | RSE or observed maximum error alone cannot certify the readout | + +Quantile domains describe an enforced source contract, not observed sample +extrema. Supplied TopK intervals must be complete and strictly separate selected +from excluded items. Omitted facts remain unknown; malformed supplied evidence +is rejected rather than treated as absent. The older TopK evidence interface +remains compatible; the scoped contract is the path for new producers. + +The backend validates the supplied record's scope and consistency. It does not +derive a source-domain proof from samples or establish the truth of a producer's +claimed contract. Producing valid external proofs remains an upstream duty. + +### Bounded classic HLL confidence + +Scoped evidence may carry `hll: {"model": +"asap-classic64-uniform-hash-linear-counting-v1", "max_distinct_per_readout": 128}`. +This declares an enforced source-domain upper bound for **each complete readout +population**, including the union of all merged panes, and the independent +uniform bucket-hash assumption. Series count, a sampled distinct count, and +ERP observed maxima do not establish this contract. The bound is supplied by +the source-contract owner; the backend validates its query/data/snapshot/time +scope, not the truth of the external source assertion. + +The model supports declared upper bounds from 1 through 4096 and precisions +4 through 18. It only certifies parameters that keep every permitted population +in classic HLL's linear-counting branch. A finite collision-arrival bound gives +failure probability for the requested relative error. Sizing searches for the +smallest supported precision meeting the same epsilon/delta contract. Explain +retains the model, population limit, hash assumption and numerical guarantee. + +The implementation is bound to the backend-local Regular HLL estimator; +collector estimators, HIP, MLE and unbounded populations are not certified by it. +Missing contracts retain generic HLL's unknown probability; infeasible targets +retain exact execution. ERP resources can still price the selected parameters, +but ERP error maxima cannot override this confidence model. Each readout's +probability is not a simultaneous guarantee for an entire dashboard. + +## Accuracy, runtime and cost remain separate + +A known accuracy guarantee does not establish executor support. The physical +compiler checks the executable DAG and runtime policy, and rejects summary +readouts whose guarantee is absent or contains unknown terms. Unknown support +must not be reported as deployment approval. + +Missing numerical cost remains unavailable, never zero. The backend implements +Planner's `candidate_cost()` directly; it does not enable uncosted legacy +selection. A cost estimate only supports logical selection. Publication requires complete backend-computed resource costs or an explicit, +applicable provider override. A cheap candidate cannot bypass +accuracy or runtime admission. + +Explain records accuracy status and symbolic guarantee, runtime support status, +cost availability, and the selection reason separately. A selected candidate +is labelled as pending backend binding. Rejected candidates retain their +reported reasons. This distinguishes missing proof, unsupported execution, +missing comparable cost and a candidate that simply lost the ranking. + +## ERP and analytical cost models + +ERP is the benchmark source for this path. The existing ERP artifact and +runtime-observation input feed both parameter planning and resource estimation; +there is no separate benchmark upload contract for candidate costs. + +```mermaid +flowchart TD + Profile[ERP benchmark artifact] --> Match[Match implementation, exact parameters and data population] + Observations[Declared distribution or validated runtime shape] --> Match + Candidate[Planner candidate with concrete parameters] --> Match + Match -->|Applicable profile| Measured[ERP resource estimate] + Match -->|No applicable profile| Analytical[Backend analytical resource estimate] + Measured --> Cost[Backend candidate_cost and source explanation] + Analytical --> Cost + Cost --> Selection[Planner logical selection] + Proof[Accuracy evidence and guarantee model] --> Selection + Selection --> Physical[Physical binding and complete workload costing] + Physical --> Admission[Deployment admission] +``` + +**Priority is applicable ERP, then analytical, then unavailable.** A larger +measured value still overrides a smaller analytical estimate. ERP matching +uses its existing implementation/runtime filters, exact sketch parameters, +minimum trial count, distribution equality or configured bounded shape match. +Catalog-scoped observations retain their existing population and freshness +validation. A profile for another parameter point or implementation cannot be +substituted. ERP v1 artifacts themselves have no per-record expiry field; do +not confuse runtime-observation freshness with a benchmark expiry guarantee. + +Cost lookup does not certify empirical accuracy: measured resource usage can +be useful even when observed error does not meet a target or cannot establish +its failure probability. The cost path reuses ERP profile matching without an +empirical-error threshold; the accuracy path separately checks the requested +guarantee. In particular, an explicit confidence target must not erase ERP +resource measurements merely because ERP v1 cannot prove that confidence. +Existing empirical-only accuracy policy still applies to accuracy decisions. + +The first backend model, `backend_state_footprint_v1`, estimates **retained +state bytes per partition** for local candidate selection. It uses ERP's +`memory_bytes` when applicable; otherwise it reuses the backend's existing +analytical retained-state formulas: matrix dimensions and heap capacity, +KLL capacity, HLL registers, fixed accumulator allowance and the current +DDSketch allowance. These are estimates, not measured limits or a complete +workload cost. Reachable shared state nodes are counted once. For multiple +observed populations the ERP proxy uses the largest matched partition, without +pooling the populations. Shared-grid families need their own applicable model; +an independent sketch profile is not a Hydra-grid measurement. + +The estimate excludes population counts, pane multiplicity, CPU, transmission +and other deployment costs. An applicable ERP profile also ranks sketch-family +candidates using the same byte estimates, with analytical estimates for the +unmeasured peers. Without an applicable profile, existing family preference +order remains the fallback policy. +It does not mix ERP CPU seconds with analytical bytes or claim to minimize +complete workload cost. Raw rewrites and exact compositions have no state-byte +estimate here; exact composition retains its separate measured recurring-cost +model. Unsupported shapes remain uncosted. + +Explain attaches the model, unit, source (`erp`, `analytical`, or `mixed`) and +ERP record IDs to each available estimate. Final deployment selection combines these unit resources with physical demand +as described below. A profile alone neither prices the workload nor approves +publication. + +Acceptance covers ERP precedence even when measured cost is higher, matching +parameters/implementation/population, insufficient trials, invalid measurements, +analytical fallback, unavailable shapes, explain provenance, and independent +accuracy and publication gates. + +## Backend-owned workload calculation + +Both backend-local startup and the PromQL/MetricsQL compile-and-publish API +compute workload costs when `workload_cost_evidence` is absent. Callers supply +queries, recurrence, data facts, deployment capabilities and optional ERP; they +do not need to manufacture complete candidate quotes. The backend binds the +bounded candidate inventory, constructs its coverage manifest, calculates every +component, and selects the lowest comparable total among feasible candidates. +The logical state-footprint proxy above remains a separate early selection +model; physical selection compares only the enumerated candidates, not every +possible logical DAG or window implementation. + +```mermaid +flowchart LR + ERP[Applicable ERP unit resources] --> Units[ERP first; analytical fallback] + Data[Fresh rate, cardinality, cadence, volume] --> Demand[Physical workload quantities] + Queries[Query frequency and common horizon] --> Demand + Plan[Bound DAG, shared state, panes, locations, transport] --> Demand + Units --> Compute[CPU seconds, memory byte-seconds, network bytes] + Demand --> Compute + Compute --> Compare[Common weights and complete component coverage] + Compare --> Select[Lowest-cost feasible physical candidate] +``` + +`backend-workload-resources-v1` is an explicit analytical reference model, not a +calibrated prediction or a monetary quote. It computes a resource vector before +weighting it: `cost = CPU_seconds + 1e-9 * memory_byte_seconds + 1e-8 * network_bytes`. +These fixed versioned weights express a default tradeoff; a deployment can still +supply complete calibrated provider quotes as an explicit override. Override +quotes are validated as one model across the inventory; missing/invalid quotes +are not silently patched with analytical values in unrelated units. + +For horizon H, input rate R, query interval I seconds and partition estimate P: + +| Component | Backend calculation | +| --- | --- | +| Source | R × H samples per distinct source/location; exact sources also retain raw samples for the required lookback plus series metadata | +| State initialization/retirement | P × (initial retained states + state rotations during H); allocation charges estimated bytes, retirement charges state count | +| State updates | R × H × overlapping full windows; panes receive each input once; rollup levels add their child-to-parent merges | +| State residency | P × retained physical states × unit state bytes × H | +| Transport | Rule emission/checkpoint count × partitions × retained states × estimated encoded bytes, plus serialization, decoding and receiving merges | +| Summary reads | H / I × P × panes read; decode each pane, merge additional panes, then charge the selected state's query/readout unit cost | +| Other query operators | Traverse each reachable physical node once per evaluation, using propagated row estimates and an explicit analytical operation model | +| Native exact subtrees | Source sample count for the lookback or full data volume, syntax complexity and a sorting allowance; charge each subtree's returned data separately from final output | +| Result | Result row estimate × row bytes × H / I, including serialization and delivery | + +ERP supplies state memory and per-update, per-merge and per-query CPU seconds. +The backend uses the actual workload quantities, not ERP's example invocation +counts. Applicability checks are the same implementation/parameter/runtime/ +population/shape/trial checks used by candidate costs. Multiple matched +population profiles use the maximum of each resource dimension per partition. +ERP measurements override analytical estimates even when more expensive. They +do not establish accuracy guarantees or runtime availability. + +Unmeasured dimensions use versioned analytical assumptions: 1e-7 CPU seconds per +item, 1e-9 CPU seconds per byte, 24 bytes per raw sample, and a 256-byte series/ +result-row allowance. Sketch update work scales with log2(state bytes / 16), +while merge/read work scales with state bytes. Native exact work uses +`(samples + 1) × log2(max(samples, 2)) × canonical syntax object count × item CPU`. +This is a transparent complexity proxy, not a benchmark of Prometheus; local +arithmetic/reduction/sort and joins have their own row-based estimates. + +The current data contract has workload-level facts, not a complete per-source +histogram. The model explicitly replicates the workload rate/cardinality to +each distinct source and collector rather than claiming known selectivities. +Grouped states use input cardinality as a group-count estimate. Without a fresh +cardinality, it estimates `ceil(R × scrape_interval)` under the stated assumption +of one sample per active series per scrape; a zero rate cannot establish an +unknown population. At-rest/full-history costing requires fresh ingestion volume. +Unknown rate/cadence, unsupported operators, invalid values and overflow make a +candidate unavailable. No partial total is admitted. + +Shared state and source upkeep are charged once per physical identity/location; +additional consumers add reads and outputs. For distributed plans, receiving +backend merges are charged in transport rather than again as raw updates. +Full-window overlap and pane retention are taken from the compiled layout; +rollup reads conservatively use base panes. Delta transport conservatively uses +full payload size and checkpoints; there is no unmeasured compression discount. +All recurring work uses the same horizon and original query demand. Existing +window implementation/lifecycle inputs still determine which physical layouts +are bound; their opaque weighted costs are not mixed with this resource model. + +Each candidate's `automatic_cost` report records the input data, decision time, +model/weights, assumptions, component resource vectors, workload multipliers and +applicable ERP record IDs. `component_costs` is the weighted projection of that +report, and covers exactly the existing manifest. Invalid provider overrides +remain explicit errors. Accuracy proofs, compiler capability checks and runtime +readiness/fallback policy remain independent admission requirements. + +Acceptance includes quote-free startup and HTTP compilation, ERP precedence and +implementation mismatch, frequency/data-size scaling, shared-state deduplication, +expired facts, overflow, complete coverage and provider-override compatibility. + +## Example and acceptance behavior + +For `quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])`: + +1. Without operand-domain evidence, the DDSketch ratio remains inspectable but + has unknown accuracy. Normal planning preserves exact execution. +2. With valid, scoped operand contracts, Planner can derive the ratio guarantee + and check the explicit root target. Valid evidence alone does not guarantee + that the target is met or that this candidate wins selection. +3. A selected candidate still needs physical support and complete backend-computed + workload costs before publication. A stale or cross-query certificate rejects the request. + +Cross-family acceptance includes absent, partial, valid, invalid and stale +evidence, an explicit root accuracy target, unavailable cost, and unsupported +runtime operations. Explain must never call an uncertified candidate approved; +direct physical submission must not bypass the guarantee check. + +The current conservative policy changes behavior for HLL and ERP v1: relative +standard error and benchmark maximum error lack a calibrated tail probability. +Those paths use exact execution when no independent valid guarantee exists. +Re-enabling them requires an appropriate proof model, not merely more benchmark +samples or a favorable shape-match score. + +Related: [integration architecture](asapplanner-integration.md), +[shape-aware ERP](shape-aware-erp-v1.md), +[Planner evidence contract](https://github.com/ProjectASAP/ASAPPlanner/blob/2ec3fc80caa922c8e1f33aa05f60b7d787e73257/docs/design_docs/architecture/evidence-dependent-candidates.md). + +## Time precision at admission + +Source cadence, query lookback, ranges and offsets retain integral millisecond +precision through workload lowering and query publication. A 100 ms source is +not rejected or rewritten to one second before costing. Existing pane layout +contracts still express storage sizes in seconds; rounding a storage sizing +bound must not change the query's read interval. A temporal producer that cannot +implement a fractional range remains unsupported for that binding, while exact +execution preserves the original range. Level-3 acceptance uses the actual +replay cadence and still requires a local executable plan. diff --git a/docs/design_docs/shape-aware-erp-v1.md b/docs/design_docs/shape-aware-erp-v1.md index 5d04d99af..0f0a7fccf 100644 --- a/docs/design_docs/shape-aware-erp-v1.md +++ b/docs/design_docs/shape-aware-erp-v1.md @@ -22,8 +22,12 @@ than publishing a biased partial snapshot. Profiles with too few benchmark events, poor fit, ambiguous confidence, or excessive cardinality/parameter distance are misses. -On a hit, empirical parameters and measured atomic costs are used. On a miss, -malformed evidence, or drift, Hybrid mode retains the theoretical parameters; +On a hit, empirical parameters and measured atomic costs inform planning. +ERP resource precedence, analytical fallback, accuracy certification and +deployment admission follow the +[evidence-dependent candidate design](evidence-dependent-candidates.md#erp-and-analytical-cost-models). + +On a miss, malformed evidence, or drift, Hybrid mode retains the theoretical parameters; if the runtime cannot deploy them or they exceed its memory limit, compilation chooses exact execution. Empirical-only mode fails closed. diff --git a/docs/evaluation/e2e-physical-dag.md b/docs/evaluation/e2e-physical-dag.md index 0666cefbf..6876bead4 100644 --- a/docs/evaluation/e2e-physical-dag.md +++ b/docs/evaluation/e2e-physical-dag.md @@ -39,9 +39,10 @@ Planner owns query semantics, summary families, parameters and candidate selection. ERP can affect supported evidence-based choices, but supplying an artifact does not prove that it was eligible or used. Freshness, source/update semantics, parameters and accuracy constraints still apply. The checked-in demo -is not an empirical-ERP benchmark. For deployment, use the priced snapshot workflow in [execution calibration](../../tools/o11y-execution/CALIBRATION.md). -`compile_workload_artifact` requires complete workload quotes; do not relabel -demo costs as measured evidence. +is not an empirical-ERP benchmark. For deployment, the backend automatically computes workload costs from ERP or +analytical unit resources and physical demand. Optional calibrated overrides use +the [cost evidence workflow](../examples/workload-cost-evidence.md). Do not relabel +analytical demo costs as measured evidence. For the existing observation → ERP-selected KLL → installed HTTP correctness fixture, see [ERP process validation](../developer_docs/erp-process-validation.md): @@ -73,12 +74,11 @@ dot -Tsvg target/physical-dag-inspection/selected.dot \ -o target/physical-dag-inspection/selected.svg ``` -`compile_workload_artifact` calls the same evidence-required snapshot compiler -used by startup. Set `ASAPQUERY_PLANNING_SNAPSHOT` to a priced snapshot prepared -using the [cost evidence workflow](../examples/workload-cost-evidence.md). -The checked-in unquoted templates support candidate discovery only. The output -includes the selected plan, logical selection trace and complete cost comparison. -MetricsQL compilation requires quotes collected for that frontend. +`compile_workload_artifact` calls the same snapshot compiler used by startup. +Set `ASAPQUERY_PLANNING_SNAPSHOT` to the workload snapshot. Without an explicit +provider override it uses automatic ERP/analytical workload costing. The output +includes the selected plan, logical selection trace, resource assumptions and +complete cost comparison. Both PromQL and MetricsQL use this path. The compiler derives sibling plans from the selected post-ASAP DAG: diff --git a/docs/examples/workload-cost-evidence.md b/docs/examples/workload-cost-evidence.md index 64e60498c..7adaefcc7 100644 --- a/docs/examples/workload-cost-evidence.md +++ b/docs/examples/workload-cost-evidence.md @@ -1,10 +1,16 @@ # Complete workload cost evidence Planning snapshots use one schema, `snapshot_version: 2`. Version 1 is rejected. -Candidate discovery may omit `workload_cost_evidence`; compiling a deployable -snapshot requires complete, valid quotes and selects by complete workload cost. -There is no unquoted snapshot deployment path. The checked-in JSON examples are -discovery templates, not ready-to-deploy plans. +Deployment automatically calculates workload costs when `workload_cost_evidence` +is absent, using applicable ERP profiles and analytical estimates together with +the workload's data facts, query frequency and physical layout. Inspect +`cost_comparison` for candidate totals, resource breakdowns and assumptions. +The checked-in examples can exercise this path at their recorded decision time; +use current data/capability inputs for a live deployment. + +The following workflow is an **optional provider override** for deployments with +calibrated complete quotes. See the [automatic calculation design](../design_docs/evidence-dependent-candidates.md#backend-owned-workload-calculation) +for the default path. ## Workflow @@ -65,7 +71,9 @@ can change which bound workload is committed, not rewrite its semantics. All quotes use one provider model's common cost units. A horizon quote includes the complete stated partition's work, data volume/cardinality, maintained -groups and retention. Do not reuse a global ingestion rate as a per-metric rate. +groups and retention. A calibrated provider should supply per-source demand rather than infer it from +a global rate. The automatic reference model discloses conservative replication +when only workload-level facts are available. Per-evaluation quotes exclude upkeep already charged in horizon components. Sunk infrastructure may explicitly cost zero under the provider's documented decision boundary; unknown costs may not. From 292b15cc78f65560716de1137be902f957b321b6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 03:12:52 +0000 Subject: [PATCH 02/45] Restack PR #756 with implementation before standalone acceptance --- control_plane/src/backend_client.rs | 22 ++ control_plane/src/main.rs | 84 +++++- control_plane/src/physical/compiler.rs | 11 + crates/asap_types/src/query_plan.rs | 214 +++++++++++++++ data_plane/docker-compose.yml.j2 | 1 - .../query-engine-rust-cli-compose.yml.j2 | 1 - data_plane/src/drivers/query/servers/http.rs | 85 +++++- data_plane/src/main.rs | 16 +- .../src/precompute_engine/output_sink.rs | 8 +- .../src/precompute_engine/subdag_scheduler.rs | 42 ++- data_plane/src/precompute_engine/worker.rs | 51 +++- .../asap_query_engine/catalog_resolver.rs | 68 ++--- .../query_engines/asap_query_engine/engine.rs | 201 +++++++++++--- .../asap_query_engine/exact_subqueries.rs | 255 +++++++++--------- .../asap_query_engine/logical_dag.rs | 7 + .../storage_engines/sketch_db/index/mod.rs | 40 ++- .../src/storage_engines/sketch_db/sds.rs | 25 +- docs/runtime-debugging.md | 9 + 18 files changed, 909 insertions(+), 231 deletions(-) create mode 100644 docs/runtime-debugging.md diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index dc356a00c..4329695f5 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -268,12 +268,16 @@ impl BackendClient { } /// Publish one authoritative catalog generation and all plans that reference it. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = publication.transmission_plan.envelope.plan_id, + plan_version = publication.transmission_plan.envelope.plan_version))] pub async fn post_catalog_plan_typed( &self, publication: &crate::physical::publication::PhysicalPlanPublication, storage_routing: Option, adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], ) -> std::result::Result<(), BackendPostError> { + tracing::debug!(target: "asap_runtime_debug", "backend plan stage request started"); let body = publication .install_request(storage_routing, adaptation_evidence.to_vec()) .map_err(|error| BackendPostError::Permanent(anyhow::anyhow!(error)))?; @@ -285,6 +289,8 @@ impl BackendClient { .await .map_err(classify_reqwest_error)?; let status = response.status(); + tracing::debug!(target: "asap_runtime_debug", http_status = %status, + "backend plan stage response received"); if status.is_success() { Ok(()) } else { @@ -297,11 +303,18 @@ impl BackendClient { } } + #[tracing::instrument( + level = "debug", + target = "asap_runtime_debug", + skip_all, + fields(plan_id, plan_version) + )] pub async fn discard_staged_physical_plan( &self, plan_id: u64, plan_version: u64, ) -> std::result::Result<(), BackendPostError> { + tracing::debug!(target: "asap_runtime_debug", "staged backend plan cleanup requested"); let response = self .http .post(format!( @@ -324,11 +337,18 @@ impl BackendClient { } } + #[tracing::instrument( + level = "debug", + target = "asap_runtime_debug", + skip_all, + fields(plan_id, plan_version) + )] pub async fn activate_physical_plan( &self, plan_id: u64, plan_version: u64, ) -> std::result::Result<(), BackendPostError> { + tracing::debug!(target: "asap_runtime_debug", "backend plan activation request started"); let url = format!("{}/activate", derive_physical_plan_url(&self.endpoint)); let response = self .http @@ -341,6 +361,8 @@ impl BackendClient { .await .map_err(classify_reqwest_error)?; let status = response.status(); + tracing::debug!(target: "asap_runtime_debug", http_status = %status, + "backend plan activation response received"); if status.is_success() { Ok(()) } else { diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 41dd15050..e8c3cdd84 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -18,10 +18,13 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::json; use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; use tracing::info; +static NEXT_PLAN_CALL_ID: AtomicU64 = AtomicU64::new(1); + use opamp::OpampServer; use physical::deployment_cost::online as online_cost_model; use physical::deployment_cost::online::{init_store as init_online_store, OnlineMetricsStore}; @@ -50,7 +53,16 @@ struct AppState { #[tokio::main] async fn main() { - tracing_subscriber::fmt::init(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_file(true) + .with_line_number(true) + .with_target(true) + .with_writer(std::io::stdout) + .init(); let api_addr = std::env::var("CONTROLLER_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".into()); let opamp_addr = @@ -266,11 +278,16 @@ async fn compile_and_publish_physical_plan( mut request: CompileAndPublishPhysicalPlanRequest, frontend: QueryFrontend, ) -> Response { + let call_id = NEXT_PLAN_CALL_ID.fetch_add(1, Ordering::Relaxed); + let started = std::time::Instant::now(); + tracing::debug!(target: "asap_runtime_debug", call_id, ?frontend, + "physical plan compilation requested"); // Serialize typed activations so an older response cannot overwrite the // catalog recorded after a newer backend activation. let mut active_catalog = st.active_summary_catalog.lock().await; if let Some(erp) = &mut request.erp { if let Err(error) = erp.hydrate_observed_shape(&st.runtime_samples) { + tracing::warn!(call_id, %error, "ERP observation hydration failed"); return (StatusCode::UNPROCESSABLE_ENTITY, error).into_response(); } let catalog = active_catalog.clone(); @@ -282,16 +299,37 @@ async fn compile_and_publish_physical_plan( (bundle, ids, timeout, adaptation, manifests) } Ok((None, ..)) => { + tracing::error!( + call_id, + "physical plan compilation produced no deployable plan" + ); return ( StatusCode::INTERNAL_SERVER_ERROR, "publication requires a selected plan", ) - .into_response() + .into_response(); + } + Err(response) => { + tracing::warn!(call_id, status = %response.0, error = %response.1, + "physical plan compilation failed"); + return physical_compile_failure(response); } - Err(response) => return physical_compile_failure(response), }; + info!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + collector_count = bundle.collector_plans.len(), + "physical plan compiled" + ); let Some(backend) = st.backend_client.as_ref() else { + tracing::warn!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + "backend endpoint is not configured" + ); return ( StatusCode::SERVICE_UNAVAILABLE, "CONTROLLER_BACKEND_ENDPOINT is required for physical-plan publication".to_string(), @@ -303,6 +341,8 @@ async fn compile_and_publish_physical_plan( .ensure_collector_plan_targets(&bundle.collector_plans, apply_timeout) .await { + tracing::warn!(call_id, plan_id = bundle.envelope.plan_id, plan_version = bundle.envelope.plan_version, + %error, "collector plan preflight failed"); return ( StatusCode::BAD_GATEWAY, format!("collector physical-plan preflight failed: {error}"), @@ -312,11 +352,14 @@ async fn compile_and_publish_physical_plan( let publication = match bundle.to_publication_artifact() { Ok(publication) => publication, Err(error) => { + tracing::error!(call_id, plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, %error, + "catalog publication artifact failed"); return ( StatusCode::INTERNAL_SERVER_ERROR, format!("invalid catalog publication: {error}"), ) - .into_response() + .into_response(); } }; if let Err(error) = backend @@ -327,17 +370,27 @@ async fn compile_and_publish_physical_plan( ) .await { + tracing::warn!(call_id, plan_id = bundle.envelope.plan_id, plan_version = bundle.envelope.plan_version, + %error, "backend plan staging failed"); return ( StatusCode::BAD_GATEWAY, format!("backend rejected physical plan: {error}"), ) .into_response(); } + info!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + "backend physical plan staged" + ); if let Err(error) = st .opamp .publish_collector_plans(&bundle.collector_plans, apply_timeout) .await { + tracing::warn!(call_id, plan_id = bundle.envelope.plan_id, plan_version = bundle.envelope.plan_version, + %error, "collector plan publication failed"); let cleanup = backend .discard_staged_physical_plan(bundle.envelope.plan_id, bundle.envelope.plan_version) .await; @@ -347,12 +400,26 @@ async fn compile_and_publish_physical_plan( ) .into_response(); } + info!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + "collector plans published" + ); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as u64; let activation_wait = bundle.envelope.activation_unix_ms.saturating_sub(now); if activation_wait > apply_timeout.as_millis() as u64 { + tracing::warn!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + activation_wait_ms = activation_wait, + timeout_ms = apply_timeout.as_millis() as u64, + "scheduled activation exceeds apply timeout; backend plan remains staged" + ); return ( StatusCode::GATEWAY_TIMEOUT, "activation time exceeds apply_timeout_ms; backend remains staged".to_string(), @@ -366,6 +433,8 @@ async fn compile_and_publish_physical_plan( .activate_physical_plan(bundle.envelope.plan_id, bundle.envelope.plan_version) .await { + tracing::warn!(call_id, plan_id = bundle.envelope.plan_id, plan_version = bundle.envelope.plan_version, + %error, "backend plan activation failed"); return ( StatusCode::BAD_GATEWAY, format!("backend physical-plan activation failed: {error}"), @@ -374,6 +443,13 @@ async fn compile_and_publish_physical_plan( } *active_catalog = Some(Arc::new(bundle.summary_catalog)); + info!( + call_id, + plan_id = bundle.envelope.plan_id, + plan_version = bundle.envelope.plan_version, + elapsed_ms = started.elapsed().as_millis() as u64, + "physical plan active" + ); Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index c5e93e98a..47a093497 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -381,11 +381,15 @@ pub fn gos_policy_from_accuracy_budget( }) } +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = envelope.plan_id, plan_version = envelope.plan_version, + producer_count = precompute.producers.len()))] pub fn build_transmission_plan( envelope: PlanEnvelope, precompute: &PrecomputePlan, runtime_policies: &BTreeMap, ) -> Result { + tracing::debug!(target: "asap_runtime_debug", "transmission plan construction started"); if envelope != precompute.envelope { return Err(TransmissionPlanError::EnvelopeMismatch); } @@ -957,12 +961,16 @@ impl DeploymentPlanCompiler { self.compile_for_frontend(request, environment, QueryFrontend::MetricsQl) } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(frontend = ?frontend, plan_version = environment.plan_version, + query_count = request.queries.len()))] pub fn compile_for_frontend( &self, mut request: PhysicalCompilationRequest, environment: PhysicalDeploymentContext, frontend: QueryFrontend, ) -> Result { + tracing::debug!(target: "asap_runtime_debug", "physical plan compiler entered"); if let Some(data) = &request.data_workload { data.validate() .map_err(|error| CompileError::Snapshot(error.to_string()))?; @@ -2158,6 +2166,8 @@ pub fn select_logical_roots_with_error_resource_profiles( select_logical_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) } +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(query_count = queries.len(), root_count = roots.len()))] pub fn select_logical_roots_with_trace( queries: &mut [QueryCompilationInput], roots: Vec>, @@ -2165,6 +2175,7 @@ pub fn select_logical_roots_with_trace( exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, ) -> Result, CompileError> { + tracing::debug!(target: "asap_runtime_debug", "logical root selection started"); let mut traces = Vec::new(); if roots.len() != queries.len() { return Err(CompileError::Snapshot( diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 989837541..0af7a2175 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -629,6 +629,138 @@ pub enum QueryPlanNode { } impl QueryPlanNode { + /// Operator label for logs: the serialized `op` tag, plus the residual + /// `kind` for logical nodes, including the operation where applicable + /// (e.g. `logical/aggregate/sum`). + pub fn op_label(&self) -> &'static str { + use residual::ResidualQueryOperator as R; + match self { + Self::RelationalJoin { .. } => "relational_join", + Self::Relational { .. } => "relational", + Self::Logical { operator, .. } => match operator { + R::CurrentSeries { .. } => "logical/current_series", + R::ExactSubquery { .. } => "logical/exact_subquery", + R::CandidateExactSubquery { .. } => "logical/candidate_exact_subquery", + R::Scan { .. } => "logical/scan", + R::UnaryNegate => "logical/unary_negate", + R::VectorToScalar => "logical/vector_to_scalar", + R::Aggregate { operation, .. } => match operation { + residual::Aggregation::Sum => "logical/aggregate/sum", + residual::Aggregation::Max => "logical/aggregate/max", + residual::Aggregation::Min => "logical/aggregate/min", + residual::Aggregation::Avg => "logical/aggregate/avg", + residual::Aggregation::Count => "logical/aggregate/count", + }, + R::Limit { .. } => "logical/limit", + R::Binary { .. } => "logical/binary", + R::Temporal { .. } => "logical/temporal", + R::Sort { .. } => "logical/sort", + R::HistogramQuantile => "logical/histogram_quantile", + R::Subquery { .. } => "logical/subquery", + }, + Self::Scalar { .. } => "scalar", + Self::Binary { .. } => "binary", + Self::ReduceSum { .. } => "reduce_sum", + Self::ReadMaterialization { .. } => "read_materialization", + Self::SummaryEstimate { .. } => "summary_estimate", + Self::ExactReadout { readout, .. } => match readout { + ExactReadout::Sum => "exact_readout/sum", + ExactReadout::Count => "exact_readout/count", + ExactReadout::Increase => "exact_readout/increase", + ExactReadout::Rate => "exact_readout/rate", + ExactReadout::Min => "exact_readout/min", + ExactReadout::Max => "exact_readout/max", + }, + Self::SummaryMerge { .. } => "summary_merge", + Self::ExternalExact { .. } => "external_exact", + Self::ExactFallback { .. } => "exact_fallback", + } + } + + /// Bounded, query-text-free operator arguments for execution logs. + /// The query ID links these details to the full installed plan when needed. + pub fn log_syntax(&self) -> String { + use residual::ResidualQueryOperator as R; + match self { + Self::RelationalJoin { join_kind, .. } => format!("join_kind={join_kind:?}"), + Self::Relational { .. } => String::new(), + Self::Logical { operator, .. } => match operator { + R::CurrentSeries { readout, .. } => format!("readout={readout:?}"), + R::ExactSubquery { .. } => String::new(), + R::CandidateExactSubquery { .. } => String::new(), + R::Scan { + metric, + matchers, + range_ms, + offset_ms, + } => format!( + "metric={} matcher_count={} range_ms={range_ms:?} offset_ms={offset_ms}", + metric + .as_deref() + .map(|name| name.chars().take(64).collect::()) + .unwrap_or_default(), + matchers.len(), + ), + R::UnaryNegate | R::VectorToScalar | R::HistogramQuantile => String::new(), + R::Aggregate { + operation, + grouping, + } => format!( + "operation={operation:?} grouping={}", + log_grouping(&grouping.labels, grouping.without) + ), + R::Limit { + n, + offset, + grouping, + } => format!( + "n={n} offset={offset} grouping={}", + log_grouping(&grouping.labels, grouping.without) + ), + R::Binary { + operation, + return_bool, + } => format!("operation={operation:?} return_bool={return_bool}"), + R::Temporal { operation } => format!("operation={operation:?}"), + R::Sort { + descending, + grouping, + } => format!( + "descending={descending} grouping={}", + log_grouping(&grouping.labels, grouping.without) + ), + R::Subquery { + range_ms, + step_ms, + offset_ms, + } => format!("range_ms={range_ms} step_ms={step_ms} offset_ms={offset_ms}"), + }, + Self::Scalar { value } => format!("value={value}"), + Self::Binary { operator, .. } => format!("operation={operator:?}"), + Self::ReduceSum { grouping, .. } => match grouping { + PhysicalGrouping::PerEntity => "grouping=per_entity".into(), + PhysicalGrouping::Reduce(labels) => { + format!("grouping=reduce({})", log_labels(labels)) + } + }, + Self::ReadMaterialization { binding } => format!( + "window_ms={} lookback_ms={:?}", + binding.window_ms, binding.readout_lookback_ms + ), + Self::SummaryEstimate { query, .. } => match query { + QueryReadout::FrequencyL2 => "readout=frequency_l2".into(), + QueryReadout::FrequencyEntropy => "readout=frequency_entropy".into(), + QueryReadout::Quantile { q } => format!("readout=quantile q={q}"), + QueryReadout::PointCount { .. } => "readout=point_count".into(), + QueryReadout::Cardinality => "readout=cardinality".into(), + QueryReadout::TopK { k } => format!("readout=top_k k={k}"), + }, + Self::ExactReadout { readout, .. } => format!("readout={readout:?}"), + Self::SummaryMerge { .. } => String::new(), + Self::ExternalExact { .. } | Self::ExactFallback { .. } => String::new(), + } + } + pub fn inputs(&self) -> &[QueryNodeId] { match self { Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { @@ -646,6 +778,26 @@ impl QueryPlanNode { } } +fn log_labels(labels: &[String]) -> String { + let mut names = labels + .iter() + .take(8) + .map(|label| label.chars().take(64).collect::()) + .collect::>(); + if labels.len() > 8 { + names.push("...".into()); + } + names.join(",") +} + +fn log_grouping(labels: &[String], without: bool) -> String { + format!( + "{}({})", + if without { "without" } else { "by" }, + log_labels(labels) + ) +} + pub use planner_types::post_asap::CandidateCompleteness; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] @@ -739,6 +891,8 @@ pub fn canonical_promql(query: &str) -> Result { #[cfg(test)] mod contract_tests { + use super::{residual, QueryPlanNode}; + // Installed plans cross producer/query threads without Planner Rc state. #[test] fn installed_query_contract_is_send_sync() { @@ -746,6 +900,66 @@ mod contract_tests { assert_send_sync::(); assert_send_sync::(); } + + #[test] + fn aggregate_log_labels_identify_the_operation() { + for (operation, expected) in [ + (residual::Aggregation::Sum, "logical/aggregate/sum"), + (residual::Aggregation::Count, "logical/aggregate/count"), + (residual::Aggregation::Avg, "logical/aggregate/avg"), + ] { + let node = QueryPlanNode::Logical { + operator: residual::ResidualQueryOperator::Aggregate { + operation, + grouping: residual::Grouping { + labels: vec!["service".into()], + without: false, + }, + }, + inputs: vec![], + }; + assert_eq!(node.op_label(), expected); + assert!(node.log_syntax().contains("grouping=by(service)")); + } + for (readout, expected) in [ + (super::ExactReadout::Sum, "exact_readout/sum"), + (super::ExactReadout::Count, "exact_readout/count"), + ] { + assert_eq!( + QueryPlanNode::ExactReadout { + input: super::QueryNodeId(1), + readout, + } + .op_label(), + expected + ); + } + } + + #[test] + fn execution_log_syntax_identifies_operator_without_query_text() { + let binary = QueryPlanNode::Logical { + operator: residual::ResidualQueryOperator::Binary { + operation: residual::BinaryOperation::CheckedDiv, + return_bool: false, + }, + inputs: vec![super::QueryNodeId(1), super::QueryNodeId(2)], + }; + assert_eq!(binary.op_label(), "logical/binary"); + assert_eq!( + binary.log_syntax(), + "operation=CheckedDiv return_bool=false" + ); + + let exact = QueryPlanNode::Logical { + operator: residual::ResidualQueryOperator::ExactSubquery { + query: "secret_metric{credential=\"secret\"}".into(), + }, + inputs: vec![], + }; + assert_eq!(exact.op_label(), "logical/exact_subquery"); + assert!(exact.log_syntax().is_empty()); + } } /// Bind portable relation semantics before an installed plan can access its sources. diff --git a/data_plane/docker-compose.yml.j2 b/data_plane/docker-compose.yml.j2 index cacf95efe..7a6502407 100644 --- a/data_plane/docker-compose.yml.j2 +++ b/data_plane/docker-compose.yml.j2 @@ -27,7 +27,6 @@ services: "--prometheus-server", "http://{{ prometheus_host }}:{{ prometheus_port }}", "--prometheus-scrape-interval", "{{ prometheus_scrape_interval }}", "--delete-existing-db", - "--log-level", "{{ log_level }}", "--output-dir", "/app/outputs", "--streaming-engine", "{{ streaming_engine }}", "--query-language", "{{ query_language }}", diff --git a/data_plane/query-engine-rust-cli-compose.yml.j2 b/data_plane/query-engine-rust-cli-compose.yml.j2 index 32fe8f962..d54bb1125 100644 --- a/data_plane/query-engine-rust-cli-compose.yml.j2 +++ b/data_plane/query-engine-rust-cli-compose.yml.j2 @@ -25,7 +25,6 @@ services: "--prometheus-server", "http://prometheus:9090", "--prometheus-scrape-interval", "{{ prometheus_scrape_interval }}", "--delete-existing-db", - "--log-level", "{{ log_level }}", "--output-dir", "/app/outputs", "--streaming-engine", "{{ streaming_engine }}", "--query-language", "{{ query_language }}", diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index e4b313551..2a577bb45 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -9,11 +9,14 @@ use axum::{ }; use serde_json::Value; use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::net::TcpListener; use tracing::{debug, info, warn}; +static NEXT_PLAN_REQUEST_ID: AtomicU64 = AtomicU64::new(1); + use crate::drivers::query::adapters::{AdapterConfig, HttpProtocolAdapter, PrometheusHttpAdapter}; use crate::drivers::query::servers::metrics as srv_metrics; use crate::query_engines::routing::{ @@ -1986,7 +1989,7 @@ async fn handle_runtime_info( let mut forwarding_headers = HashMap::new(); if let Some(auth) = headers.get(axum::http::header::AUTHORIZATION) { if let Ok(auth_str) = auth.to_str() { - debug!("Found Authorization header for runtime info: {}", auth_str); + debug!("Found Authorization header for runtime info"); forwarding_headers.insert("Authorization".to_string(), auth_str.to_string()); } } else { @@ -5294,10 +5297,14 @@ pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = request.transmission_plan.envelope.plan_id, + plan_version = request.transmission_plan.envelope.plan_version))] pub fn validate_and_build_runtime_plan( request: PhysicalPlanInstallRequest, default_routing: Arc, ) -> Result { + tracing::debug!(target: "asap_runtime_debug", "backend runtime plan validation started"); use std::collections::BTreeSet; request .precompute_plan @@ -5380,8 +5387,25 @@ async fn handle_post_physical_plan( ) -> axum::response::Response { use axum::http::StatusCode; use axum::response::IntoResponse; + let call_id = NEXT_PLAN_REQUEST_ID.fetch_add(1, Ordering::Relaxed); + let started = Instant::now(); + let requested_plan_id = request.transmission_plan.envelope.plan_id; + let requested_plan_version = request.transmission_plan.envelope.plan_version; + tracing::debug!(target: "asap_runtime_debug", + call_id, + plan_id = requested_plan_id, + plan_version = requested_plan_version, + "physical plan staging requested" + ); let Some(active_handle) = state.active_physical_plan.as_ref() else { + tracing::warn!( + call_id, + plan_id = requested_plan_id, + plan_version = requested_plan_version, + error = "physical-plan hot-reload handles are not attached", + "physical plan staging failed" + ); return ( StatusCode::SERVICE_UNAVAILABLE, axum::Json(serde_json::json!({ @@ -5391,6 +5415,13 @@ async fn handle_post_physical_plan( .into_response(); }; let Some(lifecycle) = state.physical_plan_lifecycle.as_ref() else { + tracing::warn!( + call_id, + plan_id = requested_plan_id, + plan_version = requested_plan_version, + error = "physical-plan lifecycle is not attached", + "physical plan staging failed" + ); return ( StatusCode::SERVICE_UNAVAILABLE, axum::Json(serde_json::json!({ @@ -5406,6 +5437,9 @@ async fn handle_post_physical_plan( &request.adaptation_evidence, unix_time_ms(), ) { + tracing::warn!(call_id, plan_id = requested_plan_id, + plan_version = requested_plan_version, + %error, "physical plan successor rejected"); return ( StatusCode::UNPROCESSABLE_ENTITY, axum::Json(serde_json::json!({ @@ -5420,6 +5454,9 @@ async fn handle_post_physical_plan( let active = match validate_and_build_runtime_plan(request, current.storage_routing.clone()) { Ok(active) => active, Err(error) => { + tracing::warn!(call_id, plan_id = requested_plan_id, + plan_version = requested_plan_version, %error, + "physical plan validation failed"); return ( StatusCode::UNPROCESSABLE_ENTITY, axum::Json(serde_json::json!({"status": "error", "error": error})), @@ -5435,6 +5472,13 @@ async fn handle_post_physical_plan( ) || active.precompute_plan.ingest.endpoint_path != "/api/v1/write") { + tracing::warn!( + call_id, + plan_id = requested_plan_id, + plan_version = requested_plan_version, + error = "Remote Write listener requires a non-bootstrap prometheus_remote_write_v1 plan at /api/v1/write", + "physical plan staging failed" + ); return ( StatusCode::UNPROCESSABLE_ENTITY, axum::Json(serde_json::json!({ @@ -5461,12 +5505,23 @@ async fn handle_post_physical_plan( let plan_version = active.plan_version(); let now = unix_time_ms(); if let Err(error) = lifecycle.stage(active, now) { + tracing::warn!(call_id, plan_id, plan_version, %error, "physical plan staging failed"); return ( StatusCode::CONFLICT, axum::Json(serde_json::json!({"status": "error", "error": error.to_string()})), ) .into_response(); } + tracing::info!( + call_id, + plan_id, + plan_version, + elapsed_ms = started.elapsed().as_millis() as u64, + materialization_count, + metricsql_query_count, + clickhouse_plan_count, + "physical plan staged" + ); ( StatusCode::ACCEPTED, axum::Json(serde_json::json!({ @@ -5491,10 +5546,25 @@ async fn handle_activate_physical_plan( axum::Json(request): axum::Json, ) -> axum::response::Response { use axum::response::IntoResponse; + let call_id = NEXT_PLAN_REQUEST_ID.fetch_add(1, Ordering::Relaxed); + let started = Instant::now(); + tracing::debug!(target: "asap_runtime_debug", + call_id, + plan_id = request.plan_id, + plan_version = request.plan_version, + "physical plan activation requested" + ); let (Some(lifecycle), Some(active_handle)) = ( state.physical_plan_lifecycle.as_ref(), state.active_physical_plan.as_ref(), ) else { + tracing::warn!( + call_id, + plan_id = request.plan_id, + plan_version = request.plan_version, + error = "physical-plan lifecycle is not attached", + "physical plan activation failed" + ); return ( StatusCode::SERVICE_UNAVAILABLE, axum::Json(serde_json::json!({ @@ -5527,16 +5597,27 @@ async fn handle_activate_physical_plan( ) { Ok(old) => old, Err(error) => { + tracing::warn!(call_id, plan_id = request.plan_id, plan_version = request.plan_version, + %error, "physical plan activation failed"); return ( StatusCode::CONFLICT, axum::Json(serde_json::json!({ "status": "error", "error": error.to_string() })), ) - .into_response() + .into_response(); } }; let activated = active_handle.active_snapshot(); + tracing::info!( + call_id, + plan_id = request.plan_id, + plan_version = request.plan_version, + elapsed_ms = started.elapsed().as_millis() as u64, + previous_plan_id = old.plan_id(), + previous_plan_version = old.plan_version(), + "physical plan activated; summary catalog installed" + ); let clickhouse_plan_count = activated .query_plan .entries diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 75152ca5e..b2f48d1c3 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -171,10 +171,6 @@ struct Args { #[arg(long, default_value = "/var/log/asap")] output_dir: String, - /// Log level - #[arg(long, default_value = "INFO")] - log_level: String, - /// Enable profiling (currently unused, kept for compatibility) #[arg(long)] do_profiling: bool, @@ -519,7 +515,7 @@ async fn main() -> Result<()> { // Initialize logging similar to Python's create_loggers function // Keep the guard alive for the entire lifetime of the application - let _log_guard = setup_logging(&args.output_dir, &args.log_level)?; + let _log_guard = setup_logging(&args.output_dir)?; info!("Starting Query Engine Rust"); info!("Output directory: {}", args.output_dir); @@ -1324,16 +1320,10 @@ async fn spawn_memory_diagnostics( } } -fn setup_logging( - output_dir: &str, - log_level: &str, -) -> Result { +fn setup_logging(output_dir: &str) -> Result { use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; - // Create env filter that respects RUST_LOG, with fallback to command line arg - let env_filter = EnvFilter::try_from_default_env() - .or_else(|_| EnvFilter::try_new(log_level)) - .unwrap_or_else(|_| EnvFilter::new("info")); + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); // Create file appender for logging to file let file_appender = tracing_appender::rolling::never(output_dir, "query_engine.log"); diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 6b2bfbc45..f17a6febb 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -5,7 +5,7 @@ use crate::storage_engines::types::hot_reload_config::InstalledPrecomputePlanHan use crate::storage_engines::types::{AggregateCore, PrecomputedOutput}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use tracing::{debug_span, warn}; +use tracing::{debug, debug_span, warn}; /// CQ-6 — process-global fallback for the output-sink policy-miss /// counter, used when a `SketchStoreSink` was constructed without an @@ -233,11 +233,15 @@ impl OutputSink for SketchStoreSink { if outputs.is_empty() { return Ok(()); } - let _span = debug_span!("sketch_index_insert", batch_size = outputs.len()).entered(); + let _span = debug_span!(target: "asap_runtime_debug", "sketch_index_insert", + batch_size = outputs.len()) + .entered(); let output_count = outputs.len(); let failed = consume_in_order(outputs, |(output, accumulator)| { self.append_to_index(output, accumulator.as_ref()) }); + debug!(target: "asap_runtime_debug", output_count, failed, + "precompute output batch stored"); if failed > 0 { return Err(format!( "SketchStore rejected {failed} of {} completed outputs", diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 6085fcab8..3e8c83877 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -82,6 +82,33 @@ where Ok(outputs.remove(0)) } +fn node_syntax(payload: &ExecutableOperatorPayload) -> String { + let details = match payload { + ExecutableOperatorPayload::Fallback { .. } => String::new(), + ExecutableOperatorPayload::Binary { operator } => { + format!("operator={operator:?}") + } + ExecutableOperatorPayload::Value { operation } => { + format!("operation={operation:?}") + } + ExecutableOperatorPayload::RelationalJoin { join_kind, .. } => { + format!("join_kind={join_kind:?}") + } + ExecutableOperatorPayload::SummaryAgg { + family, + input, + reduction, + .. + } => format!("family={family:?} input={input:?} reduction={reduction:?}"), + ExecutableOperatorPayload::SummaryJoin { family, .. } => format!("family={family:?}"), + ExecutableOperatorPayload::SummarySubtract => String::new(), + ExecutableOperatorPayload::SummaryDelete { .. } => String::new(), + ExecutableOperatorPayload::SummaryEstimate { query } => format!("readout={query:?}"), + ExecutableOperatorPayload::SummaryMerge => String::new(), + }; + details.chars().take(256).collect() +} + /// Evaluate all selected stored outputs with one dependency cache. Keys must /// describe the same input revision and window; only their output identity may /// differ. Validation finishes before executing or committing any output. @@ -270,10 +297,12 @@ where } }; if committed.contains(&node.0) { + tracing::debug!(target: "asap_runtime_debug", sink_node_id = node.0, "precompute DAG reused committed sink"); Ok(value) } else { - sink.commit_if_absent(key.clone(), value) - .map_err(ScheduleError::Sink) + let value = sink.commit_if_absent(key.clone(), value).map_err(ScheduleError::Sink)?; + tracing::debug!(target: "asap_runtime_debug", sink_node_id = node.0, "precompute DAG sink commit completed"); + Ok(value) } } }), @@ -310,6 +339,7 @@ impl> ) -> Result>, execution::Error> { Ok(futures::stream::once(async move { if let Some(source) = &self.source { + tracing::debug!(target: "asap_runtime_debug", node_id = self.node.id.0, "precompute node used materialized input"); return Ok(Arc::clone(source)); } let inputs = @@ -323,10 +353,16 @@ impl> .iter() .map(|value| Arc::clone(value.value())) .collect::>(); + let started = std::time::Instant::now(); + tracing::debug!(target: "asap_runtime_debug", node_id = self.node.id.0, phase = ?self.node.output_state.timing, syntax = %node_syntax(&self.node.payload), "precompute node started"); self.registry .execute(self.node, &values, context) - .map(Arc::new) + .map(|value| { + tracing::debug!(target: "asap_runtime_debug", node_id = self.node.id.0, elapsed_us = started.elapsed().as_micros() as u64, "precompute node completed"); + Arc::new(value) + }) .map_err(|e| { + tracing::warn!(node_id = self.node.id.0, elapsed_us = started.elapsed().as_micros() as u64, "precompute node failed"); *self.error.borrow_mut() = Some(e); execution::Error::Operator(format!("ingestion node {} failed", self.node.id.0)) }) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 8aad7f223..12286e935 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -335,8 +335,11 @@ impl Worker { } => { let sample_count = samples.len(); let _span = debug_span!( + target: "asap_runtime_debug", "worker_process_group", worker_id = self.id, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), sid, policy_fp = %policy_fp, group = %group_key, @@ -346,12 +349,13 @@ impl Worker { if let Err(e) = self.process_group_samples(sid, policy_fp, &group_key, samples) { processing_error = Some(e.to_string()); - warn!( - "Worker {} error processing sid={} (policy_fp={}, group={}): {}", - self.id, sid, policy_fp, group_key, e - ); + warn!(worker_id = self.id, sid, policy_fp = %policy_fp, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), + error = %e, "worker group processing failed"); } debug!( + target: "asap_runtime_debug", e2e_latency_us = ingest_received_at.elapsed().as_micros() as u64, "e2e: ingest->worker complete" ); @@ -362,17 +366,24 @@ impl Worker { ingest_received_at, } => { let _span = debug_span!( + target: "asap_runtime_debug", "worker_process_raw", worker_id = self.id, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), series = %series_key, sample_count = samples.len(), ) .entered(); if let Err(e) = self.process_samples_raw(&series_key, samples) { processing_error = Some(e.to_string()); - warn!("Worker {} raw error for {}: {}", self.id, series_key, e); + warn!(worker_id = self.id, series = %series_key, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), + error = %e, "worker raw processing failed"); } debug!( + target: "asap_runtime_debug", e2e_latency_us = ingest_received_at.elapsed().as_micros() as u64, "e2e: ingest->worker complete (raw)" ); @@ -386,8 +397,11 @@ impl Worker { ingest_received_at, } => { let _span = debug_span!( + target: "asap_runtime_debug", "worker_process_accumulator", worker_id = self.id, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), sid, policy_fp = %policy_fp, group = %group_key, @@ -403,12 +417,13 @@ impl Worker { accumulator, ) { processing_error = Some(e.to_string()); - warn!( - "Worker {} accumulator input error for sid={} (policy_fp={}, group={}): {}", - self.id, sid, policy_fp, group_key, e - ); + warn!(worker_id = self.id, sid, policy_fp = %policy_fp, + plan_id = ?self.current_catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = ?self.current_catalog_generation.as_ref().map(|g| g.plan_version), + error = %e, "worker accumulator processing failed"); } debug!( + target: "asap_runtime_debug", e2e_latency_us = ingest_received_at.elapsed().as_micros() as u64, "e2e: ingest->worker complete (accumulator)" ); @@ -528,6 +543,8 @@ impl Worker { /// `PrecomputeMaterialization` fingerprint used to resolve the bucket's /// config on first sight; `group_key` is held on the resulting /// `GroupState` for emit-time label rendering. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(worker_id = self.id, sid, policy_fp = %policy_fp, sample_count = samples.len()))] pub fn process_group_samples( &mut self, sid: u64, @@ -535,6 +552,7 @@ impl Worker { group_key: &Arc, samples: Vec<(String, i64, f64)>, // (series_key, timestamp_ms, value) ) -> Result<(), Box> { + debug!(target: "asap_runtime_debug", "worker group processing started"); let input_revision = self.current_input_revision.clone(); let worker_id = self.id; let allowed_lateness_ms = self.allowed_lateness_ms; @@ -552,6 +570,8 @@ impl Worker { return Ok(()); } let state = self.group_states.get_mut(&sid).unwrap(); + tracing::debug!(target: "asap_runtime_debug", worker_id = self.id, policy_fp = %policy_fp, + "precompute update route resolved"); #[cfg(not(test))] if state.program.is_none() { return Err("raw precompute requires an installed post-ASAP DAG producer".into()); @@ -851,6 +871,8 @@ impl Worker { /// /// `policy_fp` / `group_key` carry the same semantics as on /// `process_group_samples` — policy lookup + emit-time label rendering. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(worker_id = self.id, sid, policy_fp = %policy_fp, timestamp_ms))] pub fn process_accumulator_input( &mut self, sid: u64, @@ -859,6 +881,7 @@ impl Worker { timestamp_ms: i64, incoming: Box, ) -> Result<(), Box> { + debug!(target: "asap_runtime_debug", "worker accumulator processing started"); let worker_id = self.id; let allowed_lateness_ms = self.allowed_lateness_ms; let late_data_policy = self.late_data_policy; @@ -877,6 +900,10 @@ impl Worker { let state = self.group_states.get_mut(&sid).unwrap(); let previous_event_time = state.max_event_time_ms; + debug!(target: "asap_runtime_debug", aggregation_type = ?state.config.aggregation_type, + window_secs = state.config.window_size, slide_secs = state.config.slide_interval, + layout = ?state.config.window_layout, + "worker accumulator maintenance configuration selected"); let current_event_time = if timestamp_ms > previous_event_time { timestamp_ms } else { @@ -1021,11 +1048,14 @@ impl Worker { } /// Raw fast-path: emit each sample as a standalone `SumAccumulator`. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(worker_id = self.id, sample_count = samples.len()))] pub fn process_samples_raw( &self, series_key: &str, samples: Vec<(i64, f64)>, ) -> Result<(), Box> { + debug!(target: "asap_runtime_debug", "worker raw processing started"); let mut emit_batch: Vec<(PrecomputedOutput, Box)> = Vec::with_capacity(samples.len()); @@ -1097,7 +1127,10 @@ impl Worker { } } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(worker_id = self.id, group_count = self.group_states.len()))] fn flush_all(&mut self) -> Result<(), Box> { + debug!(target: "asap_runtime_debug", "worker flush started"); if self.pass_raw_samples { return Ok(()); } diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 8b454a6b5..c565845ab 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -119,40 +119,48 @@ pub(crate) fn validate_payload( let mut resolved = BTreeMap::new(); for id in entry.topological_order().map_err(|e| miss(e.to_string()))? { let node = &entry.nodes[&id]; - let state_ids = match node { - QueryPlanNode::ReadMaterialization { binding } => { - if let std::collections::btree_map::Entry::Vacant(entry) = - resolved.entry(binding.materialization) - { - entry.insert(resolve(catalog, binding.materialization)?); + let state_ids = (|| -> Result, EngineError> { + Ok(match node { + QueryPlanNode::ReadMaterialization { binding } => { + if let std::collections::btree_map::Entry::Vacant(entry) = + resolved.entry(binding.materialization) + { + entry.insert(resolve(catalog, binding.materialization)?); + } + BTreeSet::from([binding.materialization]) } - BTreeSet::from([binding.materialization]) - } - QueryPlanNode::SummaryMerge { inputs } => { - let mut ids = BTreeSet::new(); - for input in inputs { - let children: &BTreeSet = states - .get(input) - .ok_or_else(|| miss("summary merge has no state input"))?; - if children.is_empty() { - return Err(miss("summary merge has value input")); + QueryPlanNode::SummaryMerge { inputs } => { + let mut ids = BTreeSet::new(); + for input in inputs { + let children: &BTreeSet = states + .get(input) + .ok_or_else(|| miss("summary merge has no state input"))?; + if children.is_empty() { + return Err(miss("summary merge has value input")); + } + ids.extend(children); } - ids.extend(children); + ids } - ids - } - QueryPlanNode::SummaryEstimate { input, .. } - | QueryPlanNode::ExactReadout { input, .. } => { - let ids: &BTreeSet = states - .get(input) - .ok_or_else(|| miss("readout has no state input"))?; - if ids.is_empty() || ids.iter().any(|id| !resolved[id].supports(node)) { - return Err(miss("catalog descriptor cannot satisfy installed readout")); + QueryPlanNode::SummaryEstimate { input, .. } + | QueryPlanNode::ExactReadout { input, .. } => { + let ids: &BTreeSet = states + .get(input) + .ok_or_else(|| miss("readout has no state input"))?; + if ids.is_empty() || ids.iter().any(|id| !resolved[id].supports(node)) { + return Err(miss("catalog descriptor cannot satisfy installed readout")); + } + BTreeSet::new() } - BTreeSet::new() - } - _ => BTreeSet::new(), - }; + _ => BTreeSet::new(), + }) + })() + .map_err(|error| { + tracing::debug!(target: "asap_runtime_debug", query_id = %entry.query_id, + node_id = ?id, op = node.op_label(), %error, + "installed query node failed catalog validation"); + error + })?; states.insert(id, state_ids); } Ok(()) diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index b3969aa49..4b77545ee 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1,5 +1,40 @@ +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use tracing::debug; +use std::time::Instant; +use tracing::{debug, Instrument}; + +static NEXT_QUERY_CALL_ID: AtomicU64 = AtomicU64::new(1); + +fn query_call_id() -> u64 { + NEXT_QUERY_CALL_ID.fetch_add(1, Ordering::Relaxed) +} + +fn log_query_outcome( + call_id: u64, + operation: &'static str, + started: Instant, + remote_stats: Option<(usize, usize)>, + result: &Result, +) { + match result { + Ok(_) => match remote_stats { + Some((remote_evaluations, remote_rpcs)) => debug!(target: "asap_runtime_debug", + call_id, operation, elapsed_ms = started.elapsed().as_millis() as u64, + remote_evaluations, remote_rpcs, "query call completed"), + None => debug!(target: "asap_runtime_debug", + call_id, operation, elapsed_ms = started.elapsed().as_millis() as u64, + "query call completed"), + }, + Err(error @ crate::query_engines::EngineError::CapabilityMiss { .. }) => { + debug!(target: "asap_runtime_debug", + call_id, operation, elapsed_ms = started.elapsed().as_millis() as u64, + %error, "query call could not be served by ASAP tier") + } + Err(error) => tracing::warn!(call_id, operation, + elapsed_ms = started.elapsed().as_millis() as u64, %error, + "query call failed"), + } +} use asap_types::query_requirements::QueryRequirements; use asap_types::KeyByLabelNames; @@ -190,27 +225,49 @@ impl ASAPQueryEngine { now_ms: u64, ) -> Result { - let physical = self.active_physical_plan_snapshot().ok_or_else(|| { - crate::query_engines::EngineError::capability_miss( - "query_plan", - "no active physical plan", - ) - })?; - let planned = physical - .query_plan - .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) - .map_err(|error| { - crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) + let call_id = query_call_id(); + let started = Instant::now(); + let mut remote_stats = None; + debug!(target: "asap_runtime_debug", + call_id, + operation = "metricsql_instant", + evaluation_ms = now_ms, + "query call started" + ); + let result = async { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { + crate::query_engines::EngineError::capability_miss( + "query_plan", + "no active physical plan", + ) })?; - let leaves = self - .prepare_query_inputs(&physical, planned, &[now_ms]) - .await?; - let (mut result, mut stats) = - self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; - stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); - stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); - annotate_logical_execution(&mut result, &stats); - Ok(result) + let planned = physical + .query_plan + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) + .map_err(|error| { + crate::query_engines::EngineError::capability_miss( + "query_plan", + error.to_string(), + ) + })?; + debug!(target: "asap_runtime_debug", plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %planned.query_id, evaluation_ms = now_ms, + "installed MetricsQL instant query selected"); + let leaves = self + .prepare_query_inputs(&physical, planned, &[now_ms]) + .await?; + let (mut result, mut stats) = + self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; + stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); + stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); + annotate_logical_execution(&mut result, &stats); + remote_stats = Some((stats.remote_evaluations, stats.remote_rpcs)); + Ok(result) + } + .instrument(tracing::info_span!("query_call", call_id)) + .await; + log_query_outcome(call_id, "metricsql_instant", started, remote_stats, &result); + result } pub async fn execute_metricsql_range( @@ -221,20 +278,42 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { - let physical = self.active_physical_plan_snapshot().ok_or_else(|| { - crate::query_engines::EngineError::capability_miss( - "query_plan", - "no active physical plan", - ) - })?; - let planned = physical - .query_plan - .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) - .map_err(|error| { - crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) + let call_id = query_call_id(); + let started = Instant::now(); + debug!(target: "asap_runtime_debug", + call_id, + operation = "metricsql_range", + start_ms, + end_ms, + step_ms, + "query call started" + ); + let result = async { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { + crate::query_engines::EngineError::capability_miss( + "query_plan", + "no active physical plan", + ) })?; - self.execute_logical_range(&physical, planned, start_ms, end_ms, step_ms) - .await + let planned = physical + .query_plan + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) + .map_err(|error| { + crate::query_engines::EngineError::capability_miss( + "query_plan", + error.to_string(), + ) + })?; + debug!(target: "asap_runtime_debug", plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %planned.query_id, start_ms, end_ms, step_ms, + "installed MetricsQL range query selected"); + self.execute_logical_range(&physical, planned, start_ms, end_ms, step_ms) + .await + } + .instrument(tracing::info_span!("query_call", call_id)) + .await; + log_query_outcome(call_id, "metricsql_range", started, None, &result); + result } /// Construct the query executor. Runtime configuration is read only from @@ -271,12 +350,16 @@ impl ASAPQueryEngine { self.query_forwarding_policy = policy; self } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %entry.query_id, evaluation_count = times.len()))] async fn prepare_query_inputs( &self, physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { + debug!(target: "asap_runtime_debug", "installed query input preparation started"); super::catalog_resolver::validate_entry( physical.summary_catalog.as_deref(), entry, @@ -333,7 +416,7 @@ impl ASAPQueryEngine { ) }) { - debug!( + debug!(target: "asap_runtime_debug", language = ?entry.language, query_id = %entry.query_id, "query forwarding disabled; external exact subquery blocked" @@ -365,6 +448,9 @@ impl ASAPQueryEngine { .await } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %entry.query_id, evaluation_ms = at))] fn execute_logical_entry( &self, physical: &crate::storage_engines::types::RuntimePhysicalPlan, @@ -545,6 +631,9 @@ impl ASAPQueryEngine { result } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %entry.query_id, start_ms = start, end_ms = end, step_ms = step))] async fn execute_logical_range( &self, physical: &crate::storage_engines::types::RuntimePhysicalPlan, @@ -558,6 +647,7 @@ impl ASAPQueryEngine { query_result::{QueryResult, RangeVectorElement}, EngineError, }; + debug!(target: "asap_runtime_debug", "installed query range execution started"); if step == 0 || start > end || (end - start) / step >= 11_000 { return Err(EngineError::capability_miss( "installed_logical_dag", @@ -748,8 +838,21 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { + let call_id = query_call_id(); + let started = Instant::now(); + debug!(target: "asap_runtime_debug", + call_id, + operation = "promql_range", + start_ms, + end_ms, + step_ms, + "query call started" + ); + let result = async { if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { + debug!(target: "asap_runtime_debug", plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %entry.query_id, "installed query DAG selected"); return self .execute_logical_range(&physical, entry, start_ms, end_ms, step_ms) .await; @@ -852,6 +955,9 @@ impl ASAPQueryEngine { // Complete coverage is a prerequisite above. Hybrid stitching is // retained for legacy/test callers without an active QueryPlan only. Ok(warm_qr) + }.instrument(tracing::info_span!("query_call", call_id)).await; + log_query_outcome(call_id, "promql_range", started, None, &result); + result } } @@ -995,25 +1101,43 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu now_ms: u64, ) -> Result { + let call_id = query_call_id(); + let started = Instant::now(); + let mut remote_stats = None; + debug!(target: "asap_runtime_debug", + call_id, + operation = "promql_instant", + evaluation_ms = now_ms, + "query call started" + ); + let result = async { if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { + debug!(target: "asap_runtime_debug", plan_id = physical.plan_id(), plan_version = physical.plan_version(), + query_id = %entry.query_id, evaluation_ms = now_ms, + "installed query DAG selected"); let leaves = self .prepare_query_inputs(&physical, entry, &[now_ms]) .await .map_err(|error| { - tracing::warn!(query, error = %error, "installed query DAG preparation failed"); + tracing::warn!(plan_id = physical.plan_id(), + plan_version = physical.plan_version(), query_id = %entry.query_id, + error = %error, "installed query DAG preparation failed"); error })?; let (mut result, mut stats) = self .execute_logical_entry(&physical, entry, &leaves, now_ms) .map_err(|error| { - tracing::warn!(query, error = %error, "installed query DAG execution failed"); + tracing::warn!(plan_id = physical.plan_id(), + plan_version = physical.plan_version(), query_id = %entry.query_id, + error = %error, "installed query DAG execution failed"); error })?; stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); annotate_logical_execution(&mut result, &stats); + remote_stats = Some((stats.remote_evaluations, stats.remote_rpcs)); return Ok(result); } } @@ -1108,6 +1232,9 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), format!("ASAPQueryEngine: no sketch index for `{query}` — failing over to archive"), )) + }.instrument(tracing::info_span!("query_call", call_id)).await; + log_query_outcome(call_id, "promql_instant", started, remote_stats, &result); + result } /// Range-query entry point for the [`EngineRouter`] failover loop. diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index c7e9e1651..66d85abbd 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -296,147 +296,154 @@ pub(super) async fn prepare_external( // Equivalent exact cuts at the same time share one actual remote request. let mut remote_cache = HashMap::<(QueryLanguage, String, i64), Value>::new(); for ((id, at), leaf) in leaves(entry, times)? { - u64::try_from(at).map_err(|_| miss("subquery predates epoch"))?; - let (language, query, candidate_input) = match &leaf { - ExactLeaf::Legacy(ResidualQueryOperator::ExactSubquery { query }) => { - (QueryLanguage::PromQl, query.clone(), None) - } - ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { - query, - item_label, - }) => ( - QueryLanguage::PromQl, - query.clone(), - Some((entry.nodes[&id].inputs()[0], item_label.as_str())), - ), - ExactLeaf::External(request) => { - if !matches!( - request.language, - QueryLanguage::PromQl | QueryLanguage::MetricsQl - ) { - return Err(miss(format!( - "external exact language {:?} has no installed adapter", - request.language - ))); + let result = async { + u64::try_from(at).map_err(|_| miss("subquery predates epoch"))?; + let (language, query, candidate_input) = match &leaf { + ExactLeaf::Legacy(ResidualQueryOperator::ExactSubquery { query }) => { + (QueryLanguage::PromQl, query.clone(), None) } - let candidate = match request.input_contracts.as_slice() { - [] => None, - [ExternalExactInput::CandidateMembership { item_label }] => { - Some((entry.nodes[&id].inputs()[0], item_label.as_str())) + ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { + query, + item_label, + }) => ( + QueryLanguage::PromQl, + query.clone(), + Some((entry.nodes[&id].inputs()[0], item_label.as_str())), + ), + ExactLeaf::External(request) => { + if !matches!( + request.language, + QueryLanguage::PromQl | QueryLanguage::MetricsQl + ) { + return Err(miss(format!( + "external exact language {:?} has no installed adapter", + request.language + ))); } - _ => return Err(miss("unsupported external exact input contract")), - }; - (request.language, request.expression.clone(), candidate) - } - _ => return Err(miss("prepared leaf is not an exact subtree")), - }; - let (query, candidate_filtered) = if let Some((candidate_input, item_label)) = - candidate_input - { - if language == QueryLanguage::MetricsQl { - return Err(miss( - "candidate-filtered MetricsQL exact subqueries are not implemented", - )); - } - let candidate = prepared - .get(&(candidate_input, at)) - .ok_or_else(|| miss("candidate membership was not prepared"))?; - let Value::Vector(rows) = &candidate.value else { - return Err(miss("candidate membership is not an instant vector")); + let candidate = match request.input_contracts.as_slice() { + [] => None, + [ExternalExactInput::CandidateMembership { item_label }] => { + Some((entry.nodes[&id].inputs()[0], item_label.as_str())) + } + _ => return Err(miss("unsupported external exact input contract")), + }; + (request.language, request.expression.clone(), candidate) + } + _ => return Err(miss("prepared leaf is not an exact subtree")), }; - let mut values = rows - .iter() - .map(|(labels, _)| { - labels.get(item_label).cloned().ok_or_else(|| { - miss(format!( - "candidate membership is missing item label {item_label}" - )) + let (query, candidate_filtered) = if let Some((candidate_input, item_label)) = + candidate_input + { + if language == QueryLanguage::MetricsQl { + return Err(miss( + "candidate-filtered MetricsQL exact subqueries are not implemented", + )); + } + let candidate = prepared + .get(&(candidate_input, at)) + .ok_or_else(|| miss("candidate membership was not prepared"))?; + let Value::Vector(rows) = &candidate.value else { + return Err(miss("candidate membership is not an instant vector")); + }; + let mut values = rows + .iter() + .map(|(labels, _)| { + labels.get(item_label).cloned().ok_or_else(|| { + miss(format!( + "candidate membership is missing item label {item_label}" + )) + }) }) - }) - .collect::, _>>()?; - values.sort(); - values.dedup(); - if values.is_empty() { - prepared.insert( - (id, at), - PreparedLeaf { + .collect::, _>>()?; + values.sort(); + values.dedup(); + if values.is_empty() { + return Ok(PreparedLeaf { value: Value::Vector(Vec::new()), remote: true, remote_evaluations: 0, remote_rpcs: 0, - }, - ); - continue; - } - if values.len() > MAX_CANDIDATE_VALUES { - return Err(miss(format!( - "candidate set has {} values, exceeding limit {MAX_CANDIDATE_VALUES}", - values.len() - ))); - } - let restricted = inject_candidate_matcher(&query, item_label, &values)?; - if restricted.len() > MAX_CANDIDATE_QUERY_BYTES { - return Err(miss(format!( - "candidate-filtered exact query has {} bytes, exceeding limit {MAX_CANDIDATE_QUERY_BYTES}", - restricted.len() + }); + } + if values.len() > MAX_CANDIDATE_VALUES { + return Err(miss(format!( + "candidate set has {} values, exceeding limit {MAX_CANDIDATE_VALUES}", + values.len() ))); - } - (restricted, true) - } else { - (query, false) - }; - let key = (language, query.clone(), at); - let cached = remote_cache.contains_key(&key); - let value = if let Some(value) = remote_cache.get(&key) { - value.clone() - } else { - let endpoint = match language { - QueryLanguage::PromQl => prometheus_endpoint - .ok_or_else(|| miss("Prometheus exact endpoint unavailable"))?, - QueryLanguage::MetricsQl => metricsql_endpoint - .ok_or_else(|| miss("VictoriaMetrics exact endpoint unavailable"))?, - QueryLanguage::ClickHouseSql => { - return Err(miss("ClickHouse exact subquery needs its SQL adapter")) } + let restricted = inject_candidate_matcher(&query, item_label, &values)?; + if restricted.len() > MAX_CANDIDATE_QUERY_BYTES { + return Err(miss(format!( + "candidate-filtered exact query has {} bytes, exceeding limit {MAX_CANDIDATE_QUERY_BYTES}", + restricted.len() + ))); + } + (restricted, true) + } else { + (query, false) }; - let url = format!("{}/api/v1/query", endpoint.trim_end_matches('/')); - let time = format!("{:.3}", at as f64 / 1000.0); - // Candidate sets can be large enough to exceed proxy URL limits; - // Prometheus accepts the instant-query parameters as an encoded - // form body. Static exact cuts keep their existing GET contract. - let request = if candidate_filtered { - client - .post(url) - .form(&[("query", query.as_str()), ("time", time.as_str())]) + let key = (language, query.clone(), at); + let cached = remote_cache.contains_key(&key); + let value = if let Some(value) = remote_cache.get(&key) { + value.clone() } else { - client - .get(url) - .query(&[("query", query.as_str()), ("time", time.as_str())]) + let endpoint = match language { + QueryLanguage::PromQl => prometheus_endpoint + .ok_or_else(|| miss("Prometheus exact endpoint unavailable"))?, + QueryLanguage::MetricsQl => metricsql_endpoint + .ok_or_else(|| miss("VictoriaMetrics exact endpoint unavailable"))?, + QueryLanguage::ClickHouseSql => { + return Err(miss("ClickHouse exact subquery needs its SQL adapter")) + } + }; + let url = format!("{}/api/v1/query", endpoint.trim_end_matches('/')); + let time = format!("{:.3}", at as f64 / 1000.0); + // Candidate sets can be large enough to exceed proxy URL limits; + // Prometheus accepts the instant-query parameters as an encoded + // form body. Static exact cuts keep their existing GET contract. + let request = if candidate_filtered { + client + .post(url) + .form(&[("query", query.as_str()), ("time", time.as_str())]) + } else { + client + .get(url) + .query(&[("query", query.as_str()), ("time", time.as_str())]) + }; + let response = request + .send() + .await + .map_err(|e| miss(format!("exact request failed: {e}")))?; + if !response.status().is_success() { + return Err(miss(format!("exact endpoint HTTP {}", response.status()))); + } + let body: serde_json::Value = response + .json() + .await + .map_err(|e| miss(format!("invalid exact response: {e}")))?; + let value = parse_result(&body, at)?; + remote_cache.insert(key, value.clone()); + value }; - let response = request - .send() - .await - .map_err(|e| miss(format!("exact request failed: {e}")))?; - if !response.status().is_success() { - return Err(miss(format!("exact endpoint HTTP {}", response.status()))); - } - let body: serde_json::Value = response - .json() - .await - .map_err(|e| miss(format!("invalid exact response: {e}")))?; - let value = parse_result(&body, at)?; - remote_cache.insert(key, value.clone()); - value - }; - prepared.insert( - (id, at), - PreparedLeaf { + Ok(PreparedLeaf { value, remote: true, remote_evaluations: usize::from(!cached), remote_rpcs: usize::from(!cached), - }, - ); + }) + } + .await; + match result { + Ok(leaf) => { + prepared.insert((id, at), leaf); + } + Err(error) => { + tracing::debug!(target: "asap_runtime_debug", query_id = %entry.query_id, + node_id = ?id, op = entry.nodes[&id].op_label(), evaluation_ms = at, %error, + "installed query exact leaf could not be prepared"); + return Err(error); + } + } } Ok(prepared) } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 5573bd45a..f87f0a98c 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -135,6 +135,7 @@ where for dependency in &dependencies { if let Some(id) = identities.get(dependency) { runtime.borrow_mut().stats.memo_hits += 1; + tracing::debug!(target: "asap_runtime_debug", query_id = %entry.query_id, node_id = ?dependency.0, evaluation_ms = dependency.1, "installed query node reused within request"); input_ids.push(*id); } else { if identities.len() >= 200_000 { @@ -573,6 +574,8 @@ impl Result> })) .await?; let values = values.iter().map(|v| v.value().clone()).collect::>(); + let started = std::time::Instant::now(); + tracing::debug!(target: "asap_runtime_debug", node_id = ?self.id, evaluation_ms = self.time, op = self.node.op_label(), syntax = %self.node.log_syntax(), "installed query node started"); self.runtime .borrow_mut() .execute_node( @@ -583,7 +586,11 @@ impl Result> &self.dependencies, &context, ) + .inspect(|_| { + tracing::debug!(target: "asap_runtime_debug", node_id = ?self.id, op = self.node.op_label(), elapsed_us = started.elapsed().as_micros() as u64, "installed query node completed"); + }) .map_err(|error| { + tracing::warn!(node_id = ?self.id, op = self.node.op_label(), elapsed_us = started.elapsed().as_micros() as u64, %error, "installed query node failed"); *self.error.borrow_mut() = Some(error); physical::Error::Operator(format!( "query node {} at {} failed", diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 6682903fe..ed17fbe26 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -974,11 +974,14 @@ impl SketchStore { self.install_catalog_outputs(catalog, outputs) } + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(plan_id = catalog.plan_id, plan_version = catalog.plan_version))] fn install_catalog_outputs( &self, catalog: Arc, outputs: BTreeMap, ) -> Result<(), String> { + tracing::debug!(target: "asap_runtime_debug", "summary store catalog installation started"); for (definition, output) in &outputs { output.validate().map_err(|e| e.to_string())?; if output.definition_id != *definition || !catalog.definitions.contains_key(definition) @@ -1068,12 +1071,24 @@ impl SketchStore { // producer cannot mutate a new generation before its receipt is rejected. let mut inventory = self.admission.write().unwrap(); if inventory.validate_publication(generation, coordinate, first_revision, revision)? { + tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, + plan_version = generation.plan_version, + summary_definition = coordinate.summary_definition_id.as_u64(), + window_start_ms = coordinate.time_range.start_ms, + window_end_ms = coordinate.time_range.end_ms, revision, + "SDS summary publication replay acknowledged"); return Ok(()); } let series_id = persist(&SummaryPublicationWriter(self)).ok_or("summary state publication failed")?; inventory.record_series(generation, coordinate, series_id)?; inventory.acknowledge(generation, coordinate, revision)?; + tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, + plan_version = generation.plan_version, sid = series_id, + summary_definition = coordinate.summary_definition_id.as_u64(), + window_start_ms = coordinate.time_range.start_ms, + window_end_ms = coordinate.time_range.end_ms, revision, + "SDS summary publication acknowledged"); self.admitted_mutations .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let floor = coordinate @@ -1758,6 +1773,7 @@ impl SketchStore { start_unix_ms: u64, end_unix_ms: u64, ) -> Vec { + let started = std::time::Instant::now(); if self .incomplete_summary_lineages .get(&sid) @@ -1768,6 +1784,8 @@ impl SketchStore { }) }) { + tracing::debug!(target: "asap_runtime_debug", sid, start_unix_ms, end_unix_ms, + reason = "incomplete_summary_lineage", "SketchStore sketch range read skipped"); return Vec::new(); } // Result is keyed by the resolved label MAP so the in-memory tier @@ -1961,14 +1979,18 @@ impl SketchStore { // state; the guard is belt-and-suspenders). self.union_disk_parts_into(sid, start_unix_ms, end_unix_ms, &mut by_label_map); - by_label_map + let result: Vec<_> = by_label_map .into_iter() .map(|(label_values, samples)| SketchTimeSeries { sid, series_label_values: label_values, samples, }) - .collect() + .collect(); + tracing::debug!(target: "asap_runtime_debug", sid, start_unix_ms, end_unix_ms, + series_count = result.len(), elapsed_us = started.elapsed().as_micros() as u64, + "SketchStore sketch range read completed"); + result } /// Resolve the sorted group-by KEYS for a sid from its instance @@ -2203,6 +2225,7 @@ impl SketchStore { BTreeMap, BTreeMap>, )> { + let started = std::time::Instant::now(); // Key by the resolved label MAP (not `LabelValuesId`) so the // in-memory tier and the durable disk tier — which carry // independent intern spaces — union by label identity. Mirrors @@ -2263,7 +2286,11 @@ impl SketchStore { // of the range. In-memory wins on a window-end collision. self.union_disk_exact_agg_into(sid, start_unix_ms, end_unix_ms, &mut by_label_map); - by_label_map.into_iter().collect() + let result: Vec<_> = by_label_map.into_iter().collect(); + tracing::debug!(target: "asap_runtime_debug", sid, start_unix_ms, end_unix_ms, + series_count = result.len(), elapsed_us = started.elapsed().as_micros() as u64, + "SketchStore exact range read completed"); + result } /// Union the durable disk tier's exact-aggregation entries into @@ -3387,6 +3414,13 @@ impl SketchStore { accumulator.clone_boxed_core(), ), }; + tracing::debug!(target: "asap_runtime_debug", sid, + policy_fp = %output.policy_fp, + aggregation_type = ?agg_cfg.aggregation_type, + window_start_ms = output.start_timestamp, + window_end_ms = output.end_timestamp, + accepted, + "SketchStore precompute window append completed"); accepted.then_some(sid) } } diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 99f023a7e..7d05d14eb 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -143,8 +143,14 @@ impl SummaryDescriptorRegistry { asap_types::sds::StoredOutputReference, >, ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { - catalog.validate()?; - let reference = catalog.reference()?; + catalog.validate().inspect_err(|error| { + tracing::warn!(plan_id = catalog.plan_id, plan_version = catalog.plan_version, + %error, "storage descriptor catalog validation failed"); + })?; + let reference = catalog.reference().inspect_err(|error| { + tracing::warn!(plan_id = catalog.plan_id, plan_version = catalog.plan_version, + %error, "storage descriptor catalog reference failed"); + })?; let generation = Arc::new(asap_types::sds::CatalogGeneration { schema_version: reference.schema_version, plan_id: reference.plan_id, @@ -152,6 +158,11 @@ impl SummaryDescriptorRegistry { snapshot_sha256: reference.snapshot_sha256, }); *self.authoritative_catalog.write().unwrap() = Some((catalog, generation, outputs)); + tracing::info!( + plan_id = reference.plan_id, + plan_version = reference.plan_version, + "storage descriptor catalog installed" + ); Ok(()) } @@ -257,6 +268,16 @@ impl SummaryDescriptorRegistry { } }; + let catalog_generation = authoritative + .as_ref() + .map(|(_, generation, _)| generation.clone()); + tracing::debug!(target: "asap_runtime_debug", sid = metadata.storage_handle, + policy_fp = %metadata.policy_fp, + summary_descriptor_hash = format_args!("{:016x}", xxhash_rust::xxh64::xxh64(summary_descriptor.id().canonical().as_bytes(), 0)), + data_descriptor_hash = format_args!("{:016x}", xxhash_rust::xxh64::xxh64(data_id.canonical().as_bytes(), 0)), + plan_id = catalog_generation.as_ref().map(|g| g.plan_id), + plan_version = catalog_generation.as_ref().map(|g| g.plan_version), + "SDS descriptor binding resolved"); Ok(SdsBinding { stored_output_reference, metadata: Arc::new(metadata), diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md new file mode 100644 index 000000000..29384fcdc --- /dev/null +++ b/docs/runtime-debugging.md @@ -0,0 +1,9 @@ +# Runtime debugging logs + +Set `RUST_LOG=info,asap_runtime_debug=debug` before starting both controller and backend to enable workflow debug events while keeping other targets at `info`. Set `RUST_LOG=info` to hide debug events. Both processes use `RUST_LOG` and default to `info` when it is unset or invalid. Their console streams use the same `tracing_subscriber` text layout with timestamp, level, target, source file and line, span context, and event fields. The controller writes to stdout. The backend writes to stdout and `/query_engine.log`; the file layer disables terminal colors. + +Filter both streams by `plan_id` and `plan_version`. A `call_id` distinguishes concurrent plan requests or query calls within one process; it is local to that process and is not shared between controller and backend. The controller reports compilation, collector preflight and publication, backend staging, and activation. The backend reports staging validation, activation, and summary catalog installation. At `debug`, the query engine reports call start and completion or failure with elapsed time, the selected installed query DAG, and each evaluated node with its `node_id`, operator label `op` (for example `logical/aggregate/sum` or `exact_readout/count`), and inclusive duration. Node start events also include `inputs` and `syntax`: bounded operator arguments such as `operation=Sum grouping=by(service)`, `operation=Div return_bool=false`, or window parameters. The syntax field omits complete query strings, matchers, and samples. Prepared leaves and memo hits have separate events with the node type in `op`. When input preparation fails, the backend logs the node that failed catalog validation or the exact leaf that could not be prepared, with its `node_id` and `op`; a failing DAG logs the first failing node rather than repeating its error at every ancestor. Instant-query completion also includes remote evaluation and RPC counts. Failed query calls that the ASAP tier cannot serve are logged at `debug`; backend failures and failed plan operations are logged at `warn` or `error`. Error messages can contain query text, so protect access to these logs accordingly. + +Events share a rendered layout, but their fields depend on the operation. Plan events use `plan_id` and `plan_version`; query events add `query_id`, `call_id`, and elapsed time when available; node events add `node_id` and `op`. Inner spans cover compiler selection and transmission construction, backend publication and validation, worker processing, summary catalog installation, and installed query preparation and execution. They carry identifiers and counts without dumping request bodies or samples. Existing messages elsewhere in the system still include free-form text. These logs are text output, not a JSON schema or a distributed trace ID. + +For deeper precompute and storage work, use `RUST_LOG=info,asap_runtime_debug=debug,data_plane::precompute_engine=debug,data_plane::storage_engines=debug` on the backend. The precompute DAG span carries plan ID, version, sink node, summary definition, and window bounds. Node events add the operator type, bounded syntax arguments, input node IDs, and duration; a reused materialization or committed sink has a separate event. Worker processing logs the aggregation type and window layout. SketchStore logs accepted or rejected window appends and range read counts. SDS logs descriptor bindings by a short hash of each canonical descriptor ID, and admitted publication receipts with their catalog generation and summary definition. The new event fields omit raw samples and full fallback expressions; existing worker spans may include group label values. An error before backend staging points to plan compilation or publication; an error after activation with a matching plan identifier points to runtime query or storage behavior. From e901822ab1a5f281b1d062db3c04f972a82fdd0d Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 03:30:49 +0000 Subject: [PATCH 03/45] fix: reject unavailable external execution before workload pricing --- control_plane/src/physical/compiler.rs | 24 +++++++++++ control_plane/src/physical/workload_cost.rs | 45 +++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 934da0b8c..6700c6a36 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -158,6 +158,8 @@ pub struct PhysicalCompilationRequest { pub planner_selection_trace: Vec, /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. pub allow_mixed_summary_and_exact_execution: bool, + /// Deployment feasibility: external exact dependencies cannot be bound. + pub require_backend_local_execution: bool, /// Enabled optional candidate keys: None enables all eligible keys; an /// explicitly empty set enables none. These are not catalog definition IDs. pub enabled_materialization_keys: Option>, @@ -370,6 +372,9 @@ pub struct BackendLocalPlanningInput { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct BackendLocalPhysicalInputs { + /// Constrain selection to local execution when no exact upstream is available. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub require_backend_local_execution: bool, pub lifecycle_costs: LifecycleUnitCosts, pub evidence_observed_at_unix_ms: u64, pub evidence_valid_for_ms: u64, @@ -949,6 +954,9 @@ impl BackendLocalPlanningInput { PhysicalCompilationRequest { planner_selection_trace, allow_mixed_summary_and_exact_execution: true, + require_backend_local_execution: self + .physical_inputs + .require_backend_local_execution, enabled_materialization_keys: None, query_workload: Some(workload), data_workload: Some(data_workload), @@ -2252,6 +2260,20 @@ impl DeploymentPlanCompiler { reason: error.to_string(), })?; } + if request.require_backend_local_execution { + for entry in query_plan.entries.values() { + if entry.nodes.values().any(|node| matches!(node, + crate::query_plan::QueryPlanNode::ExactFallback { .. } + | crate::query_plan::QueryPlanNode::Logical { + operator: crate::query_plan::residual::ResidualQueryOperator::ExactSubquery { .. } + | crate::query_plan::residual::ResidualQueryOperator::CandidateExactSubquery { .. }, .. })) { + return Err(CompileError::Query { + query_id: entry.query_id.clone(), + reason: "external execution is unavailable in this deployment".into(), + }); + } + } + } query_plan.validate_against_catalog(&summary_catalog)?; let storage_routing = crate::emit::backend_wire::storage_routing_document( crate::emit::backend_wire::DEFAULT_TENANT, @@ -4901,6 +4923,7 @@ pub(crate) mod tests { planner_selection_trace: Vec::new(), canonical_roots: Vec::new(), allow_mixed_summary_and_exact_execution: false, + require_backend_local_execution: false, enabled_materialization_keys: None, query_workload: None, data_workload: None, @@ -7813,6 +7836,7 @@ pub(crate) mod tests { query_workload, data_workload, physical_inputs: BackendLocalPhysicalInputs { + require_backend_local_execution: false, lifecycle_costs: template.summary_lifecycle_inputs.costs, evidence_observed_at_unix_ms: 9_500, evidence_valid_for_ms: 60_000, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 338dc5b0b..e56fc846e 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -914,6 +914,51 @@ mod tests { use super::super::compiler::BackendLocalPlanningInput; use super::*; + /// A deployment without external execution rejects native candidates before pricing. + #[test] + fn local_only_deployment_rejects_external_candidate_before_pricing() { + let mut value = serde_json::to_value(fixture()).unwrap(); + value["implementation"]["require_backend_local_execution"] = json!(true); + let input: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let native = candidates + .iter() + .find(|candidate| { + candidate.queries.iter().all(|query| { + matches!( + query.selected_plan_root.expr, + planner_types::post_asap::SummaryExpr::KeepPreAsap(_) + ) + }) + }) + .expect("native candidate remains inspectable") + .clone(); + let error = DeploymentPlanCompiler + .compile_promql(native, environment) + .unwrap_err(); + assert!( + error + .to_string() + .contains("external execution is unavailable"), + "{error}" + ); + let selected = input.compile_promql().unwrap(); + assert!(selected.query_plan.entries.values().all(|entry| entry + .nodes + .values() + .all(|node| !matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. })))); + let report = selected.cost_comparison.unwrap(); + assert!(report + .candidate_evaluations + .iter() + .any(|candidate| candidate + .unavailable_reason + .as_ref() + .is_some_and(|reason| reason.contains("external execution is unavailable")) + && candidate.total_cost.is_none())); + } + /// Deployment computes and compares complete costs without external quotes. #[test] fn deployment_automatically_prices_workload() { From 04ea95c19704eb6db8de577a863d1fe393272e62 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 03:42:16 +0000 Subject: [PATCH 04/45] fix: supply local execution policy in planning endpoint --- control_plane/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 303caaef3..5b4356756 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -662,6 +662,7 @@ fn compile_physical_plan_request( queries, allow_mixed_summary_and_exact_execution: request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, + require_backend_local_execution: false, enabled_materialization_keys: None, topk_membership_evidence_by_query_id: request.evidence, exact_composition_costs: request.exact_composition_costs, From 44fe42c733ae96e16cb19a38e44ed200b8d09ce8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 04:02:09 +0000 Subject: [PATCH 05/45] test: cover local current-series count candidates --- control_plane/src/physical/compiler.rs | 1 + control_plane/src/physical/workload_cost.rs | 29 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 6700c6a36..24d03460e 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4101,6 +4101,7 @@ pub(crate) mod tests { "quantile by (job) (0.99, a)", "topk by (job) (1, a)", "topk by (job) (5, a)", + "count by (job) (a)", ]; snapshot.query_workload.repeating_queries = Some( queries diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index e56fc846e..2958399fd 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -959,6 +959,35 @@ mod tests { && candidate.total_cost.is_none())); } + /// Instant counts select current membership, never accumulated observations. + #[test] + fn local_grouped_count_has_a_bindable_candidate() { + let mut input = fixture(); + input.workload_cost_evidence = None; + input.physical_inputs.require_backend_local_execution = true; + input.data_workload.data_ingestion_interval.value = + Some(planner_types::workload::DurationMs(60_000)); + input.physical_inputs.scrape_interval_ms = 60_000; + let queries = input.query_workload.repeating_queries.as_mut().unwrap(); + queries.truncate(1); + queries[0].query = planner_types::workload::Query("count by(job)(m)".into()); + let plan = input.compile_promql().unwrap(); + assert!(plan + .query_plan + .entries + .values() + .all(|entry| entry.nodes.values().any(|node| matches!( + node, + crate::query_plan::QueryPlanNode::Logical { + operator: crate::query_plan::residual::ResidualQueryOperator::CurrentSeries { + readout: asap_types::query_plan::current_series::SeriesReadout::Count, + .. + }, + .. + } + )))); + } + /// Deployment computes and compares complete costs without external quotes. #[test] fn deployment_automatically_prices_workload() { From ff6ab6c4646b0f4d9e394b1cba4a376997d65b57 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 04:04:49 +0000 Subject: [PATCH 06/45] fix: retain population candidates in mixed workloads --- control_plane/src/physical/workload_cost.rs | 34 ++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 2958399fd..a3d191a2b 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -779,11 +779,7 @@ pub fn select_candidates( pub fn enumerate_exact_and_materialized_candidates( request: PhysicalCompilationRequest, ) -> Result, CompileError> { - let already_selected = super::maintained_population::supported(&request); let mut candidates = materialization_candidates(request)?; - if already_selected { - return Ok(candidates); - } let roots: Vec<_> = candidates .last() .expect("exact alternative") @@ -959,6 +955,36 @@ mod tests { && candidate.total_cost.is_none())); } + /// Selecting one population must not suppress candidates for other roots. + #[test] + fn existing_population_does_not_suppress_other_population_candidates() { + let mut input = fixture(); + let queries = input.query_workload.repeating_queries.as_mut().unwrap(); + let template = queries[0].clone(); + *queries = ["quantile by(job)(0.9,m)", "count by(job)(m)"] + .into_iter() + .map(|q| { + let mut entry = template.clone(); + entry.query = planner_types::workload::Query(q.into()); + entry + }) + .collect(); + let (mut request, _) = input.into_physical_compilation_request().unwrap(); + let strategy = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( + &request.canonical_roots, + ); + request.queries[0].selected_plan_root = + strategy.candidate(&request.canonical_roots[0]).unwrap(); + request.queries[1].selected_plan_root = + crate::planner_selection::keep_pre_asap(&request.canonical_roots[1]).unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + assert!(candidates + .iter() + .any(|candidate| candidate.queries.iter().all(|query| { + super::super::maintained_population::supported_node(&query.selected_plan_root) + }))); + } + /// Instant counts select current membership, never accumulated observations. #[test] fn local_grouped_count_has_a_bindable_candidate() { From e8f5686bc21fbd58fa97b887c5740a35a4000f96 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 04:17:58 +0000 Subject: [PATCH 07/45] refactor: delegate candidate winner selection to shared Planner --- control_plane/src/physical/compiler.rs | 42 ++++++++++++++++++++- control_plane/src/physical/workload_cost.rs | 39 ++++++++++++------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 24d03460e..149a03a0b 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2675,9 +2675,49 @@ pub fn select_post_asap( delete: false, }, ); + // These fixtures exercise collector heap transport. Collector fixtures do + // not offer backend-only temporal value readouts as a physical capability. + struct CollectorFixtureModel(ControlPlaneCostModel); + impl asap_aware_mapping::CostModel for CollectorFixtureModel { + fn rank_candidates( + &self, + intent: &planner_types::pre_asap::AggIntent, + candidates: &[planner_types::post_asap::SketchAlgorithm], + ) -> Vec { + self.0.rank_candidates(intent, candidates) + } + fn size_params( + &self, + kind: planner_types::post_asap::SketchAlgorithm, + intent: &planner_types::pre_asap::AggIntent, + eps: f64, + delta: f64, + ) -> planner_types::post_asap::SketchParams { + self.0.size_params(kind, intent, eps, delta) + } + fn candidate_cost( + &self, + candidate: &asap_aware_mapping::ReplacementSubDAG, + target: &asap_aware_mapping::TargetSubDAG<'_>, + ) -> Option { + self.0.candidate_cost(candidate, target) + } + fn summary_support_evidence(&self, summary: &SummaryNode) -> Option { + if matches!( + summary.expr, + SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::Limit { .. }, + .. + } + ) { + return Some(false); + } + self.0.summary_support_evidence(summary) + } + } crate::planner_selection::select_query_with_models( expr, - &model, + &CollectorFixtureModel(model), &DefaultAccuracyModel, &QueryEvidence { topk: evidence, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index a3d191a2b..8a7848717 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -670,13 +670,7 @@ pub fn select_candidates( let mut comparison_workload = None; let mut comparison_inputs = None; let mut candidate_evaluations = Vec::new(); - let mut best_index = 0; - let mut best: Option<( - Cost, - CompiledPhysicalPlan, - WorkloadCostManifest, - BTreeMap, - )> = None; + let mut priced_candidates = Vec::new(); for candidate in candidates { if evidence.is_none() { let inputs = json!({"data":candidate.data_workload,"erp":candidate.erp}); @@ -737,10 +731,13 @@ pub fn select_candidates( description.status = CandidateEvaluationStatus::Unselected; description.total_cost = Some(cost.0); candidate_evaluations.push(description); - if best.as_ref().is_none_or(|(previous, ..)| cost < *previous) { - best_index = candidate_evaluations.len() - 1; - best = Some((cost, plan, manifest, components)); - } + priced_candidates.push(Ok(( + cost, + plan, + manifest, + components, + candidate_evaluations.len() - 1, + ))); } Err((status, reason)) => { description.status = status; @@ -749,13 +746,27 @@ pub fn select_candidates( } } } - let (_, mut plan, selected_manifest, component_costs) = best.ok_or_else(|| { + let selected = asap_physical_operators::physical_planner::select_candidate( + priced_candidates, + |(cost, _, manifest, _, _)| { + Ok(Some( + asap_physical_operators::physical_planner::CandidateCost { + workload_scope: serde_json::to_string(&manifest.workload).map_err(|error| { + asap_physical_operators::Error::Invalid(error.to_string()) + })?, + horizon_seconds: manifest.horizon_seconds, + total_cost: cost.0, + }, + )) + }, + ) + .map_err(|error| { CompileError::Candidates( json!({"status": "all_infeasible", "logical_selection": planner_selection_trace, - "candidates": candidate_evaluations}), + "candidates": candidate_evaluations, "selection_error": error.to_string()}), ) })?; - // Exactly the winner retained by the existing strict-less-than selector. + let (_, mut plan, selected_manifest, component_costs, best_index) = selected.candidate; candidate_evaluations[best_index].status = CandidateEvaluationStatus::Selected; plan.cost_comparison = Some(CandidatePlanSelectionReport { planner_selection_trace, From 36b6176dd61b1e19ae4b931d78d424e13e43f4ae Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 05:15:37 +0000 Subject: [PATCH 08/45] Use scoped fixture evidence after the accuracy contract extension --- control_plane/src/physical/compiler.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 17a50cf4e..149a03a0b 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2719,7 +2719,11 @@ pub fn select_post_asap( expr, &CollectorFixtureModel(model), &DefaultAccuracyModel, - &QueryEvidence(evidence), + &QueryEvidence { + topk: evidence, + scoped: None, + now_ms: 0, + }, ) } From 395f026055e3496e8914bfb744fd011ab2614388 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 17:33:25 +0000 Subject: [PATCH 09/45] fix(costing): use deployed output routing identity after SDS split --- .../src/physical/workload_cost/automatic.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index 127b90d18..7257bc9be 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -221,7 +221,7 @@ pub(super) fn estimate( .find(|m| m.policy_fingerprint() == schema.materialization.fingerprint()) .ok_or_else(|| invalid("missing materialization"))?; states.insert( - schema.materialization.0, + schema.materialization.fingerprint(), state(m, request, cardinality as u64)?, ); } @@ -379,7 +379,7 @@ pub(super) fn estimate( return Err(invalid("unknown transmission cadence")); } let s = states - .get(&rule.materialization.0) + .get(&rule.materialization.fingerprint()) .ok_or_else(|| invalid("unknown transport state"))?; let checkpoints = rule .full_checkpoint_every_ms @@ -396,7 +396,11 @@ pub(super) fn estimate( * s.retained; let bytes = frames * s.profile.memory_bytes; insert( - format!("transport:{}:{}", rule.producer_id, rule.materialization.0), + format!( + "transport:{}:{}", + rule.producer_id, + rule.materialization.fingerprint() + ), resources( bytes * CPU_PER_BYTE * 2.0 + frames * s.profile.merge_cpu_seconds, 0.0, @@ -429,7 +433,7 @@ pub(super) fn estimate( let cpu = match node { Node::ReadMaterialization { binding } => { let s = states - .get(&binding.materialization.0) + .get(&binding.materialization.fingerprint()) .ok_or_else(|| invalid("unknown read state"))?; let panes = if binding.full_window_slide_ms.is_some() { 1.0 From 2d83f9fae5ac73bdf0eb37affe8461ee8c4a53b6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 26 Sep 2026 17:36:53 +0000 Subject: [PATCH 10/45] fix(diagnostics): report deployed output identity for SDS coordinates --- data_plane/src/storage_engines/sketch_db/index/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 1c0dc3af1..fa9bab05a 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1087,7 +1087,7 @@ impl SketchStore { if inventory.validate_publication(generation, coordinate, first_revision, revision)? { tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, plan_version = generation.plan_version, - summary_definition = coordinate.summary_definition_id.as_u64(), + stored_output = coordinate.stored_output_id.as_u64(), window_start_ms = coordinate.time_range.start_ms, window_end_ms = coordinate.time_range.end_ms, revision, "SDS summary publication replay acknowledged"); @@ -1099,7 +1099,7 @@ impl SketchStore { inventory.acknowledge(generation, coordinate, revision)?; tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, plan_version = generation.plan_version, sid = series_id, - summary_definition = coordinate.summary_definition_id.as_u64(), + stored_output = coordinate.stored_output_id.as_u64(), window_start_ms = coordinate.time_range.start_ms, window_end_ms = coordinate.time_range.end_ms, revision, "SDS summary publication acknowledged"); From 67082a124119988f5fbfe7410be238b7a4036625 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 12:44:50 +0000 Subject: [PATCH 11/45] feat: trace deployment selection and bound SDS lifecycle --- control_plane/src/physical/compiler.rs | 4 +- control_plane/src/physical/workload_cost.rs | 16 +++++ .../asap_query_engine/summary_executor.rs | 13 ++++ .../storage_engines/sketch_db/index/mod.rs | 12 +++- .../src/storage_engines/sketch_db/sds.rs | 3 + docs/runtime-debugging.md | 72 +++++++++++++++++-- 6 files changed, 111 insertions(+), 9 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index bcc884fc0..889f369eb 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -970,7 +970,7 @@ impl DeploymentPlanCompiler { environment: PhysicalDeploymentContext, frontend: QueryFrontend, ) -> Result { - tracing::debug!(target: "asap_runtime_debug", "physical plan compiler entered"); + tracing::debug!(target: "asap_runtime_debug", stage = "deployment.bind", "backend deployment compiler entered"); if let Some(data) = &request.data_workload { data.validate() .map_err(|error| CompileError::Snapshot(error.to_string()))?; @@ -2205,7 +2205,7 @@ pub fn select_logical_roots_with_trace( exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, ) -> Result, CompileError> { - tracing::debug!(target: "asap_runtime_debug", "logical root selection started"); + tracing::debug!(target: "asap_runtime_debug", stage = "planner.select", "Planner selection entered"); let mut traces = Vec::new(); if roots.len() != queries.len() { return Err(CompileError::Snapshot( diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 963ebccac..03d6f3409 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -707,6 +707,17 @@ fn select_candidates( } } } + tracing::debug!(target: "asap_runtime_debug", stage = "planner.candidate_inventory", + candidate_count = candidate_evaluations.len(), + search_coverage = ?materialization_search_coverage, + "bounded candidate evaluation completed; absence is not a cost rejection"); + for candidate in &candidate_evaluations { + tracing::debug!(target: "asap_runtime_debug", stage = "planner.candidate_evaluation", + candidate_id = ?candidate.candidate_id, + physical_candidate_id = ?candidate.physical_candidate_id, + status = ?candidate.status, total_cost = candidate.total_cost, + reason = ?candidate.unavailable_reason, "candidate evaluated"); + } let (_, mut plan, selected_manifest, component_costs) = best.ok_or_else(|| { CompileError::Candidates( json!({"status": "all_infeasible", "logical_selection": planner_selection_trace, @@ -715,6 +726,11 @@ fn select_candidates( })?; // Exactly the winner retained by the existing strict-less-than selector. candidate_evaluations[best_index].status = CandidateEvaluationStatus::Selected; + tracing::debug!(target: "asap_runtime_debug", stage = "planner.candidate_selected", + plan_id = plan.envelope.plan_id, plan_version = plan.envelope.plan_version, + candidate_id = ?candidate_evaluations[best_index].candidate_id, + total_cost = candidate_evaluations[best_index].total_cost, + "lowest quoted cost selected within the admitted inventory"); plan.cost_comparison = Some(CandidatePlanSelectionReport { planner_selection_trace, materialization_search_coverage, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 0511e1aad..f0f65c424 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -237,6 +237,11 @@ impl QueryExecutionContext<'_> { /// Resolve exactly one compiler-bound materialization. This is the formal /// QueryPlan path: the validated stored output resolves to its definition's /// generation-scoped SID index. Metadata checks never broaden that set. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "sds.bound_read", + stored_output_id = binding.stored_output_reference.stored_output_id.as_u64(), + definition_id = ?binding.stored_output_reference.definition_id, + window_start_ms = self.t0_ms, window_end_ms = self.t1_ms), err(Debug, level = "debug"))] pub fn read_bound_materialization( &self, binding: &asap_types::query_plan::MaterializationBinding, @@ -275,6 +280,9 @@ impl QueryExecutionContext<'_> { )); } + tracing::debug!(target: "asap_runtime_debug", stage = "sds.validate_binding", + plan_id = catalog.plan_id, plan_version = catalog.plan_version, + "installed output and semantic identity validated"); let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { start_ms: i64::try_from(self.t0_ms).map_err(|_| { @@ -342,6 +350,8 @@ impl QueryExecutionContext<'_> { .storage_handles_for_output(&binding.stored_output_reference); sids.sort_unstable(); sids.dedup(); + tracing::debug!(target: "asap_runtime_debug", stage = "sds.locate_records", + storage_handle_count = sids.len(), "bound output lookup completed"); let mut matched_metadata = 0usize; let mut by_group: BTreeMap, Vec> = BTreeMap::new(); let mut source_order = BTreeMap::new(); @@ -545,6 +555,9 @@ impl QueryExecutionContext<'_> { "summary input changed during read", )); } + tracing::debug!(target: "asap_runtime_debug", stage = "sds.validate_records", + group_count = result.len(), + "state format, applicable coverage and stable revision checks completed"); Ok(result) } diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index fa9bab05a..a50ae0443 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1087,7 +1087,7 @@ impl SketchStore { if inventory.validate_publication(generation, coordinate, first_revision, revision)? { tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, plan_version = generation.plan_version, - stored_output = coordinate.stored_output_id.as_u64(), + stage = "sds.publish", stored_output_id = coordinate.stored_output_id.as_u64(), window_start_ms = coordinate.time_range.start_ms, window_end_ms = coordinate.time_range.end_ms, revision, "SDS summary publication replay acknowledged"); @@ -1099,7 +1099,7 @@ impl SketchStore { inventory.acknowledge(generation, coordinate, revision)?; tracing::debug!(target: "asap_runtime_debug", plan_id = generation.plan_id, plan_version = generation.plan_version, sid = series_id, - stored_output = coordinate.stored_output_id.as_u64(), + stage = "sds.publish", stored_output_id = coordinate.stored_output_id.as_u64(), window_start_ms = coordinate.time_range.start_ms, window_end_ms = coordinate.time_range.end_ms, revision, "SDS summary publication acknowledged"); @@ -3599,6 +3599,12 @@ impl SketchStore { /// missing only for parts written before this feature landed (or a /// fresh dir) — those sids stay invisible until a live DataPoint /// re-registers them, the same as pre-fix behavior. + #[tracing::instrument( + level = "debug", + target = "asap_runtime_debug", + skip_all, + fields(stage = "sds.recover") + )] pub fn register_recovered_disk_series(&self, disk_path: &std::path::Path) -> usize { use crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataFile; @@ -3681,6 +3687,8 @@ impl SketchStore { registered += 1; } } + tracing::debug!(target: "asap_runtime_debug", registered, + "durable identities validated and eligible storage handles restored"); registered } diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 114008c23..911c4dbf3 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -320,6 +320,9 @@ impl SummaryDescriptorRegistry { .as_ref() .map(|(_, generation, _)| generation.clone()); tracing::debug!(target: "asap_runtime_debug", sid = metadata.storage_handle, + stage = "sds.bind_storage_handle", + stored_output_id = stored_output_reference.as_ref().map(|r| r.stored_output_id.as_u64()), + definition_id = ?stored_output_reference.as_ref().map(|r| &r.definition_id), policy_fp = %metadata.policy_fp, summary_descriptor_hash = format_args!("{:016x}", xxhash_rust::xxh64::xxh64(summary_descriptor.id().canonical().as_bytes(), 0)), data_descriptor_hash = format_args!("{:016x}", xxhash_rust::xxh64::xxh64(data_id.canonical().as_bytes(), 0)), diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 29384fcdc..4fc6aedc8 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -1,9 +1,71 @@ -# Runtime debugging logs +# Runtime diagnostics along the execution path -Set `RUST_LOG=info,asap_runtime_debug=debug` before starting both controller and backend to enable workflow debug events while keeping other targets at `info`. Set `RUST_LOG=info` to hide debug events. Both processes use `RUST_LOG` and default to `info` when it is unset or invalid. Their console streams use the same `tracing_subscriber` text layout with timestamp, level, target, source file and line, span context, and event fields. The controller writes to stdout. The backend writes to stdout and `/query_engine.log`; the file layer disables terminal colors. +Audience: developers tracing an installed plan from selection to a result. +The [architecture](design_docs/asapplanner-integration.md) and +[SDS contract](design_docs/summary-catalog-sds-architecture.md) define ownership; +diagnostics observe these decisions and never select replacement computations. -Filter both streams by `plan_id` and `plan_version`. A `call_id` distinguishes concurrent plan requests or query calls within one process; it is local to that process and is not shared between controller and backend. The controller reports compilation, collector preflight and publication, backend staging, and activation. The backend reports staging validation, activation, and summary catalog installation. At `debug`, the query engine reports call start and completion or failure with elapsed time, the selected installed query DAG, and each evaluated node with its `node_id`, operator label `op` (for example `logical/aggregate/sum` or `exact_readout/count`), and inclusive duration. Node start events also include `inputs` and `syntax`: bounded operator arguments such as `operation=Sum grouping=by(service)`, `operation=Div return_bool=false`, or window parameters. The syntax field omits complete query strings, matchers, and samples. Prepared leaves and memo hits have separate events with the node type in `op`. When input preparation fails, the backend logs the node that failed catalog validation or the exact leaf that could not be prepared, with its `node_id` and `op`; a failing DAG logs the first failing node rather than repeating its error at every ancestor. Instant-query completion also includes remote evaluation and RPC counts. Failed query calls that the ASAP tier cannot serve are logged at `debug`; backend failures and failed plan operations are logged at `warn` or `error`. Error messages can contain query text, so protect access to these logs accordingly. +## Enable and correlate -Events share a rendered layout, but their fields depend on the operation. Plan events use `plan_id` and `plan_version`; query events add `query_id`, `call_id`, and elapsed time when available; node events add `node_id` and `op`. Inner spans cover compiler selection and transmission construction, backend publication and validation, worker processing, summary catalog installation, and installed query preparation and execution. They carry identifiers and counts without dumping request bodies or samples. Existing messages elsewhere in the system still include free-form text. These logs are text output, not a JSON schema or a distributed trace ID. +Set `RUST_LOG=info,asap_runtime_debug=debug` on controller and backend. Use +`RUST_LOG=info` to hide these events. Logs contain timestamps, target, file/line, +span context and event fields. Controller output goes to stdout; backend output +also goes to `/query_engine.log` without terminal colors. -For deeper precompute and storage work, use `RUST_LOG=info,asap_runtime_debug=debug,data_plane::precompute_engine=debug,data_plane::storage_engines=debug` on the backend. The precompute DAG span carries plan ID, version, sink node, summary definition, and window bounds. Node events add the operator type, bounded syntax arguments, input node IDs, and duration; a reused materialization or committed sink has a separate event. Worker processing logs the aggregation type and window layout. SketchStore logs accepted or rejected window appends and range read counts. SDS logs descriptor bindings by a short hash of each canonical descriptor ID, and admitted publication receipts with their catalog generation and summary definition. The new event fields omit raw samples and full fallback expressions; existing worker spans may include group label values. An error before backend staging points to plan compilation or publication; an error after activation with a matching plan identifier points to runtime query or storage behavior. +Correlate by `plan_id` and `plan_version`, then `query_id` and the process-local +`call_id`. A call ID is not a distributed trace ID. For state reads and writes, +use `stored_output_id` for the deployed producer and `definition_id` for its +meaning. A `sid`/storage handle is only a local row locator. Descriptor hashes +are diagnostics, not semantic IDs or authorization tokens. + +## Follow the architecture in order + +| Step | Owner and operation | Diagnostic evidence | +| --- | --- | --- | +| 1 | Planner selects the Logical Post-ASAP DAG and maintenance requirements | `planner.select` entry and selection errors. `planner.candidate_inventory`, `planner.candidate_evaluation` and `planner.candidate_selected` distinguish compilation/pricing failures from selection within the bounded inventory. Backend spans bracket Planner calls; they do not instrument every internal Planner optimization. | +| 2 | Planner compiles physical operators, dependencies and typed boundaries | Inspect the selected DAG artifact alongside compilation spans. Do not interpret a backend adapter node as a second Planner physical node. | +| 3 | Backend Deployment Plan Compiler binds sources, stored outputs and deployment policy | `deployment.bind`, transmission construction and publication spans; plan generation and query counts. The backend does not reselect operators. | +| 4 | Backend validates, stages and activates one generation | HTTP staging/activation and summary-catalog installation events. Activation does not imply state readiness. | +| 5 | Precompute engine resolves inputs and invokes the shared executor | Worker and precompute DAG spans: node/dependency IDs, execution timing, window, sink and inclusive node duration. Reused inputs and already-committed sinks have separate events. | +| 6 | SummaryStore admits and publishes state | `sds.bind_storage_handle` includes both bound identities; `sds.publish` includes output, plan generation, window and revision. Replay acknowledgment is distinguished from a new publication. | +| 7 | Query engine selects the installed QueryPlan | Query call/preparation events and the selected query DAG. A bound query does not search for semantically similar outputs. | +| 8 | Bound read validates identity, locates records and validates eligibility | `sds.bound_read` carries output ID, definition ID and requested range. `sds.validate_binding` checks the installed reference; `sds.locate_records` reports candidate storage handles; `sds.validate_records` follows format, applicable coverage and stable-revision checks. Failures terminate the read span. | +| 9 | Shared operators compute the result; backend serves or applies installed fallback | Node start/completion/failure, memo hits, query completion and remote/RPC counts. Match fallback evidence to the request rather than assuming a successful response ran locally. | +| 10 | SummaryStore recovers persisted state on restart | `sds.recover` brackets metadata replay and reports eligible restored handles. Existing warnings distinguish provenance mismatch, missing identity and unsupported state. A successful disk read alone does not establish semantic eligibility. | + +Step 8 follows the actual implementation: it validates the installed reference +before lookup, then validates concrete state. Lookup is scoped to the selected +output; equal definition IDs never authorize switching to another deployed +output. Coverage rules depend on the selected state family: additive panes need +contiguous coverage, while supported counter state carries sample endpoints. +No diagnostic introduces a universal no-gap rule for every operator. + +## Timing and node identity + +Node durations are inclusive. Parent and child intervals can overlap and shared +producers can serve multiple consumers, so do not sum them as CPU time. These +logs do not provide exclusive kernel, allocation or lock timing. Pair them with +#766's profiling workflow for overhead attribution. + +Backend QueryPlan adapter IDs and Planner DAG IDs occupy different namespaces. +The installed bindings connect their sinks; retain the plan artifact when +following an edge across that boundary. Build, merge and readout are operators, +not fixed precompute/query phases. Interpret placement from the selected DAG. + +## Detail and limitations + +For storage and worker details, add +`data_plane::precompute_engine=debug,data_plane::storage_engines=debug`. +The diagnostic target omits samples, complete query text and full fallback +expressions from its new structured fields. Existing errors or worker fields can +still contain query text or group labels. Logs are text events, not a versioned +JSON API. Missing a completion event can indicate an interrupted process; it +is not proof that work committed. Publication receipts and recovered state remain +the source of truth. + +A missing candidate is not evidence that its placement costs more. For example, +the retained #728 grouped-rate fixture exports per-series Rate state plus +query-time Sum and an exact fallback; it does not export a precomputed grouped +Rate-result candidate. The selected plan therefore cannot establish that query-time +Sum beats that absent alternative. Search coverage is scoped to the declared +inventory, never a claim of exhaustive physical optimization. From 421132267db40fda02e8bff06a851703db490cd3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 20:16:05 +0000 Subject: [PATCH 12/45] build: adopt recoverable Planner physical candidates --- Cargo.lock | 14 +++++++------- Cargo.toml | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be01b30a3..141eb841c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c27cd14b8e052ce1f4641c619488ad539ad71f56#c27cd14b8e052ce1f4641c619488ad539ad71f56" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index 1172c0a36..5cf1a8d86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c27cd14b8e052ce1f4641c619488ad539ad71f56" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } From c36097b6d471fcfe65b0bf4dc155441c8c435acd Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 20:22:44 +0000 Subject: [PATCH 13/45] fix: retain root candidates for deployment costing without Cartesian expansion --- control_plane/src/main.rs | 1 + control_plane/src/physical/compiler.rs | 146 +++++++++++++++++--- control_plane/src/physical/workload_cost.rs | 130 +++++++++++++---- control_plane/src/planner_selection.rs | 38 +++++ 4 files changed, 266 insertions(+), 49 deletions(-) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 5b4356756..8655f6a8d 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -655,6 +655,7 @@ fn compile_physical_plan_request( .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; } let compilation_request = physical::compiler::PhysicalCompilationRequest { + planner_candidate_forests: Vec::new(), planner_selection_trace, query_workload: Some(query_workload), data_workload: Some(request.data_workload), diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 53a232af2..a183b4305 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -154,6 +154,9 @@ pub struct SummaryLifecyclePlanningInputs { #[derive(Debug, Clone, Default)] pub struct PhysicalCompilationRequest { + /// Complete Planner candidates retained until deployment admission/pricing. + /// Empty for callers that explicitly supply one already-selected forest. + pub planner_candidate_forests: Vec>, /// Diagnostic projections of the original Planner search; never consumed by selection. pub planner_selection_trace: Vec, /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. @@ -167,7 +170,7 @@ pub struct PhysicalCompilationRequest { pub query_workload: Option, /// Source evidence supplied independently from query demand. pub data_workload: Option, - /// Workload-lowered roots retained across physical alternative enumeration. + /// Workload-lowered roots retained across physical candidate enumeration. pub canonical_roots: Vec>, pub queries: Vec, pub topk_membership_evidence_by_query_id: HashMap, @@ -625,7 +628,7 @@ pub struct MaterializationLifecycleEstimate { pub enum CompileError { #[error("invalid backend-local workload snapshot: {0}")] Snapshot(String), - #[error("no feasible completely costed alternative: {0}")] + #[error("no feasible completely costed candidate: {0}")] Candidates(serde_json::Value), #[error("planner revision mismatch: request={request}, compiler={compiler}")] PlannerRevision { @@ -932,7 +935,8 @@ impl BackendLocalPlanningInput { exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); } } - let planner_selection_trace = select_logical_roots_with_scoped_evidence_and_trace( + let mut candidate_roots = Vec::new(); + let mut planner_selection_trace = logical_roots_and_candidates( &mut queries, canonical_roots.clone(), &topk_evidence_by_id, @@ -940,6 +944,7 @@ impl BackendLocalPlanningInput { &exact_costs_by_id, self.physical_inputs.erp.as_ref(), self.environment.observed_at_unix_ms, + Some(&mut candidate_roots), )?; for query in &mut queries { prepare_window_implementations( @@ -949,9 +954,36 @@ impl BackendLocalPlanningInput { self.physical_inputs.query_retention_margin_ms, )?; } + let mut planner_candidate_forests = Vec::new(); + for roots in candidate_roots { + let mut forest = queries.clone(); + for (index, root) in roots { + forest[index].selected_plan_root = root; + } + let preparation = forest.iter_mut().try_for_each(|query| { + prepare_window_implementations( + query, + &self.physical_inputs.window_cost_model, + self.environment.target, + self.physical_inputs.query_retention_margin_ms, + ) + }); + if let Err(error) = preparation { + planner_selection_trace.push(serde_json::json!({ + "stage": "deployment.window_feasibility", + "status": "rejected", + "logical_root_ids": forest.iter().map(|query| crate::planner_selection::explained_root_id( + &query.selected_plan_root, &query.accuracy_target)).collect::>(), + "reason": error.to_string(), + })); + continue; + } + planner_candidate_forests.push(forest); + } // Composable lowering residualizes unsafe leaves individually; retain Planner siblings. Ok(( PhysicalCompilationRequest { + planner_candidate_forests, planner_selection_trace, allow_mixed_summary_and_exact_execution: true, require_backend_local_execution: self @@ -1370,7 +1402,7 @@ impl DeploymentPlanCompiler { .flatten() }); // Masks enumerate counter/max choices only. Other selected - // summaries remain required by this physical alternative. + // summaries remain required by this physical candidate. key.is_none_or(|key| policy.contains(&key)) }) }) @@ -2462,6 +2494,28 @@ pub fn select_logical_roots_with_scoped_evidence_and_trace( exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, now_ms: u64, +) -> Result, CompileError> { + logical_roots_and_candidates( + queries, + roots, + evidence, + scoped_evidence, + exact_costs, + erp, + now_ms, + None, + ) +} + +fn logical_roots_and_candidates( + queries: &mut [QueryCompilationInput], + roots: Vec>, + evidence: &HashMap, + scoped_evidence: &HashMap, + exact_costs: &HashMap>, + erp: Option<&super::erp::ErpPlanningInput>, + now_ms: u64, + mut candidates_out: Option<&mut Vec)>>>, ) -> Result, CompileError> { let mut traces = Vec::new(); if roots.len() != queries.len() { @@ -2548,6 +2602,27 @@ pub fn select_logical_roots_with_scoped_evidence_and_trace( AccuracyTarget::Exact => 0.0, }, }; + let mut candidate_assembly_rejections = Vec::new(); + if let Some(output) = candidates_out.as_deref_mut() { + let inventory = crate::planner_selection::enumerate_workload_candidates( + roots.clone(), + accuracy.clone(), + &model, + &QueryEvidence { + topk: certificate, + scoped: scoped_certificate, + now_ms, + }, + &accuracy_model, + ) + .map_err(|error| CompileError::Snapshot(error.to_string()))?; + candidate_assembly_rejections = inventory.rejected_assemblies; + let candidates = inventory.candidates; + // Retain every root's candidates without multiplying independent + // cohorts. Deployment evaluates each root substitution in the + // preferred workload context; this is not exhaustive joint search. + output.extend(candidates); + } let (selected, mut trace) = crate::planner_selection::select_workload_with_accuracy_model_and_trace( roots, @@ -2561,6 +2636,14 @@ pub fn select_logical_roots_with_scoped_evidence_and_trace( &accuracy_model, ) .map_err(|error| CompileError::Snapshot(error.to_string()))?; + trace["candidate_assembly_rejections"] = serde_json::json!(candidate_assembly_rejections); + if candidates_out.is_some() { + trace["computation_search_scope"] = serde_json::json!({ + "inventory": "all_root_candidates", + "deployment_evaluation": "single_root_substitutions_in_preferred_workload", + "joint_workload_search_exhaustive": false, + }); + } if let Some(evidence) = scoped_certificate { trace["accuracy_evidence_scope"] = serde_json::json!({ "query_id": scope, @@ -2598,7 +2681,7 @@ pub fn select_logical_roots_with_scoped_evidence_and_trace( Ok(traces) } -fn requires_exact_erp_fallback( +pub(super) fn requires_exact_erp_fallback( node: &SummaryNode, accuracy: &AccuracyTarget, erp: &super::erp::ErpPlanningInput, @@ -3395,12 +3478,12 @@ fn select_lifecycle( lifecycle_cost: plan.deployments[0] .alternatives .iter() - .find(|alternative| { - alternative.rejection.is_none() - && alternative.summary_maintenance_lifecycle + .find(|candidate| { + candidate.rejection.is_none() + && candidate.summary_maintenance_lifecycle == guarantee.summary_maintenance_lifecycle }) - .and_then(|alternative| alternative.total_cost) + .and_then(|candidate| candidate.total_cost) .ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), reason: "missing selected lifecycle cost".into(), @@ -4739,9 +4822,10 @@ pub(crate) mod tests { ); } - // Capability normalization precedes candidate enumeration, avoiding duplicate exact quotes. + // A larger computation inventory must retain both maintained and native + // execution; its cardinality is not the correctness contract. #[test] - fn counter_only_snapshot_has_distinct_local_and_native_cost_alternatives() { + fn counter_only_snapshot_has_distinct_local_and_native_cost_candidates() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -4749,13 +4833,28 @@ pub(crate) mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("rate(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert_eq!( + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .len(), - 3 + .unwrap(); + assert_eq!( + candidates + .iter() + .filter(|candidate| !candidate.allow_mixed_summary_and_exact_execution) + .count(), + 1 ); + let plans = candidates + .into_iter() + .map(|candidate| DeploymentPlanCompiler.compile_promql(candidate, environment.clone())) + .collect::, _>>() + .unwrap(); + assert!(plans + .iter() + .any(|plan| !plan.precompute_plan.materializations.is_empty())); + assert!(plans + .iter() + .any(|plan| plan.precompute_plan.materializations.is_empty())); } // Count and value rankings must configure different state update contracts. @@ -4991,6 +5090,7 @@ pub(crate) mod tests { evidence_by_query.insert(query_id.to_string(), evidence); } Ok(PhysicalCompilationRequest { + planner_candidate_forests: Vec::new(), planner_selection_trace: Vec::new(), canonical_roots: Vec::new(), allow_mixed_summary_and_exact_execution: false, @@ -6999,9 +7099,9 @@ pub(crate) mod tests { } // A tumbling shape pairs only with `Pane` in the validator's - // framework/layout table, so there is no alternative to price against it. + // framework/layout table, so there is no candidate to price against it. #[test] - fn tumbling_shapes_have_no_layout_alternative_to_rank() { + fn tumbling_shapes_have_no_layout_candidate_to_rank() { let cost = planning_snapshot().physical_inputs.window_cost_model.cost; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", @@ -7436,7 +7536,8 @@ pub(crate) mod tests { request = super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() - .pop() + .into_iter() + .find(|candidate| !candidate.allow_mixed_summary_and_exact_execution) .unwrap(); let bundle = DeploymentPlanCompiler .compile_promql(request, environment) @@ -8045,7 +8146,7 @@ pub(crate) mod tests { } #[test] - fn checked_in_per_entity_snapshot_preserves_native_alternative() { + fn checked_in_per_entity_snapshot_preserves_native_candidate() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict canonical workload fixture"); @@ -8122,8 +8223,9 @@ pub(crate) mod tests { let native = crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() - .pop() - .unwrap(); + .into_iter() + .find(|candidate| !candidate.allow_mixed_summary_and_exact_execution) + .expect("native candidate retained"); let plan = DeploymentPlanCompiler .compile_promql(native, environment) .expect("native demo compiles"); diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 8a7848717..61ad7cfb0 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -489,7 +489,7 @@ fn candidate_description(candidate: &PhysicalCompilationRequest) -> CandidatePla CandidatePlanEvaluation { candidate_id: complete.then(|| { crate::planner_selection::explain_identity( - "alternative", + "candidate", &( &logical_root_ids, candidate.allow_mixed_summary_and_exact_execution, @@ -522,6 +522,19 @@ fn compile_candidate_for_pricing( Box, > { let mut description = candidate_description(&candidate); + if let Some(policy) = &candidate.erp { + if candidate.queries.iter().any(|query| { + super::compiler::requires_exact_erp_fallback( + &query.selected_plan_root, + &query.accuracy_target, + policy, + ) + }) { + description.unavailable_reason = + Some("candidate does not satisfy ERP accuracy admission".into()); + return Err(Box::new(description)); + } + } let queries = candidate.queries.clone(); let compiled = super::realization::RealizationProvider::compile( &super::realization::ExistingRealizations, @@ -644,9 +657,9 @@ pub fn select_candidates( if let Some(evidence) = evidence { evidence.validate(&env)?; } - if candidates.is_empty() || candidates.len() > 64 { + if candidates.is_empty() || candidates.len() > 4096 { return Err(invalid( - "candidate inventory must contain 1..=64 candidates", + "candidate inventory must contain 1..=4096 candidates", )); } let candidate_key_sets: BTreeSet<_> = candidates @@ -788,17 +801,55 @@ pub fn select_candidates( /// The current executor exposes continuously maintained state and the native /// exact backend. Additional Planner-produced forests can use `select_lowest_cost_candidate` directly. pub fn enumerate_exact_and_materialized_candidates( - request: PhysicalCompilationRequest, + mut request: PhysicalCompilationRequest, +) -> Result, CompileError> { + let forests = std::mem::take(&mut request.planner_candidate_forests); + if forests.is_empty() { + return enumerate_frontier_candidates(request); + } + // Keep the existing deterministic ordering for equal-cost candidates; + // every discovered forest still reaches deployment admission and pricing. + let mut candidates: Vec = Vec::new(); + for forest in std::iter::once(request.queries.clone()).chain(forests) { + let mut candidate = request.clone(); + candidate.queries = forest; + for candidate in enumerate_frontier_candidates(candidate)? { + if !candidates.iter().any(|existing| { + existing.allow_mixed_summary_and_exact_execution + == candidate.allow_mixed_summary_and_exact_execution + && existing.enabled_materialization_keys + == candidate.enabled_materialization_keys + && existing + .queries + .iter() + .zip(&candidate.queries) + .all(|(a, b)| a.selected_plan_root == b.selected_plan_root) + }) { + candidates.push(candidate); + } + if candidates.len() > 4096 { + return Err(invalid( + "deployment candidate inventory exceeds budget; no partial inventory selected", + )); + } + } + } + Ok(candidates) +} + +fn enumerate_frontier_candidates( + mut request: PhysicalCompilationRequest, ) -> Result, CompileError> { + request.planner_candidate_forests.clear(); let mut candidates = materialization_candidates(request)?; let roots: Vec<_> = candidates .last() - .expect("exact alternative") + .expect("exact candidate") .queries .iter() .map(|q| match &q.selected_plan_root.expr { planner_types::post_asap::SummaryExpr::KeepPreAsap(root) => std::rc::Rc::clone(root), - _ => unreachable!("native alternative retains canonical roots"), + _ => unreachable!("native candidate retains canonical roots"), }) .collect(); let strategy = @@ -813,11 +864,11 @@ pub fn enumerate_exact_and_materialized_candidates( .collect(); if maintained_roots.iter().any(Option::is_some) { // Current-series rules are compatible with window summaries in other - // workload roots. Preserve each priced temporal alternative and mask. + // workload roots. Preserve each priced temporal candidate and mask. let maintained: Vec<_> = candidates .iter() - .map(|alternative| { - let mut candidate = alternative.clone(); + .map(|candidate| { + let mut candidate = candidate.clone(); for (query, selected) in candidate.queries.iter_mut().zip(&maintained_roots) { if let Some(selected) = selected { query.selected_plan_root = std::rc::Rc::clone(selected); @@ -889,7 +940,7 @@ fn materialization_candidates( &query.selected_plan_root, ) { Ok(found) => keys.extend(found), - // A failed local projection must not make the native alternative + // A failed local projection must not make the native candidate // disappear. Compile/select_lowest_cost_candidate retains its concrete unavailability. Err(_) => return Ok(vec![request, exact]), } @@ -921,6 +972,34 @@ mod tests { use super::super::compiler::BackendLocalPlanningInput; use super::*; + /// Deployment pricing must see candidate sketch families, not only an + /// upstream winner chosen before runtime resource evidence is applied. + #[test] + fn planner_quantile_inventory_reaches_backend_before_family_selection() { + let mut input = fixture(); + let queries = input.query_workload.repeating_queries.as_mut().unwrap(); + queries.truncate(1); + queries[0].query = planner_types::workload::Query("quantile_over_time(0.9,m[1m])".into()); + queries[0].requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( + crate::types::AccuracyTarget::Epsilon(0.05), + ); + let (request, _) = input.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let roots = candidates + .iter() + .flat_map(|c| &c.queries) + .map(|q| format!("{:?}", q.selected_plan_root)) + .collect::>(); + assert!( + roots.iter().any(|r| r.contains("Kll")), + "KLL disappeared before backend pricing" + ); + assert!( + roots.iter().any(|r| r.contains("DDSketch")), + "DDSketch disappeared before backend pricing" + ); + } + /// A deployment without external execution rejects native candidates before pricing. #[test] fn local_only_deployment_rejects_external_candidate_before_pricing() { @@ -1256,7 +1335,7 @@ mod tests { let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); request.allow_mixed_summary_and_exact_execution = false; request.queries[0].window_realization_candidates.clear(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let candidates = enumerate_frontier_candidates(request).unwrap(); let (manifests, explanations) = compile_candidates_for_pricing( candidates, env, @@ -1382,7 +1461,7 @@ mod tests { crate::types::AccuracyTarget::Exact, ); let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let candidates = enumerate_frontier_candidates(request).unwrap(); assert_eq!( candidates.len(), 5, @@ -1446,7 +1525,7 @@ mod tests { WorkloadCostEvidence, ) { let (request, env) = fixture().into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let candidates = enumerate_frontier_candidates(request).unwrap(); let quotes = candidates .iter() .map(|candidate| { @@ -1480,7 +1559,7 @@ mod tests { // Retained local input is priced once per metric, separate from the native service. #[test] - fn counter_materialization_manifest_prices_owned_state_and_distinct_native_alternative() { + fn counter_materialization_manifest_prices_owned_state_and_distinct_native_candidate() { use planner_types::workload::{AccuracyRequirement, Query}; let mut snapshot = fixture(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; @@ -1535,19 +1614,14 @@ mod tests { ), ("sum_over_time(m[1m]) + count_over_time(m[1m])", vec!["m"]), ] { - let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); - request.queries[0].query_string = query.into(); - request - .query_workload - .as_mut() - .unwrap() - .repeating_queries - .as_mut() - .unwrap()[0] - .query = planner_types::workload::Query(query.into()); + let mut input = fixture(); + input.query_workload.repeating_queries.as_mut().unwrap()[0].query = + planner_types::workload::Query(query.into()); + let (request, env) = input.into_physical_compilation_request().unwrap(); let exact = enumerate_exact_and_materialized_candidates(request) .unwrap() - .pop() + .into_iter() + .find(|candidate| !candidate.allow_mixed_summary_and_exact_execution) .unwrap(); let plan = DeploymentPlanCompiler .compile_promql(exact.clone(), env.clone()) @@ -1663,7 +1737,9 @@ mod tests { assert!(plan.cost_comparison.unwrap().candidate_evaluations[0] .unavailable_reason .is_some()); - evidence.quotes[1].executable = false; + for quote in &mut evidence.quotes[1..] { + quote.executable = false; + } assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence @@ -1764,7 +1840,7 @@ mod tests { } #[test] - fn exact_alternative_does_not_require_unused_state_implementation_evidence() { + fn exact_candidate_does_not_require_unused_state_implementation_evidence() { let (candidates, env, evidence) = quoted(); let mut exact = candidates[1].clone(); assert_eq!( diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index a97342bb1..1221e0cce 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -331,6 +331,44 @@ pub fn select_workload_with_accuracy_model_and_trace( Ok((selected, trace)) } +/// Preserve Planner candidates for deployment admission and pricing. This +/// does not select a winner or interpret an absent runtime quote as illegality. +/// Each returned forest has one root; independent roots remain factored rather +/// than materializing the workload's Cartesian product. +pub fn enumerate_workload_candidates( + roots: Vec<(usize, Rc)>, + accuracy: AccuracyTarget, + cost_model: &ControlPlaneCostModel, + evidence: &dyn AccuracyEvidenceProvider, + accuracy_model: &dyn AccuracyModel, +) -> Result, SelectionError> { + let strategies = replacement_strategies(cost_model, evidence, accuracy_model); + let space = asap_aware_mapping::search_workload_with_targets( + roots + .into_iter() + .map(|(id, root)| (id, root, Some(accuracy.clone()))) + .collect(), + &strategies, + accuracy_model, + ); + let mut inventory = asap_aware_mapping::replacement::CandidateDagInventory { + candidates: Vec::new(), + rejected_assemblies: Vec::new(), + }; + for (id, _) in &space.roots { + let root = space + .enumerate_candidate_dags_for_root(id, 65_536) + .map_err(|error| SelectionError::Workload(error.to_string()))?; + inventory.candidates.extend(root.candidates); + inventory.rejected_assemblies.extend( + root.rejected_assemblies + .into_iter() + .map(|reason| format!("query {id}: {reason}")), + ); + } + Ok(inventory) +} + /// The [`ReplacementStrategy`] set this deployment registers, carrying its own /// cost and accuracy models. This is deliberately not Planner's /// `default_strategies_with`: two of the strategies that list would give us are From 9a2f3ca3c133c89f131760feef8e38ec74b45cb4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 20:44:56 +0000 Subject: [PATCH 14/45] fix: execute persisted SQL physical DAGs without request-time lowering --- control_plane/src/physical/compiler.rs | 2 + control_plane/src/query_plan.rs | 14 +- control_plane/src/query_plan/residual.rs | 1 + crates/asap_types/src/query_plan.rs | 9 +- crates/asap_types/src/query_plan/native.rs | 326 ++++++++++++++++++ .../accelerator.rs | 6 +- .../asap_clickhouse_query_engine/execution.rs | 159 +++------ .../query_engines/asap_query_engine/engine.rs | 2 + .../asap_query_engine/exact_subqueries.rs | 1 + .../asap_query_engine/live_serve.rs | 1 + .../asap_query_engine/logical_dag.rs | 4 + .../asap_query_engine/post_asap_readout.rs | 4 + .../asap_query_engine/test_plan.rs | 1 + .../sketch_db/current_series.rs | 1 + .../types/hot_reload_config.rs | 1 + data_plane/tests/support/physical_fixture.rs | 1 + 16 files changed, 419 insertions(+), 114 deletions(-) create mode 100644 crates/asap_types/src/query_plan/native.rs diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index a183b4305..7c3f97533 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2046,6 +2046,7 @@ impl DeploymentPlanCompiler { .expect("compiled Planner DAG"); query_node_bindings.insert((query_index, compiled.dag.root), root); Ok(crate::query_plan::QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: query.query_id.clone(), canonical_query: canonical.clone(), @@ -2077,6 +2078,7 @@ impl DeploymentPlanCompiler { // Keep native semantics instead of lowering an unbound summary. let root = crate::query_plan::QueryNodeId(0); Ok(crate::query_plan::QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: query.query_id.clone(), canonical_query: canonical.clone(), diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index b75e4ea49..003e7a5f9 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -41,6 +41,7 @@ where }; let root = compiler.lower(root)?; Ok(QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id, canonical_query, @@ -80,7 +81,8 @@ where lowered: Some(&mut lowered), }; let root = compiler.lower(root)?; - Ok(QueryPlanEntry { + let mut entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::ClickHouseSql, query_id, canonical_query, @@ -89,7 +91,9 @@ where nodes: compiler.nodes, instant, fallback, - }) + }; + entry.compile_relational_physical_dag()?; + Ok(entry) } /// Compile a composable query while exposing the stable mapping from @@ -123,6 +127,7 @@ where }; let root = compiler.lower(root)?; let mut entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id, canonical_query, @@ -977,6 +982,7 @@ mod catalog_binding_tests { config.pane_origin_ms = Some(0); let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); let entry = QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "sum_over_time(m[1m])".into(), @@ -1232,6 +1238,7 @@ mod tests { #[test] fn language_tag_preserves_query_entry_serde() { let entry = QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: canonical_promql("up").unwrap(), @@ -1259,6 +1266,7 @@ mod tests { #[test] fn language_catalog_keys_keep_equal_query_text_distinct() { let base = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: "prom".into(), canonical_query: "shared".into(), @@ -1329,6 +1337,7 @@ mod tests { }, ); let entry = QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), @@ -1355,6 +1364,7 @@ mod tests { reason: "prepared".into(), }; let entry = QueryPlanEntry { + physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "topk(2, rate(m[5m]))".into(), diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index dabd5e047..0083566d8 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -298,6 +298,7 @@ pub fn compile_logical( }; let root = lower.lower(&expr)?; let entry = QueryPlanEntry { + physical_dag: None, language: super::QueryLanguage::PromQl, query_id, canonical_query, diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index b4c1182b2..a11aa0f2f 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -6,6 +6,7 @@ //! searching for compatible materializations. pub mod current_series; +mod native; pub mod residual; use std::collections::{BTreeMap, BTreeSet}; @@ -295,6 +296,9 @@ pub use crate::executable_plan::QueryNodeId; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct QueryPlanEntry { + /// Planner-selected native computation, persisted before activation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub physical_dag: Option, #[serde(default)] pub language: QueryLanguage, pub query_id: String, @@ -391,8 +395,10 @@ impl QueryPlanEntry { self.query_id, self.root.0 ))); } + if self.relation_output_schema()?.is_some() || self.physical_dag.is_some() { + self.recover_relational_physical_dag()?; + } for (id, node) in &self.nodes { - validate_native_relation(*id, node)?; if let QueryPlanNode::Logical { operator, inputs } = node { operator.validate(inputs.len())?; } @@ -773,6 +779,7 @@ mod contract_tests { } /// Bind portable relation semantics before an installed plan can access its sources. +#[cfg(test)] fn validate_native_relation(id: QueryNodeId, node: &QueryPlanNode) -> Result<(), QueryPlanError> { use planner_types::post_asap::{ ExecutableDagNode, ExecutableOperatorPayload as Payload, ExecutionDataState, PostAsapNodeId, diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs new file mode 100644 index 000000000..e05572f78 --- /dev/null +++ b/crates/asap_types/src/query_plan/native.rs @@ -0,0 +1,326 @@ +//! Installation of native SQL fragments; recovery only decodes physical operators. +use super::*; +use asap_physical_operators::{dag, physical_planner::CompiledPhysicalDag}; +use planner_types::post_asap::{ + ExecutableDagNode, ExecutableOperatorPayload as Payload, ExecutionDataState, PostAsapNodeId, + SummarySchema, +}; +use std::sync::Arc; + +impl QueryPlanEntry { + /// Invoke during plan compilation, after external/storage boundaries are fixed. + pub fn compile_relational_physical_dag(&mut self) -> Result<(), QueryPlanError> { + let Some(expected) = self.relation_output_schema()? else { + self.physical_dag = None; + return Ok(()); + }; + let compiled = self + .compile_relation(self.root, &expected) + .map_err(QueryPlanError::Invalid)?; + self.physical_dag = Some( + serde_json::from_slice( + &compiled + .encode() + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?, + ) + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?, + ); + Ok(()) + } + + fn compile_relation( + &self, + root: QueryNodeId, + expected: &SummarySchema, + ) -> Result { + let mut pending = vec![(root, expected.clone())]; + let mut schemas = BTreeMap::::new(); + let mut operations = BTreeMap::new(); + let mut sources = Vec::new(); + // Bind the entire computation before reading any storage source. + while let Some((id, expected)) = pending.pop() { + if let Some(previous) = schemas.get(&id) { + if previous != &expected { + return Err("inconsistent relation schemas".into()); + } + continue; + } + schemas.insert(id, expected.clone()); + let (payload, inputs) = match self.nodes.get(&id) { + Some(QueryPlanNode::Relational { + input, + operation, + input_schema, + output_schema, + }) => { + if output_schema != &expected { + return Err("relational output schema mismatch".into()); + } + let operation = + serde_json::from_value(operation.clone()).map_err(|e| e.to_string())?; + ( + Payload::Value { operation }, + vec![(*input, input_schema.clone())], + ) + } + Some(QueryPlanNode::RelationalJoin { + inputs, + join_kind, + pred, + left_schema, + right_schema, + output_schema, + pruning, + }) => { + if output_schema != &expected { + return Err("join output schema mismatch".into()); + } + if pruning.is_some() { + return Err( + "candidate pruning is not bound for this relation source".into() + ); + } + ( + Payload::RelationalJoin { + join_kind: join_kind.clone(), + pred: serde_json::from_value(pred.clone()) + .map_err(|e| e.to_string())?, + pruning: None, + }, + vec![ + (inputs[0], left_schema.clone()), + (inputs[1], right_schema.clone()), + ], + ) + } + Some(_) => { + sources.push(id); + continue; + } + None => return Err("missing relation node".into()), + }; + let node = ExecutableDagNode { + id: PostAsapNodeId( + u32::try_from(id.0).map_err(|_| "relation node ID exceeds Planner range")?, + ), + payload, + output_state: ExecutionDataState::QUERY_ROWS, + output_schema: expected, + guarantee: None, + }; + let op = dag::planner::compile_node( + &node, + &inputs + .iter() + .map(|(_, schema)| Arc::new(schema.clone())) + .collect::>(), + ) + .map_err(|e| e.to_string())?; + operations.insert( + id, + (inputs.iter().map(|(id, _)| id.0).collect::>(), op), + ); + pending.extend(inputs); + } + let compiled = + asap_physical_operators::physical_planner::CompiledPhysicalDag::from_operators( + sources + .iter() + .map(|id| { + ( + id.0, + asap_physical_operators::physical_planner::InputContract::bounded( + Arc::new(schemas[id].clone()), + ), + ) + }) + .collect(), + operations.into_iter().map(|(id, op)| (id.0, op)).collect(), + vec![root.0], + ) + .map_err(|e| e.to_string())?; + Ok(compiled) + } + + pub fn relation_output_schema(&self) -> Result, QueryPlanError> { + match self.nodes.get(&self.root) { + Some( + QueryPlanNode::Relational { output_schema, .. } + | QueryPlanNode::RelationalJoin { output_schema, .. }, + ) => Ok(Some(output_schema.clone())), + Some(QueryPlanNode::ExternalExact { request, .. }) => match &request.output { + ExternalExactOutput::Relation { schema } => serde_json::from_value(schema.clone()) + .map(Some) + .map_err(|error| QueryPlanError::Invalid(error.to_string())), + _ => Ok(None), + }, + _ => Ok(None), + } + } + + /// Validate persisted computation and boundary schemas without logical lowering. + pub fn recover_relational_physical_dag(&self) -> Result { + let invalid = |message: String| QueryPlanError::Invalid(message); + let value = self + .physical_dag + .as_ref() + .ok_or_else(|| invalid("missing installed physical DAG".into()))?; + let dag = CompiledPhysicalDag::decode( + &serde_json::to_vec(value).map_err(|e| invalid(e.to_string()))?, + ) + .map_err(|e| invalid(e.to_string()))?; + if dag.roots() != [self.root.0] { + return Err(invalid( + "installed physical root differs from query root".into(), + )); + } + let expected = self + .relation_output_schema()? + .ok_or_else(|| invalid("physical relation has no output schema".into()))?; + if *dag + .output_contract(self.root.0) + .map_err(|e| invalid(e.to_string()))? + .schema + != expected + { + return Err(invalid("installed physical output schema mismatch".into())); + } + let mut expected_inputs = BTreeMap::new(); + let mut pending = vec![(self.root, expected)]; + let mut seen = BTreeMap::new(); + while let Some((id, schema)) = pending.pop() { + if let Some(previous) = seen.insert(id, schema.clone()) { + if previous != schema { + return Err(invalid("inconsistent relation schemas".into())); + } + continue; + } + match self.nodes.get(&id) { + Some(QueryPlanNode::Relational { + input, + input_schema, + output_schema, + .. + }) => { + if output_schema != &schema { + return Err(invalid("relation output schema mismatch".into())); + } + pending.push((*input, input_schema.clone())); + } + Some(QueryPlanNode::RelationalJoin { + inputs, + left_schema, + right_schema, + output_schema, + .. + }) => { + if output_schema != &schema { + return Err(invalid("join output schema mismatch".into())); + } + pending.extend([ + (inputs[0], left_schema.clone()), + (inputs[1], right_schema.clone()), + ]); + } + Some(_) => { + expected_inputs.insert(id.0, schema); + } + None => { + return Err(invalid( + "physical input references absent query node".into(), + )) + } + } + } + let actual: BTreeMap<_, _> = dag + .input_contracts() + .map(|(id, c)| (id, c.schema.as_ref().clone())) + .collect(); + if actual != expected_inputs { + return Err(invalid( + "physical input boundaries differ from installed bindings".into(), + )); + } + Ok(dag) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::{ + post_asap::{SummaryFamilyType, SummaryField}, + pre_asap::DataType, + }; + + fn installed() -> QueryPlanEntry { + let schema = SummarySchema { + fields: vec![SummaryField { + name: "value".into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }], + time_index: None, + }; + let mut entry = QueryPlanEntry { + physical_dag: None, + language: QueryLanguage::ClickHouseSql, + query_id: "native".into(), + canonical_query: "SELECT value".into(), + fixed_evaluation: None, + root: QueryNodeId(0), + nodes: BTreeMap::from([( + QueryNodeId(0), + QueryPlanNode::ExternalExact { + request: ExternalExactRequest { + language: QueryLanguage::ClickHouseSql, + expression: "SELECT value".into(), + output: ExternalExactOutput::Relation { + schema: serde_json::to_value(schema).unwrap(), + }, + parameters: BTreeMap::new(), + start_parameter: None, + end_parameter: None, + input_contracts: vec![], + }, + inputs: vec![], + }, + )]), + instant: InstantExecution { + lookback_ms: 0, + full_history: false, + cumulative_readout: false, + }, + fallback: FallbackPolicy::Reject, + }; + entry.compile_relational_physical_dag().unwrap(); + entry + } + + // Restart keeps the physical program; unknown formats and stale bindings fail activation. + #[test] + fn recovery_validates_physical_version_roots_and_input_schema() { + let entry = installed(); + let encoded = serde_json::to_vec(&entry).unwrap(); + let restored: QueryPlanEntry = serde_json::from_slice(&encoded).unwrap(); + restored.validate(&BTreeSet::new()).unwrap(); + let mut corrupt = restored.clone(); + corrupt.physical_dag.as_mut().unwrap()["version"] = serde_json::json!(99); + assert!(corrupt.validate(&BTreeSet::new()).is_err()); + let mut corrupt = restored.clone(); + corrupt.root = QueryNodeId(1); + assert!(corrupt.validate(&BTreeSet::new()).is_err()); + let mut corrupt = restored.clone(); + if let QueryPlanNode::ExternalExact { request, .. } = + corrupt.nodes.get_mut(&QueryNodeId(0)).unwrap() + { + if let ExternalExactOutput::Relation { schema } = &mut request.output { + schema["fields"][0]["name"] = serde_json::json!("different"); + } + } + assert!(corrupt.validate(&BTreeSet::new()).is_err()); + let mut missing = restored; + missing.physical_dag = None; + assert!(missing.validate(&BTreeSet::new()).is_err()); + } +} diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 3e2f11496..29fb3715d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -496,6 +496,7 @@ mod tests { }, ); entry.root = project; + entry.compile_relational_physical_dag().unwrap(); entry } @@ -574,6 +575,7 @@ mod tests { let sort = QueryNodeId(4); let root = QueryNodeId(5); let executable = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::ClickHouseSql, query_id: "SELECT value FROM samples".into(), canonical_query: "SELECT value FROM samples".into(), @@ -707,7 +709,8 @@ mod tests { } else { BTreeMap::new() }; - let entry = QueryPlanEntry { + let mut entry = QueryPlanEntry { + physical_dag: None, query_id: sql.clone(), canonical_query: canonical_sql.clone(), language: QueryLanguage::ClickHouseSql, @@ -721,6 +724,7 @@ mod tests { instant: executable.instant, fallback: executable.fallback, }; + entry.compile_relational_physical_dag().unwrap(); let query_plan = QueryPlan { plan_id: 41, plan_version: 1, diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index ae0681c05..59438d33c 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -79,117 +79,25 @@ impl RelationDagExecutor<'_> { use super::relational_adapter::native; use asap_physical_operators::dag::{self, operators::Operator}; use futures::{FutureExt, StreamExt}; - use planner_types::post_asap::{ - ExecutableDagNode, ExecutableOperatorPayload as Payload, ExecutionDataState, - PostAsapNodeId, SummarySchema, - }; - use std::sync::Arc; - let mut pending = vec![(root, expected.clone())]; - let mut schemas = BTreeMap::::new(); - let mut operations = BTreeMap::new(); - let mut sources = Vec::new(); - // Bind the entire computation before reading any storage source. - while let Some((id, expected)) = pending.pop() { - if let Some(previous) = schemas.get(&id) { - if previous != &expected { - return Err("inconsistent relation schemas".into()); - } - continue; - } - schemas.insert(id, expected.clone()); - let (payload, inputs) = match self.entry.nodes.get(&id) { - Some(QueryPlanNode::Relational { - input, - operation, - input_schema, - output_schema, - }) => { - if output_schema != &expected { - return Err("relational output schema mismatch".into()); - } - let operation = - serde_json::from_value(operation.clone()).map_err(|e| e.to_string())?; - ( - Payload::Value { operation }, - vec![(*input, input_schema.clone())], - ) - } - Some(QueryPlanNode::RelationalJoin { - inputs, - join_kind, - pred, - left_schema, - right_schema, - output_schema, - pruning, - }) => { - if output_schema != &expected { - return Err("join output schema mismatch".into()); - } - if pruning.is_some() { - return Err( - "candidate pruning is not bound for this relation source".into() - ); - } - ( - Payload::RelationalJoin { - join_kind: join_kind.clone(), - pred: serde_json::from_value(pred.clone()) - .map_err(|e| e.to_string())?, - pruning: None, - }, - vec![ - (inputs[0], left_schema.clone()), - (inputs[1], right_schema.clone()), - ], - ) - } - Some(_) => { - sources.push(id); - continue; - } - None => return Err("missing relation node".into()), - }; - let node = ExecutableDagNode { - id: PostAsapNodeId( - u32::try_from(id.0).map_err(|_| "relation node ID exceeds Planner range")?, - ), - payload, - output_state: ExecutionDataState::QUERY_ROWS, - output_schema: expected, - guarantee: None, - }; - let op = dag::planner::compile_node( - &node, - &inputs - .iter() - .map(|(_, schema)| Arc::new(schema.clone())) - .collect::>(), - ) + let compiled = self + .entry + .recover_relational_physical_dag() .map_err(|e| e.to_string())?; - operations.insert( - id, - (inputs.iter().map(|(id, _)| id.0).collect::>(), op), - ); - pending.extend(inputs); + if compiled.roots() != [root.0] + || compiled + .output_contract(root.0) + .map_err(|e| e.to_string())? + .schema + .as_ref() + != expected + { + return Err("requested relation differs from installed physical root".into()); } - let compiled = - asap_physical_operators::physical_planner::CompiledPhysicalDag::from_operators( - sources - .iter() - .map(|id| { - ( - id.0, - asap_physical_operators::physical_planner::InputContract::bounded( - Arc::new(schemas[id].clone()), - ), - ) - }) - .collect(), - operations.into_iter().map(|(id, op)| (id.0, op)).collect(), - vec![root.0], - ) - .map_err(|e| e.to_string())?; + let schemas: BTreeMap<_, _> = compiled + .input_contracts() + .map(|(id, contract)| (QueryNodeId(id), contract.schema.as_ref().clone())) + .collect(); + let sources: Vec<_> = schemas.keys().copied().collect(); let mut resolved_inputs = BTreeMap::new(); let context = dag::RunContext::new( dag::Scope::Query { @@ -575,6 +483,7 @@ mod tests { fn external_entry(schema: &SummarySchema) -> QueryPlanEntry { QueryPlanEntry { + physical_dag: None, language: QueryLanguage::ClickHouseSql, query_id: "shared-external".into(), canonical_query: "SELECT x".into(), @@ -609,7 +518,8 @@ mod tests { #[test] fn relation_dag_memoizes_a_shared_node_and_enforces_one_edge_schema() { let schema = relation_schema("x"); - let entry = external_entry(&schema); + let mut entry = external_entry(&schema); + entry.compile_relational_physical_dag().unwrap(); let relation = ClickHouseRelation::from_json_compact( &schema, br#"{"meta":[{"name":"x","type":"Int64"}],"data":[[1]]}"#, @@ -664,6 +574,9 @@ mod tests { }, ); entry.root = QueryNodeId(1); + entry.compile_relational_physical_dag().unwrap(); + let encoded = serde_json::to_vec(&entry).unwrap(); + let entry: QueryPlanEntry = serde_json::from_slice(&encoded).unwrap(); let relation = ClickHouseRelation::from_json_compact( &schema, br#"{"meta":[{"name":"x","type":"Int64"}],"data":[[1],[2]]}"#, @@ -697,6 +610,32 @@ mod tests { ); } + // Missing installed computation must not trigger serving-time re-lowering. + #[test] + fn relation_execution_requires_an_installed_physical_dag() { + let schema = relation_schema("x"); + let entry = external_entry(&schema); + let relation = ClickHouseRelation::from_json_compact( + &schema, + br#"{"meta":[{"name":"x","type":"Int64"}],"data":[[1]]}"#, + ) + .unwrap(); + let prepared = BTreeMap::from([(QueryNodeId(0), relation)]); + let index = SketchStore::new(); + let mut executor = RelationDagExecutor { + index: &index, + entry: &entry, + prepared: &prepared, + t0_ms: 0, + t1_ms: 1, + is_cumulative: false, + memo: BTreeMap::new(), + schemas: BTreeMap::new(), + evaluations: BTreeMap::new(), + }; + assert!(executor.execute(entry.root, &schema).is_err()); + } + #[test] fn exact_accumulator_window_end_coverage_includes_its_pane_start() { assert!(complete_pane_coverage( diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 6725b4e39..ede83919d 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -105,6 +105,7 @@ mod forwarding_policy_tests { let query = "sum(rate(m[5m]))"; let canonical = asap_types::query_plan::canonical_promql(query).unwrap(); let entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, @@ -2738,6 +2739,7 @@ mod range_stitch_tests { plan.query_plan.entries.insert( asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), QueryPlanEntry { + physical_dag: None, language: QueryLanguage::MetricsQl, query_id: "vm-scalar".into(), canonical_query: identity.clone(), diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index d717030e5..de8a0c90c 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -447,6 +447,7 @@ mod tests { use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn entry(nodes: BTreeMap) -> QueryPlanEntry { QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "remote-cut".into(), canonical_query: "a / b".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 8c9e26466..fdf481272 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -174,6 +174,7 @@ mod tests { ); } let entry = asap_types::query_plan::QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-sum".into(), canonical_query: "sum_over_time(bytes[1s])".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 262be3552..96eaf5ccb 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -1377,6 +1377,7 @@ mod topk_tests { TemporalOperation::Rate, ] { let entry = QueryPlanEntry { + physical_dag: None, language, query_id: "labels".into(), canonical_query: "test".into(), @@ -1454,6 +1455,7 @@ mod topk_tests { let summary = QueryNodeId(0); let root = QueryNodeId(1); let entry = QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "summary-rate-topk".into(), canonical_query: "topk(2, rate(requests_total[5m]))".into(), @@ -1605,6 +1607,7 @@ mod topk_tests { let filter = QueryNodeId(2); let root = QueryNodeId(3); let entry = QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), canonical_query: "topk(1, rate(requests_total[5m]))".into(), @@ -1771,6 +1774,7 @@ mod shared_runtime_tests { fn entry() -> QueryPlanEntry { QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: "shared-grid".into(), canonical_query: "shared-grid".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 813312ca4..e4805b193 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1127,6 +1127,7 @@ mod tests { #[test] fn multi_root_readout_uses_one_parent_context() { let entry = asap_types::query_plan::QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "shared".into(), canonical_query: "1+1".into(), @@ -1411,6 +1412,7 @@ mod tests { .unwrap(); idx.register(metadata); let mut entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::MetricsQl, query_id: "quantile".into(), canonical_query: "quantile_over_time(0.9, latency_ms[1s])".into(), @@ -1849,6 +1851,7 @@ mod tests { } let entry = asap_types::query_plan::QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), @@ -1950,6 +1953,7 @@ mod tests { idx.append_precompute(7, BTreeMap::new(), (0, 60_000), Box::new(accumulator)); let entry = asap_types::query_plan::QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index ef0a54c52..7da038b36 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -44,6 +44,7 @@ pub(super) fn entry( ) -> QueryPlanEntry { let canonical = canonical_promql(query).unwrap(); QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, diff --git a/data_plane/src/storage_engines/sketch_db/current_series.rs b/data_plane/src/storage_engines/sketch_db/current_series.rs index 2b99ea3ed..1887647ff 100644 --- a/data_plane/src/storage_engines/sketch_db/current_series.rs +++ b/data_plane/src/storage_engines/sketch_db/current_series.rs @@ -586,6 +586,7 @@ mod tests { plan.entries.insert( "test".into(), QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: "test".into(), canonical_query: "quantile by (job) (0.5, a)".into(), diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 8bc2e7ef6..096d2f41e 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -763,6 +763,7 @@ mod tests { fn readout_programs_follow_replaced_query_plan_bindings() { use asap_types::query_plan::*; let entry = |id: u64| QueryPlanEntry { + physical_dag: None, language: asap_types::QueryLanguage::PromQl, query_id: "sum_over_time(m[1m])".into(), canonical_query: "sum_over_time(m[1m])".into(), diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 1b9b03823..e86a100e9 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -147,6 +147,7 @@ pub fn artifact_from_materializations( query_plan.entries.insert( canonical.clone(), QueryPlanEntry { + physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, From 79d9fc1166efd4637c021caef45a48b6604c6311 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 20:48:12 +0000 Subject: [PATCH 15/45] feat: trace installed physical compilation recovery and input binding --- crates/asap_types/src/query_plan/native.rs | 4 ++++ .../asap_clickhouse_query_engine/execution.rs | 14 +++++++++++--- docs/runtime-debugging.md | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index e05572f78..916259638 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -9,6 +9,8 @@ use std::sync::Arc; impl QueryPlanEntry { /// Invoke during plan compilation, after external/storage boundaries are fixed. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.compile_install", query_id = %self.query_id), err)] pub fn compile_relational_physical_dag(&mut self) -> Result<(), QueryPlanError> { let Some(expected) = self.relation_output_schema()? else { self.physical_dag = None; @@ -159,6 +161,8 @@ impl QueryPlanEntry { } /// Validate persisted computation and boundary schemas without logical lowering. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.recover_validate", query_id = %self.query_id), err)] pub fn recover_relational_physical_dag(&self) -> Result { let invalid = |message: String| QueryPlanError::Invalid(message); let value = self diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index 59438d33c..489ddeb16 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -71,6 +71,8 @@ impl RelationDagExecutor<'_> { #[cfg(not(test))] fn record_evaluation(&mut self, _id: QueryNodeId) {} + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.execute", query_id = %self.entry.query_id, root = root.0), err)] fn execute_graph( &mut self, root: QueryNodeId, @@ -156,9 +158,15 @@ impl RelationDagExecutor<'_> { ) as asap_physical_operators::physical_planner::Source<'_>, ); } - let graph = compiled - .instantiate(resolved_inputs) - .map_err(|e| e.to_string())?; + let graph = { + let _binding = + tracing::debug_span!(target: "asap_runtime_debug", "physical_input_binding", + stage = "physical.bind_inputs", input_count = resolved_inputs.len()) + .entered(); + compiled + .instantiate(resolved_inputs) + .map_err(|e| e.to_string())? + }; let mut output = graph .execute(&[root.0], context) .map_err(|e| e.to_string())? diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 8e346de08..25628aaa6 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -69,3 +69,17 @@ query-time Sum and an exact fallback; it does not export a precomputed grouped Rate-result candidate. The selected plan therefore cannot establish that query-time Sum beats that absent alternative. Search coverage is scoped to the declared inventory, never a claim of exhaustive physical optimization. + +### Installed SQL physical DAG stages + +SQL plans compile native operators before installation (`physical.compile_install`). +Activation and serving validate the persisted program (`physical.recover_validate`); +this stage decodes physical operators and does not lower logical expressions. +`physical.bind_inputs` attaches the resolved external/SDS batches, and +`physical.execute` covers the request's physical execution path. These debug spans +include the query identity, root or input count, and errors without logging batches +or sketch payloads. They use the existing `asap_runtime_debug` target. + +These stages describe the installed SQL path. They do not establish that the +remaining PromQL candidate-selection and maintenance paths have migrated to the +same physical handoff. From 0fa6be7a12982fc9d451e1f55b9f23b089922b0f Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 20:58:54 +0000 Subject: [PATCH 16/45] build: adopt typed physical output persistence codecs --- Cargo.lock | 14 +++++++------- Cargo.toml | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 141eb841c..d156b753c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986#2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index 5cf1a8d86..01d38f16d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "2bdf1d2e8eca0ce5aabdc998ca0755b21da7f986" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } From 9a812802f78f71de3e7a3daa922741e11631b43d Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 21:23:58 +0000 Subject: [PATCH 17/45] feat: persist and recover bound native physical summary outputs --- Cargo.lock | 14 +- Cargo.toml | 12 +- crates/asap_types/src/precompute_plan.rs | 68 ++- .../asap_types/src/precompute_plan/catalog.rs | 2 +- .../sketch_db/index/maintenance.rs | 39 +- .../storage_engines/sketch_db/index/mod.rs | 3 + .../storage_engines/sketch_db/index/native.rs | 572 ++++++++++++++++++ .../sketch_db/persistence/part.rs | 1 + .../catalog-physical-plan-runtime.md | 31 + 9 files changed, 721 insertions(+), 21 deletions(-) create mode 100644 data_plane/src/storage_engines/sketch_db/index/native.rs diff --git a/Cargo.lock b/Cargo.lock index d156b753c..53360cc36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=26542909b0b37a45220d8f9d1e4b7963c8938dc0#26542909b0b37a45220d8f9d1e4b7963c8938dc0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index 01d38f16d..74a9f2478 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "26542909b0b37a45220d8f9d1e4b7963c8938dc0" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 32a46c010..96c558ce9 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -152,6 +152,8 @@ pub struct IngestContract { pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, + /// Backend-produced typed physical output, distinct from legacy sketch frames. + NativeBatchV1, ExactAccumulatorV1, /// Persisted backend state with explicit Planner family and population layout. PlannerExactAccumulatorV1, @@ -855,7 +857,7 @@ impl PrecomputePlan { crate::WindowKind::Session => None, } || schema.window.pane_origin_ms != materialization.pane_origin_ms - || schema.encodings != state_encodings(&accumulator.family) + || !state_encodings_match(&accumulator.family, &schema.encodings) { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), @@ -903,8 +905,22 @@ pub(crate) fn state_schema_id(fingerprint: crate::PolicyFingerprint) -> String { format!("{}:summary-state:v1:{}", BACKEND_COMPAT, fingerprint.0) } +// Persisted schemas may declare a subset of formats. Adding a decoder must +// not invalidate existing installed plans that use only older supported codecs. +pub(crate) fn state_encodings_match( + family: &SummaryFamilyType, + encodings: &[StateEncoding], +) -> bool { + let supported = state_encodings(family); + !encodings.is_empty() + && encodings.iter().collect::>().len() == encodings.len() + && encodings + .iter() + .all(|encoding| supported.contains(encoding)) +} + pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec { - match family { + let mut encodings = match family { SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate, @@ -932,7 +948,25 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec StateEncoding::SketchCoreMsgpackV1, ], _ => Vec::new(), + }; + if matches!( + family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum + | planner_types::post_asap::ExactKind::Count + | planner_types::post_asap::ExactKind::Min + | planner_types::post_asap::ExactKind::Max + | planner_types::post_asap::ExactKind::Rate + | planner_types::post_asap::ExactKind::Increase, + _ + ) + ) || matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch | SketchAlgorithm::Hll | + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + { + encodings.push(StateEncoding::NativeBatchV1); } + encodings } #[cfg(test)] @@ -950,6 +984,36 @@ mod source_window_cohort_tests { })) .unwrap() } + // New native-format support must not invalidate persisted older codec subsets. + #[test] + fn older_encoding_subsets_remain_valid_and_unknown_codecs_fail() { + use planner_types::post_asap::{SketchKind, SketchParams}; + let family = SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + Default::default(), + ); + assert!(state_encodings_match( + &family, + &[ + StateEncoding::SketchlibProtobufV1, + StateEncoding::SketchCoreMsgpackV1 + ] + )); + assert!(state_encodings_match( + &family, + &[StateEncoding::NativeBatchV1] + )); + assert!(!state_encodings_match(&family, &[])); + assert!(!state_encodings_match( + &family, + &[StateEncoding::ExactAccumulatorV1] + )); + assert!(!state_encodings_match( + &family, + &[StateEncoding::NativeBatchV1, StateEncoding::NativeBatchV1] + )); + } + #[test] fn producer_roster_roundtrip_and_watermark_scope() { let envelope = PlanEnvelope { diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index f2322cd93..4307f13e0 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -207,7 +207,7 @@ impl PrecomputePlan { schema_id: schema.schema_id.clone(), }); } - if schema.encodings != state_encodings(&family) { + if !state_encodings_match(&family, &schema.encodings) { return Err(invalid("encoding does not match state family")); } if config.num_aggregates_to_retain == Some(0) { diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index 3224b7248..bf31f27d8 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -275,6 +275,28 @@ impl SketchStore { generation: &Arc, expected_windows: &BTreeSet<(u64, u64)>, group: &BTreeMap, + ) -> Result { + self.read_frozen_windows_with( + sid, + definition, + generation, + expected_windows, + group, + |name, _, bytes| { + reconstruct_exact_agg(name, bytes) + .ok_or_else(|| "immutable input accumulator cannot be decoded".to_string()) + }, + ) + } + + pub(super) fn read_frozen_windows_with( + &self, + sid: u64, + definition: StoredOutputId, + generation: &Arc, + expected_windows: &BTreeSet<(u64, u64)>, + group: &BTreeMap, + mut decode: impl FnMut(&str, u8, &[u8]) -> Result, String>, ) -> Result { let start_ms = expected_windows .iter() @@ -377,8 +399,11 @@ impl SketchStore { if population != *group { continue; } - let state = reconstruct_exact_agg(&entry.sketch_type_name, &entry.sketch_bytes) - .ok_or("immutable input accumulator cannot be decoded")?; + let state = decode( + &entry.sketch_type_name, + entry.encoding_tag, + &entry.sketch_bytes, + )?; if windows .insert((record.start_ts, record.end_ts), Arc::from(state)) .is_some() @@ -720,9 +745,13 @@ impl SketchStore { labels: labels.into_values().collect(), }), sketch_type_name: state.type_name().to_string(), - encoding_tag: match binding.metadata.agg_kind { - AggKind::Sketch { .. } => encoding_to_tag(SketchEncoding::MsgpackFull), - AggKind::ExactAgg { .. } => 0, + encoding_tag: if state.type_name() == "NativePhysicalOutputV1" { + encoding_to_tag(SketchEncoding::NativeBatchV1) + } else { + match binding.metadata.agg_kind { + AggKind::Sketch { .. } => encoding_to_tag(SketchEncoding::MsgpackFull), + AggKind::ExactAgg { .. } => 0, + } }, sketch_bytes: bytes, }], diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index b1ae0b45a..2e7067f35 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -60,6 +60,7 @@ fn encoding_to_tag(enc: SketchEncoding) -> u8 { SketchEncoding::ProtoDelta => t::PROTO_DELTA, SketchEncoding::MsgpackFull => t::MSGPACK_FULL, SketchEncoding::MsgpackDelta => t::MSGPACK_DELTA, + SketchEncoding::NativeBatchV1 => t::NATIVE_BATCH_V1, } } @@ -73,6 +74,7 @@ fn tag_to_encoding(tag: u8) -> SketchEncoding { t::PROTO_DELTA => SketchEncoding::ProtoDelta, t::MSGPACK_FULL => SketchEncoding::MsgpackFull, t::MSGPACK_DELTA => SketchEncoding::MsgpackDelta, + t::NATIVE_BATCH_V1 => SketchEncoding::NativeBatchV1, // t::PROTO_FULL and t::UNKNOWN (legacy) both → Full. _ => SketchEncoding::ProtoFull, } @@ -3868,6 +3870,7 @@ pub use SummarySeriesMetadata as SketchInstanceMetadata; // alongside the store that uses it. mod admission; mod maintenance; +mod native; pub(crate) use maintenance::{CompleteRawMaintenanceCohort, FrozenExactWindows}; pub mod epoch_columnar; diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs new file mode 100644 index 000000000..1493cddaf --- /dev/null +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -0,0 +1,572 @@ +//! Bound native summary snapshots use the existing immutable publication and +//! recovery path. Window snapshots are not additive hot-state updates. +use super::*; +use crate::drivers::ingest::series_resolver::SeriesIdResolver; +use asap_physical_operators::{ + stored_state::native::{decode_batch, encode_batch}, + values::{Batch, Schema, Value}, + AggregateCore, SerializableToSink, +}; + +const NATIVE_OUTPUT_TYPE: &str = "NativePhysicalOutputV1"; +const NATIVE_OUTPUT_TAG: u8 = persistence::part::encoding_tag::NATIVE_BATCH_V1; + +#[derive(Clone)] +struct NativeSummaryOutput { + batch: Batch, + bytes: Vec, + kind: AggregationType, +} +impl NativeSummaryOutput { + fn new(batch: Batch, max_bytes: usize) -> Result { + let [row] = batch.rows() else { + return Err("one stored summary record requires one output row".into()); + }; + let states: Vec<_> = row + .iter() + .filter_map(|value| match value { + Value::Summary { state, .. } => Some(state), + _ => None, + }) + .collect(); + let [state] = states.as_slice() else { + return Err("stored native summary requires exactly one summary state".into()); + }; + let kind = state.get_accumulator_type(); + let bytes = encode_batch(&batch).map_err(|error| error.to_string())?; + if bytes.len() > max_bytes || batch.bytes() > max_bytes { + return Err("native summary exceeds publication/read budget".into()); + } + Ok(Self { batch, bytes, kind }) + } + fn validate_group(&self, group: &BTreeMap) -> Result<(), String> { + for (key, value) in group { + let column = self + .batch + .schema() + .fields + .iter() + .position(|field| &field.name == key) + .ok_or("native output is missing its stored group key")?; + if !matches!(&self.batch.rows()[0][column], Value::Utf8(actual) if actual.as_ref() == value) + { + return Err("native output group differs from stored address".into()); + } + } + Ok(()) + } +} +impl SerializableToSink for NativeSummaryOutput { + fn serialize_to_bytes(&self) -> Vec { + self.bytes.clone() + } + fn serialize_to_json(&self) -> serde_json::Value { + serde_json::json!({"format": NATIVE_OUTPUT_TYPE, "bytes": self.bytes}) + } +} +impl AggregateCore for NativeSummaryOutput { + fn clone_boxed_core(&self) -> Box { + Box::new(self.clone()) + } + fn type_name(&self) -> &'static str { + NATIVE_OUTPUT_TYPE + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn get_accumulator_type(&self) -> AggregationType { + self.kind + } + fn get_keys(&self) -> Option> { + None + } + fn approx_memory_bytes(&self) -> usize { + self.bytes.len() + self.batch.bytes() + } + fn merge_with( + &self, + _: &dyn AggregateCore, + ) -> Result, Box> { + Err("native output snapshots require an explicit physical merge operator".into()) + } + fn query_statistic( + &self, + _: asap_types::Statistic, + _: &Option, + _: &HashMap, + ) -> Result> { + Err("native output readout requires the installed physical DAG".into()) + } +} + +impl SketchStore { + /// Publish a finalized native result only after the complete raw input + /// cohort is durable. Existing publication fences prevent duplicate commits. + pub fn publish_native_summary_output( + &self, + resolver: &SeriesIdResolver, + config: &asap_types::PrecomputeMaterialization, + output: &crate::storage_engines::types::PrecomputedOutput, + batch: Batch, + max_bytes: usize, + ) -> Result { + use sha2::{Digest, Sha256}; + let generation = output + .catalog_generation + .as_ref() + .ok_or("native output requires a catalog generation")?; + let program = config + .derived_input + .as_ref() + .ok_or("native output requires installed input lineage")?; + let window = (output.start_timestamp, output.end_timestamp); + let cohort = + self.read_complete_raw_maintenance_cohort(generation, &program.inputs, window)?; + let (population_key, group) = build_attrs_fp_and_label_map(config, output)?; + let state = NativeSummaryOutput::new(batch, max_bytes)?; + let family = config.accumulator_spec().map_err(|e| e.to_string())?.family; + for value in &state.batch.rows()[0] { + if let Value::Summary { family: actual, .. } = value { + if actual != &family { + return Err("native output family differs from installed definition".into()); + } + } + } + state.validate_group(&group)?; + let sid = self.resolve_output_storage_handle( + resolver, + config.policy_fingerprint().into(), + &population_key, + Some(generation), + )?; + let mut digest = Sha256::new(); + digest.update(serde_json::to_vec(&(program, window)).map_err(|e| e.to_string())?); + for input in cohort.inputs() { + digest.update( + serde_json::to_vec(&(&input.stored_output_reference, &input.group)) + .map_err(|e| e.to_string())?, + ); + for (window, state) in &input.windows { + digest.update(serde_json::to_vec(window).map_err(|e| e.to_string())?); + let bytes = state.serialize_to_bytes(); + digest.update((bytes.len() as u64).to_le_bytes()); + digest.update(bytes); + } + } + self.publish_complete_raw_maintenance_output( + sid, + config, + output, + &state, + &cohort, + digest.finalize().into(), + ) + } + + /// Resolve the installed output/group prefix without minting a new handle, + /// then read exactly the requested immutable window and validate its format. + pub fn read_native_summary_output( + &self, + resolver: &SeriesIdResolver, + address: &asap_types::sds::StoredSummaryKey, + reference: &StoredOutputReference, + expected_schema: Schema, + max_bytes: usize, + ) -> Result { + address.validate().map_err(|e| e.to_string())?; + let (catalog, generation) = self + .descriptors + .authoritative_snapshot() + .ok_or("native read requires an installed catalog")?; + if (address.plan_id, address.plan_version) != (generation.plan_id, generation.plan_version) + || address.stored_output_id != reference.stored_output_id + || catalog + .output_reference(address.stored_output_id) + .map_err(|e| e.to_string())? + != *reference + { + return Err("native read differs from its installed output reference".into()); + } + let identity = &catalog.outputs[&address.stored_output_id]; + let data = &catalog.data_descriptors[&identity.data_descriptor_id]; + let pairs: Vec<_> = address + .population + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + .collect(); + let population_key = crate::drivers::ingest::population_attrs_fingerprint( + data.population_key_encoding, + &pairs, + )?; + let identity = serde_json::to_string(&(address.plan_id, address.plan_version, reference)) + .map_err(|e| e.to_string())?; + let sid = resolver + .lookup("stored-output", &population_key, &identity) + .ok_or("native stored output/group is unavailable")?; + let window = ( + u64::try_from(address.window.start_ms).map_err(|_| "negative native window")?, + u64::try_from(address.window.end_ms).map_err(|_| "negative native window")?, + ); + let frozen = self.read_frozen_windows_with( + sid, + address.stored_output_id, + &generation, + &BTreeSet::from([window]), + &address.population, + |name, tag, bytes| { + if name != NATIVE_OUTPUT_TYPE || tag != NATIVE_OUTPUT_TAG { + return Err("stored record is not a native physical output".into()); + } + let batch = decode_batch(bytes, expected_schema.clone(), max_bytes) + .map_err(|e| e.to_string())?; + let output = NativeSummaryOutput::new(batch, max_bytes)?; + output.validate_group(&address.population)?; + Ok(Box::new(output) as Box) + }, + )?; + let state = frozen.windows[&window] + .as_any() + .downcast_ref::() + .ok_or("native output decoder mismatch")?; + Ok(state.batch.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage_engines::types::PrecomputedOutput; + use asap_physical_operators::{ + dag::{operators::Operator, Limits, RunContext, Scope}, + physical_planner::{CompiledPhysicalDag, InputContract, Source}, + summary_kernels::SumAccumulator, + }; + use futures::{executor::block_on, StreamExt}; + use planner_types::{ + post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, + pre_asap::DataType, + }; + + fn run(input: Batch, operator: Operator) -> Batch { + let schema = input.schema().clone(); + let plan = CompiledPhysicalDag::from_operators( + BTreeMap::from([(0, InputContract::bounded(schema.clone()))]), + BTreeMap::from([(1, (vec![0], operator))]), + vec![1], + ) + .unwrap(); + let plan = CompiledPhysicalDag::decode(&plan.encode().unwrap()).unwrap(); + let graph = plan + .instantiate(BTreeMap::from([( + 0, + Box::new(Operator::source(schema, vec![input]).unwrap()) as Source<'_>, + )])) + .unwrap(); + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: 60_000, + revision: 0, + }, + Limits::default(), + ) + .unwrap(); + let stream = graph.execute(&[1], context).unwrap().remove(0); + let output = block_on(stream.collect::>()); + assert_eq!(output.len(), 1); + (*output.into_iter().next().unwrap().unwrap()).clone() + } + + // Group columns in the payload must agree with the bound record address. + #[test] + fn native_summary_payload_cannot_be_relabelled_as_another_group() { + let family = SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ); + let schema = Arc::new(SummarySchema { + fields: vec![ + SummaryField { + name: "job".into(), + dtype: SummaryFamilyType::Plain(DataType::Utf8), + nullable: false, + }, + SummaryField { + name: "state".into(), + dtype: family.clone(), + nullable: false, + }, + ], + time_index: None, + }); + let batch = Batch::try_new( + schema, + vec![vec![ + Value::Utf8("api".into()), + Value::Summary { + family, + state: Arc::new(SumAccumulator::new()), + }, + ]], + ) + .unwrap(); + let output = NativeSummaryOutput::new(batch, 1 << 20).unwrap(); + output + .validate_group(&BTreeMap::from([("job".into(), "api".into())])) + .unwrap(); + assert!(output + .validate_group(&BTreeMap::from([("job".into(), "worker".into())])) + .is_err()); + assert!(output + .validate_group(&BTreeMap::from([("unknown".into(), "api".into())])) + .is_err()); + } + + // Real durable source panes -> native build -> immutable publication -> bound + // lookup -> native readout, repeated after both store and resolver restart. + #[test] + fn native_summary_publication_and_bound_recovery_preserve_identity_and_format() { + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut query = fixture["query_workload"]["repeating_queries"][3].clone(); + query["query"] = "quantile(1.0, sum_over_time(immutable_value[1m]))".into(); + query["demand"]["fixed_interval_at"]["interval"] = 60_000.into(); + query["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); + fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); + let snapshot = serde_json::from_value(fixture).unwrap(); + let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let source = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.derived_input.is_none()) + .unwrap(); + let target = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.derived_input.is_some()) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let resolver_path = directory.path().join("resolver.jsonl"); + let resolver = SeriesIdResolver::open(resolver_path.clone()).unwrap(); + let persistence_config = || { + let mut config = persistence::config::SketchStorePersistenceConfig::with_memory_limit( + 1 << 24, + directory.path().join("store"), + ); + config.delete_older_than_ms = None; + config.hot_window_ms = None; + config + }; + let store = Arc::new(SketchStore::new()); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + let generation = store.active_catalog_generation().unwrap(); + let mut persistence = store.start_persistence(persistence_config()).unwrap(); + for (instance, value) in [("a", 0.125), ("b", 0.25)] { + let group = BTreeMap::from([("instance".to_string(), instance.to_string())]); + let coordinate = asap_types::sds::SummaryInstanceCoordinates { + stored_output_id: source.policy_fingerprint().into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 60_000, + }, + group_values: group.clone(), + }; + let revision = store + .admit_summary_updates(&generation, BTreeSet::from([coordinate.clone()])) + .unwrap(); + let mut output = PrecomputedOutput::new(0, 60_000, None, source.policy_fingerprint()); + output.population_labels = Some(group); + output.catalog_generation = Some(generation.clone()); + let mut state = SumAccumulator::new(); + state.update(value); + store + .publish_admitted_summary_update( + &generation, + &coordinate, + revision, + revision, + 120_000, + |writer| writer.ingest_precompute_with_series_id(700, source, &output, &state), + ) + .unwrap(); + } + let deadline = crate::tests::test_utilities::timing::deadline(Duration::from_secs(5)); + while !store.seal_finite_summary_input(&generation).unwrap() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + } + let cohort = store + .read_complete_raw_maintenance_cohort( + &generation, + &target.derived_input.as_ref().unwrap().inputs, + (0, 60_000), + ) + .unwrap(); + let values = cohort + .inputs() + .iter() + .flat_map(|input| input.windows.values()) + .map(|state| { + vec![Value::Float64( + state + .query_statistic(asap_types::Statistic::Sum, &None, &HashMap::new()) + .unwrap(), + )] + }) + .collect(); + let raw_schema = Arc::new(SummarySchema { + fields: vec![SummaryField { + name: "value".into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }], + time_index: None, + }); + let input = Batch::try_new(raw_schema.clone(), values).unwrap(); + let build = Operator::summary_build( + raw_schema, + target.accumulator_spec().unwrap().family, + 0, + None, + vec![], + ) + .unwrap(); + let output_batch = run(input, build); + let expected_schema = output_batch.schema().clone(); + let mut output = PrecomputedOutput::new(0, 60_000, None, target.policy_fingerprint()); + output.catalog_generation = Some(generation.clone()); + let before = persistence.manifest.live_parts().len(); + assert!(store + .publish_native_summary_output( + &resolver, + target, + &output, + output_batch.clone(), + 1 << 20 + ) + .unwrap()); + store + .publish_native_summary_output(&resolver, target, &output, output_batch, 1 << 20) + .unwrap(); + assert_eq!(persistence.manifest.live_parts().len(), before + 1); + let reference = plan + .summary_catalog + .output_reference(target.policy_fingerprint().into()) + .unwrap(); + let address = asap_types::sds::StoredSummaryKey { + plan_id: generation.plan_id, + plan_version: generation.plan_version, + stored_output_id: reference.stored_output_id, + population: BTreeMap::new(), + window: HalfOpenTimeRange { + start_ms: 0, + end_ms: 60_000, + }, + }; + let check = |store: &SketchStore, resolver: &SeriesIdResolver| { + let batch = store + .read_native_summary_output( + resolver, + &address, + &reference, + expected_schema.clone(), + 1 << 20, + ) + .unwrap(); + let handles = store.storage_handles_for_output(&reference); + assert_eq!(handles.len(), 1); + let frames = store.query_range(handles[0], 0, 60_000); + assert_eq!( + frames[0].samples[&60_000][0].encoding, + SketchEncoding::NativeBatchV1 + ); + let count = resolver.len(); + let mut other_group = address.clone(); + other_group + .population + .insert("instance".into(), "unknown".into()); + assert!(store + .read_native_summary_output( + resolver, + &other_group, + &reference, + expected_schema.clone(), + 1 << 20 + ) + .is_err()); + assert_eq!( + resolver.len(), + count, + "bound reads must not allocate outputs" + ); + let bytes = encode_batch(&batch).unwrap(); + let readout = Operator::readout( + batch.schema().clone(), + 0, + asap_types::Statistic::Quantile, + HashMap::from([("quantile".into(), "1.0".into())]), + ) + .unwrap(); + let values = run(batch, readout); + assert!(matches!(values.rows()[0][0], Value::Float64(v) if (v - 0.25).abs() < 0.01)); + let mut wrong = reference.clone(); + wrong.definition_id = plan + .summary_catalog + .output_reference(source.policy_fingerprint().into()) + .unwrap() + .definition_id; + assert!(store + .read_native_summary_output( + resolver, + &address, + &wrong, + expected_schema.clone(), + 1 << 20 + ) + .is_err()); + let mut missing = address.clone(); + missing.window.end_ms = 120_000; + assert!(store + .read_native_summary_output( + resolver, + &missing, + &reference, + expected_schema.clone(), + 1 << 20 + ) + .is_err()); + assert!(store + .read_native_summary_output( + resolver, + &address, + &reference, + expected_schema.clone(), + 1 + ) + .is_err()); + bytes + }; + let bytes = check(&store, &resolver); + persistence.shutdown(); + drop(store); + drop(resolver); + let recovered = Arc::new(SketchStore::new()); + recovered + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + let mut persistence = recovered.start_persistence(persistence_config()).unwrap(); + let resolver = SeriesIdResolver::open(resolver_path).unwrap(); + assert_eq!(check(&recovered, &resolver), bytes); + persistence.shutdown(); + } +} diff --git a/data_plane/src/storage_engines/sketch_db/persistence/part.rs b/data_plane/src/storage_engines/sketch_db/persistence/part.rs index 52078104c..c760668cb 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/part.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/part.rs @@ -105,6 +105,7 @@ pub mod encoding_tag { pub const PROTO_DELTA: u8 = 2; pub const MSGPACK_FULL: u8 = 3; pub const MSGPACK_DELTA: u8 = 4; + pub const NATIVE_BATCH_V1: u8 = 5; } /// One entry inside a decoded part. The `start_ts`/`end_ts`/`label` diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 9b2ef1c64..48ee6bbe6 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -28,3 +28,34 @@ The HTTP lifecycle is exposed through `/api/v1/physical-plan`, The current V1 storage path reads only the installed plan version. Matching definition identity does not authorize cross-version payload reuse; that would require an explicit reader binding and separate compatibility support. + +## Persisted physical computation and outputs + +Installed SQL entries include a serialized native physical DAG. Installation +compiles it once; activation and requests validate its version, roots and input +schemas, then bind concrete batches. Missing physical programs require plan +recompilation. Serving does not lower relational expressions again. + +Native summary snapshots use `native_batch_v1`, with an explicit storage tag +separate from legacy sketch frames. The payload preserves the physical schema, +Float64 values, typed heap identities and the summary codec. Legacy sketch +readers reject this tag. Existing installed schemas may keep a nonempty, unique +subset of supported encodings when new decoders are added. + +`SketchStore::publish_native_summary_output` publishes one summary row for one +group/window through the existing immutable commit path. It requires the complete +durable raw input cohort and validates the installed family and group. Repeated +publication of the same lineage reuses the existing commit. + +`read_native_summary_output` resolves the installed output/group through the +persistent series resolver without allocating an identity. It validates the +plan version, definition, exact window, completeness, encoding, physical schema, +group values and read budget before returning a batch to the physical executor. +The supported path reads a complete immutable window; pane composition remains +an explicitly planned physical operation. + +The storage integration test covers durable per-series sums, native quantile +state construction, publication, bound readout and restart of both the store and +resolver. Automatic PromQL physical-candidate installation, Rate-to-heap storage +execution and spatial latest-value heap execution remain pending. This test does +not establish those paths. From a2320ffca0d75956dc91ceb30afda7b4189088da Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 21:29:31 +0000 Subject: [PATCH 18/45] perf: index bound SDS reads by output and window range --- .../sketch_db/index/maintenance.rs | 5 +- .../sketch_db/persistence/cache.rs | 4 +- .../sketch_db/persistence/part.rs | 122 ++++++++++++++---- .../catalog-physical-plan-runtime.md | 7 + 4 files changed, 107 insertions(+), 31 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index bf31f27d8..7e355c417 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -386,10 +386,7 @@ impl SketchStore { .part_cache .get_or_load(part.part_id) .map_err(|e| e.to_string())?; - for record in reader.index_records() { - if record.agg_id != sid || record.start_ts < start_ms || record.end_ts > end_ms { - continue; - } + for record in reader.window_records(sid, start_ms, end_ms) { let entry = reader.load_entry(&record).map_err(|e| e.to_string())?; if entry.label.as_ref().map_or(0, |label| label.labels.len()) != keys.len() { return Err("immutable input label arity differs from its descriptor".into()); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/cache.rs b/data_plane/src/storage_engines/sketch_db/persistence/cache.rs index 0c235a451..ceda43c88 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/cache.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/cache.rs @@ -34,9 +34,9 @@ impl PartCache { Some( Cache::builder() .weigher(|_k: &PartId, v: &LoadedPart| -> u32 { - // Weight = sum of mmap'd bytes. Moka's weigher + // Include mmap bytes and the in-memory window index. The weigher // returns u32, so clamp large parts. - let len = v.meta.data_len + v.meta.index_len; + let len = v.resident_bytes(); len.min(u32::MAX as u64) as u32 }) .max_capacity(byte_budget) diff --git a/data_plane/src/storage_engines/sketch_db/persistence/part.rs b/data_plane/src/storage_engines/sketch_db/persistence/part.rs index c760668cb..6eae5f043 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/part.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/part.rs @@ -430,13 +430,15 @@ pub struct IndexRecord { pub data_offset: u64, } -/// mmap-backed reader for a single part. Cheap to construct (three -/// mmaps + one header parse), safe to share across threads via `Arc`. +/// mmap-backed reader with a sorted output/window index built once on open. +/// Safe to share across threads via `Arc`. pub struct PartReader { pub meta: PartMeta, pub part_dir: PathBuf, data_mmap: Arc, index_mmap: Arc, + // Older parts preserve flush order, so keep a separate sorted lookup. + window_index: Vec, } impl std::fmt::Debug for PartReader { @@ -479,12 +481,25 @@ impl PartReader { ))); } - Ok(Self { + let mut reader = Self { meta, part_dir: part_dir.to_path_buf(), data_mmap: Arc::new(data_mmap), index_mmap: Arc::new(index_mmap), - }) + window_index: Vec::new(), + }; + let mut positions: Vec<_> = (0..reader.meta.num_entries as usize).collect(); + positions.sort_unstable_by_key(|&position| { + let record = reader.index_record(position); + ( + record.agg_id, + record.start_ts, + record.end_ts, + record.data_offset, + ) + }); + reader.window_index = positions; + Ok(reader) } /// Read and verify just the meta.bin header (cheap — 64 bytes). @@ -537,28 +552,51 @@ impl PartReader { }) } - /// Return all index records. Small (32 B × num_entries) — cheap to - /// materialize. - pub fn index_records(&self) -> Vec { - let n = self.meta.num_entries as usize; - let mut out = Vec::with_capacity(n); - for i in 0..n { - let off = i * INDEX_ENTRY_SIZE; - let agg_id = u64::from_le_bytes(self.index_mmap[off..off + 8].try_into().unwrap()); - let start_ts = - u64::from_le_bytes(self.index_mmap[off + 8..off + 16].try_into().unwrap()); - let end_ts = - u64::from_le_bytes(self.index_mmap[off + 16..off + 24].try_into().unwrap()); - let data_offset = - u64::from_le_bytes(self.index_mmap[off + 24..off + 32].try_into().unwrap()); - out.push(IndexRecord { - agg_id, - start_ts, - end_ts, - data_offset, - }); + fn index_record(&self, position: usize) -> IndexRecord { + let off = position * INDEX_ENTRY_SIZE; + let read = + |offset| u64::from_le_bytes(self.index_mmap[offset..offset + 8].try_into().unwrap()); + IndexRecord { + agg_id: read(off), + start_ts: read(off + 8), + end_ts: read(off + 16), + data_offset: read(off + 24), } - out + } + + /// Preserve disk order for callers that need to inspect the entire part. + pub fn index_records(&self) -> Vec { + (0..self.meta.num_entries as usize) + .map(|position| self.index_record(position)) + .collect() + } + + /// Find records contained in the requested window under one stored-output + /// prefix. Duplicate windows remain visible so callers can reject ambiguity. + pub fn window_records( + &self, + agg_id: u64, + start_ts: u64, + end_ts: u64, + ) -> impl Iterator + '_ { + let first = self.window_index.partition_point(|&position| { + let record = self.index_record(position); + (record.agg_id, record.start_ts) < (agg_id, start_ts) + }); + self.window_index[first..] + .iter() + .map(|&position| self.index_record(position)) + .take_while(move |record| record.agg_id == agg_id && record.start_ts <= end_ts) + .filter(move |record| record.end_ts <= end_ts) + } + + pub fn resident_bytes(&self) -> u64 { + (self.data_mmap.len() as u64) + .saturating_add(self.index_mmap.len() as u64) + .saturating_add( + (self.window_index.capacity() as u64) + .saturating_mul(std::mem::size_of::() as u64), + ) } /// Resolve a single index record into a [`SnapshotEntry`] by reading @@ -658,6 +696,40 @@ mod tests { } } + // Old parts can be unsorted; lookup must isolate the prefix/range while + // preserving duplicate records for the immutable reader's ambiguity check. + #[test] + fn window_lookup_handles_unsorted_parts_duplicates_and_recovery() { + let tmp = TempDir::new().unwrap(); + let part_dir = tmp.path().join("lookup"); + let mut snapshots = Vec::new(); + for id in [99, 42, 1] { + let mut snapshot = make_snapshot(); + snapshot.agg_id = id; + snapshot.entries.reverse(); + if id == 42 { + snapshot.entries.push(snapshot.entries[1].clone()); + } + snapshots.push(snapshot); + } + PartWriter::write_part(&part_dir, 1, &snapshots).unwrap(); + for _ in 0..2 { + let reader = PartReader::open(&part_dir).unwrap(); + let records = reader.window_records(42, 1_000, 1_500).collect::>(); + assert_eq!(records.len(), 2); + assert!(records.iter().all(|record| record.agg_id == 42 + && record.start_ts == 1_000 + && record.end_ts == 1_500)); + assert_ne!(records[0].data_offset, records[1].data_offset); + assert_eq!(reader.window_records(42, 1_500, 2_000).count(), 1); + assert_eq!(reader.window_records(43, 0, u64::MAX).count(), 0); + assert_eq!(reader.window_records(42, 1_001, 1_999).count(), 0); + assert_eq!(reader.window_records(42, 2_000, 1_000).count(), 0); + assert_eq!(reader.index_records()[0].agg_id, 99); + assert!(reader.resident_bytes() > reader.meta.data_len + reader.meta.index_len); + } + } + #[test] fn part_round_trip_writes_and_reads_back() { let tmp = TempDir::new().unwrap(); diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 48ee6bbe6..bfdbe06b7 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -59,3 +59,10 @@ state construction, publication, bound readout and restart of both the store and resolver. Automatic PromQL physical-candidate installation, Rate-to-heap storage execution and spatial latest-value heap execution remain pending. This test does not establish those paths. + +Bound frozen reads resolve the stored-output/group prefix through the persistent +series resolver. Each cached part builds a sorted output/window index once on +open, including older parts written in flush order. Range reads inspect only the +matching prefix and start-time range, then validate end times and exact coverage. +Duplicate windows remain visible and are rejected as ambiguous. The index memory +counts toward the part-cache budget; the on-disk part format is unchanged. From 01584c0c10bd968379a4170a499200949b656db5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 21:30:34 +0000 Subject: [PATCH 19/45] feat: trace native SDS publication and bound recovery --- .../src/storage_engines/sketch_db/index/native.rs | 7 +++++++ docs/runtime-debugging.md | 14 +++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index 1493cddaf..4b87e1617 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -105,6 +105,9 @@ impl AggregateCore for NativeSummaryOutput { impl SketchStore { /// Publish a finalized native result only after the complete raw input /// cohort is durable. Existing publication fences prevent duplicate commits. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "sds.publish_native", start_ms = output.start_timestamp, + end_ms = output.end_timestamp, max_bytes), err)] pub fn publish_native_summary_output( &self, resolver: &SeriesIdResolver, @@ -168,6 +171,10 @@ impl SketchStore { /// Resolve the installed output/group prefix without minting a new handle, /// then read exactly the requested immutable window and validate its format. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "sds.read_native", plan_version = address.plan_version, + start_ms = address.window.start_ms, end_ms = address.window.end_ms, + max_bytes), err)] pub fn read_native_summary_output( &self, resolver: &SeriesIdResolver, diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 25628aaa6..3a5d97c90 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -67,7 +67,7 @@ A missing candidate is not evidence that its placement costs more. For example, the retained #728 grouped-rate fixture exports per-series Rate state plus query-time Sum and an exact fallback; it does not export a precomputed grouped Rate-result candidate. The selected plan therefore cannot establish that query-time -Sum beats that absent alternative. Search coverage is scoped to the declared +Sum beats that absent candidate. Search coverage is scoped to the declared inventory, never a claim of exhaustive physical optimization. ### Installed SQL physical DAG stages @@ -83,3 +83,15 @@ or sketch payloads. They use the existing `asap_runtime_debug` target. These stages describe the installed SQL path. They do not establish that the remaining PromQL candidate-selection and maintenance paths have migrated to the same physical handoff. + +### Native SDS publication and recovery + +`sds.publish_native` covers complete-cohort validation and immutable publication; +`sds.read_native` covers the installed output lookup and native decoding. Their +fields include the requested window and byte budget; read spans also include the +plan version. Group labels and summary payloads are not logged. Successful return +from an idempotent publication can mean the output was already committed; the +publication receipt remains authoritative. + +These spans instrument the native storage API, including its recovery E2E. They +do not imply that all PromQL maintenance candidates use that API yet. From 8f12a8e0bfad245c235978c0eecc9130657aa1c2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 22:37:57 +0000 Subject: [PATCH 20/45] feat: install Planner native physical program for current-series TopK --- Cargo.lock | 14 +-- Cargo.toml | 12 +- control_plane/src/physical/compiler.rs | 4 + .../src/physical/maintained_population.rs | 49 ++++++++ control_plane/src/physical/workload_cost.rs | 9 +- .../src/physical/workload_cost/automatic.rs | 18 ++- crates/asap_types/src/query_plan.rs | 4 +- .../src/query_plan/current_series.rs | 10 +- crates/asap_types/src/query_plan/native.rs | 81 ++++++++++++++ .../asap_query_engine/logical_dag.rs | 8 ++ .../logical_dag/native_values.rs | 105 ++++++++++++++++++ .../sketch_db/current_series.rs | 9 ++ .../tests/support/current_series_process.rs | 22 ++++ .../catalog-physical-plan-runtime.md | 10 ++ 14 files changed, 336 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53360cc36..ea20d207f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=e2c67fc921df88b72224d0de1bbfe6cd8cc725a0#e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index 74a9f2478..8041344b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e2c67fc921df88b72224d0de1bbfe6cd8cc725a0" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 7c3f97533..79813ea41 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2138,6 +2138,10 @@ impl DeploymentPlanCompiler { if frontend == QueryFrontend::MetricsQl { entry.language = crate::query_plan::QueryLanguage::MetricsQl; } + super::maintained_population::install_native_topk( + &mut entry, + &query.selected_plan_root, + )?; let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); if query_entries.insert(catalog_key, entry).is_some() { return Err(CompileError::Query { diff --git a/control_plane/src/physical/maintained_population.rs b/control_plane/src/physical/maintained_population.rs index b5b7664e6..7bb955031 100644 --- a/control_plane/src/physical/maintained_population.rs +++ b/control_plane/src/physical/maintained_population.rs @@ -109,3 +109,52 @@ pub(super) fn operator( readout, })) } + +/// The maintained population is a deployment source; ranking is compiled by +/// Planner before this candidate is priced or installed. +pub(super) fn install_native_topk( + entry: &mut asap_types::query_plan::QueryPlanEntry, + selected: &std::rc::Rc, +) -> Result<(), CompileError> { + use asap_types::query_plan::QueryPlanNode; + let Some(QueryPlanNode::Logical { + operator: + ResidualQueryOperator::CurrentSeries { + population, + readout: SeriesReadout::TopK { .. }, + }, + .. + }) = entry.nodes.get(&entry.root) + else { + return Ok(()); + }; + if population.grouping.without { + return Ok(()); + } + let compiled = + asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout( + selected, + ) + .map_err(|error| CompileError::Query { + query_id: entry.query_id.clone(), + reason: error.to_string(), + })?; + let encoded = compiled.encode().map_err(|error| CompileError::Query { + query_id: entry.query_id.clone(), + reason: error.to_string(), + })?; + entry.physical_dag = Some( + serde_json::from_slice(&encoded) + .map_err(|error| CompileError::Snapshot(error.to_string()))?, + ); + let Some(QueryPlanNode::Logical { + operator: ResidualQueryOperator::CurrentSeries { readout, .. }, + .. + }) = entry.nodes.get_mut(&entry.root) + else { + unreachable!() + }; + *readout = SeriesReadout::Snapshot; + entry.recover_population_physical_dag()?; + Ok(()) +} diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 61ad7cfb0..53bd8c978 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -309,7 +309,7 @@ pub fn manifest( for node_id in entry.topological_order()? { add( format!("query:{}:{}", entry.query_id, node_id.0), - json!({"node": entry.nodes[&node_id], "query": entry.canonical_query, "instant": entry.instant}), + json!({"node": entry.nodes[&node_id], "query": entry.canonical_query, "instant": entry.instant, "physical_dag": entry.physical_dag}), "query_evaluation", evaluations, ); @@ -572,6 +572,12 @@ fn compile_candidate_for_pricing( }) .collect::>(); placement.sort(); + let native_programs = plan + .query_plan + .entries + .values() + .map(|entry| (&entry.query_id, &entry.physical_dag)) + .collect::>(); description.physical_candidate_id = description .candidate_id @@ -584,6 +590,7 @@ fn compile_candidate_for_pricing( bindings, placement, &plan.precompute_plan.ingest, + native_programs, ), ) }); diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index 7257bc9be..e6b3b634c 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -505,7 +505,16 @@ pub(super) fn estimate( .. } => { output_rows = cardinality; - cardinality * CPU_PER_ITEM + if entry.population_snapshot().is_some() { + detail = json!({"input_rows": cardinality, + "physical_program": entry.physical_dag, + "workspace_bytes_bound": cardinality * SERIES_BYTES * 3.0, + "formula": "rows * (log2(max(rows, 2)) + 2) * cpu_seconds_per_item", + "cpu_seconds_per_item": CPU_PER_ITEM}); + cardinality * (cardinality.max(2.0).log2() + 2.0) * CPU_PER_ITEM + } else { + cardinality * CPU_PER_ITEM + } } Node::RelationalJoin { inputs, @@ -572,13 +581,18 @@ pub(super) fn estimate( } else { BTreeSet::new() }; + let workspace_byte_seconds = if entry.population_snapshot().is_some() { + cardinality * SERIES_BYTES * 3.0 * cpu * evaluations + } else { + 0.0 + }; detail["evaluations"] = json!(evaluations); detail["output_rows_bound"] = json!(output_rows); insert( id, resources( cpu * evaluations, - 0.0, + workspace_byte_seconds, network * evaluations, ids.into_iter().collect(), detail, diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index a11aa0f2f..bbf0ceb33 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -395,7 +395,9 @@ impl QueryPlanEntry { self.query_id, self.root.0 ))); } - if self.relation_output_schema()?.is_some() || self.physical_dag.is_some() { + if self.population_snapshot().is_some() { + self.recover_population_physical_dag()?; + } else if self.relation_output_schema()?.is_some() || self.physical_dag.is_some() { self.recover_relational_physical_dag()?; } for (id, node) in &self.nodes { diff --git a/crates/asap_types/src/query_plan/current_series.rs b/crates/asap_types/src/query_plan/current_series.rs index c99498457..9aed86961 100644 --- a/crates/asap_types/src/query_plan/current_series.rs +++ b/crates/asap_types/src/query_plan/current_series.rs @@ -49,8 +49,14 @@ impl SeriesPopulation { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum SeriesReadout { - Quantile { q: f64 }, - TopK { k: u64 }, + /// All eligible members for a Planner-compiled physical readout. + Snapshot, + Quantile { + q: f64, + }, + TopK { + k: u64, + }, Sum, Count, Average, diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index e05572f78..da3637e0a 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -245,6 +245,87 @@ impl QueryPlanEntry { } } +impl QueryPlanEntry { + pub fn population_snapshot(&self) -> Option<¤t_series::SeriesPopulation> { + if self.nodes.len() != 1 { + return None; + } + match self.nodes.get(&self.root) { + Some(QueryPlanNode::Logical { + operator: + residual::ResidualQueryOperator::CurrentSeries { + population, + readout: current_series::SeriesReadout::Snapshot, + }, + inputs, + }) if inputs.is_empty() => Some(population), + _ => None, + } + } + + /// Recover an installed population readout; the source binds the complete + /// maintained vector, while the physical program owns ranking and limiting. + pub fn recover_population_physical_dag(&self) -> Result { + let invalid = |message: &str| QueryPlanError::Invalid(message.into()); + let population = self + .population_snapshot() + .ok_or_else(|| invalid("missing population source binding"))?; + population.validate()?; + let value = self + .physical_dag + .as_ref() + .ok_or_else(|| invalid("missing installed population physical DAG"))?; + let dag = CompiledPhysicalDag::decode( + &serde_json::to_vec(value) + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?, + ) + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + let inputs = dag.input_contracts().collect::>(); + let [(_, input)] = inputs.as_slice() else { + return Err(invalid("population physical DAG requires one source")); + }; + let [root] = dag.roots() else { + return Err(invalid("population physical DAG requires one root")); + }; + let output = dag + .output_contract(*root) + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + if input.schema != output.schema { + return Err(invalid( + "population ranking must preserve complete source rows", + )); + } + use planner_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; + let fields = &input.schema.fields; + let column = |name: &str, dtype: DataType| { + fields.iter().any(|field| { + field.name == name + && field.dtype == SummaryFamilyType::Plain(dtype.clone()) + && !field.nullable + }) + }; + if !column( + asap_physical_operators::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN, + DataType::Utf8, + ) || !column("value", DataType::Float64) + || input.schema.time_index.is_none_or(|index| { + fields[index].dtype != SummaryFamilyType::Plain(DataType::Timestamp) + }) + || population.grouping.without + || population.grouping.labels.iter().any(|label| { + !fields.iter().any(|field| { + &field.name == label && field.dtype == SummaryFamilyType::Plain(DataType::Utf8) + }) + }) + { + return Err(invalid( + "population physical source loses identity, value, timestamp or grouping", + )); + } + Ok(dag) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 96eaf5ccb..06b9af019 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -96,6 +96,14 @@ pub(crate) fn execute_installed( where F: FnMut(QueryNodeId, u64) -> Result, { + if entry.population_snapshot().is_some() { + if !leaves.is_empty() { + return Err(miss( + "population physical input must use its installed source binding", + )); + } + return native_values::execute_population(entry, at, callback); + } execute_values(entry, leaves, at, callback) } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index cdc8663a2..c7471ea9e 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -129,3 +129,108 @@ pub(super) fn semi_join( .map_err(|e| miss(e.to_string()))?; output(values, result) } + +pub(super) fn execute_population( + entry: &asap_types::query_plan::QueryPlanEntry, + at: u64, + mut callback: F, +) -> Result< + ( + crate::query_engines::query_result::QueryResult, + super::ExecutionStats, + ), + EngineError, +> +where + F: FnMut( + asap_types::query_plan::QueryNodeId, + u64, + ) -> Result, +{ + use crate::{ + query_engines::query_result::{InstantVectorElement, QueryResult}, + storage_engines::types::KeyByLabelValues, + }; + use asap_physical_operators::physical_planner::{ + promql_rows::{decode_series_identity, series_row, SERIES_IDENTITY_COLUMN}, + Source, + }; + use futures::{executor::block_on, StreamExt}; + use std::collections::BTreeMap; + let program = entry + .recover_population_physical_dag() + .map_err(|error| miss(error.to_string()))?; + let (input_id, input) = program.input_contracts().next().unwrap(); + let at_signed = i64::try_from(at).map_err(|_| miss("evaluation timestamp overflow"))?; + let values = super::vector(super::from_result(callback(entry.root, at)?)?)?; + let rows = values + .into_iter() + .map(|(labels, value)| { + series_row(&input.schema, &labels, at_signed, value) + .map_err(|error| miss(error.to_string())) + }) + .collect::, _>>()?; + let batch = + Batch::try_new(input.schema.clone(), rows).map_err(|error| miss(error.to_string()))?; + let source = Operator::source(input.schema.clone(), vec![batch]) + .map_err(|error| miss(error.to_string()))?; + let graph = program + .instantiate(BTreeMap::from([(input_id, Box::new(source) as Source<'_>)])) + .map_err(|error| miss(error.to_string()))?; + let context = dag::RunContext::new( + dag::Scope::Query { + evaluation_time_ms: at_signed, + revision: 0, + }, + dag::Limits { + max_bytes: entry.population_snapshot().unwrap().max_bytes as usize, + ..dag::Limits::default() + }, + ) + .map_err(|error| miss(error.to_string()))?; + let mut stream = graph + .execute(program.roots(), context) + .map_err(|error| miss(error.to_string()))? + .remove(0); + let values = block_on(async { + let mut values = Vec::new(); + while let Some(batch) = stream.next().await { + let batch = batch.map_err(|error| miss(error.to_string()))?; + let identity = batch + .schema() + .fields + .iter() + .position(|field| field.name == SERIES_IDENTITY_COLUMN) + .ok_or_else(|| miss("physical output loses series identity"))?; + let value = batch + .schema() + .fields + .iter() + .position(|field| field.name == "value") + .ok_or_else(|| miss("physical output loses sample value"))?; + for row in batch.rows() { + let (Value::Utf8(encoded), Value::Float64(sample)) = (&row[identity], &row[value]) + else { + return Err(miss("invalid population physical output")); + }; + let labels = + decode_series_identity(encoded).map_err(|error| miss(error.to_string()))?; + values.push( + InstantVectorElement::new( + KeyByLabelValues::new_with_labels(labels.values().cloned().collect()), + *sample, + ) + .with_label_keys_override(labels.into_keys().collect()), + ); + } + } + Ok(values) + })?; + Ok(( + QueryResult::vector(values, at), + super::ExecutionStats { + summary_readout_evaluations: 1, + ..Default::default() + }, + )) +} diff --git a/data_plane/src/storage_engines/sketch_db/current_series.rs b/data_plane/src/storage_engines/sketch_db/current_series.rs index 1887647ff..d3885dc06 100644 --- a/data_plane/src/storage_engines/sketch_db/current_series.rs +++ b/data_plane/src/storage_engines/sketch_db/current_series.rs @@ -194,6 +194,14 @@ impl Population { } } fn read(&mut self, readout: &SeriesReadout) -> Vector { + if matches!(readout, SeriesReadout::Snapshot) { + return self + .groups + .values() + .flat_map(|group| group.ordered.iter()) + .map(|member| (member.labels.clone(), member.value)) + .collect(); + } let mut result = vec![]; for (labels, group) in &mut self.groups { if group.cached.is_none() { @@ -221,6 +229,7 @@ impl Population { } let (values, top, sum, average) = group.cached.as_ref().unwrap(); match readout { + SeriesReadout::Snapshot => unreachable!("snapshot returned above"), SeriesReadout::Quantile { q } => { // `values` is only populated for a quantile-carrying population. // `ResidualQueryOperator::validate` rejects the mismatched pairing at diff --git a/data_plane/tests/support/current_series_process.rs b/data_plane/tests/support/current_series_process.rs index bc7f1140d..0456f2faa 100644 --- a/data_plane/tests/support/current_series_process.rs +++ b/data_plane/tests/support/current_series_process.rs @@ -103,6 +103,28 @@ async fn current_series_quantiles_topk_share_and_replace_values() { }); let planned = snapshot.clone().compile_promql().unwrap(); for entry in planned.query_plan.entries.values() { + if entry.canonical_query.starts_with("topk") { + assert!( + entry.population_snapshot().is_some(), + "TopK must bind a complete population source" + ); + let restored: asap_types::query_plan::QueryPlanEntry = + serde_json::from_slice(&serde_json::to_vec(entry).unwrap()).unwrap(); + let physical = restored.recover_population_physical_dag().unwrap(); + let encoded = String::from_utf8(physical.encode().unwrap()).unwrap(); + assert!(encoded.contains("Sort") && encoded.contains("Limit")); + assert!(encoded.contains("$promql_series_identity")); + assert!( + !encoded.contains("CurrentSeries"), + "the stored population boundary must not be recomputed" + ); + let mut missing = restored.clone(); + missing.physical_dag = None; + assert!(missing.recover_population_physical_dag().is_err()); + let mut version = restored; + version.physical_dag.as_mut().unwrap()["version"] = 999.into(); + assert!(version.recover_population_physical_dag().is_err()); + } for node in entry.nodes.values() { if let asap_types::query_plan::QueryPlanNode::Logical { operator: diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index bfdbe06b7..10dcb744b 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -66,3 +66,13 @@ open, including older parts written in flush order. Range reads inspect only the matching prefix and start-time range, then validate end times and exact coverage. Duplicate windows remain visible and are rejected as ambiguous. The index memory counts toward the part-cache budget; the on-disk part format is unchanged. + +For selected explicit-`by` current-series TopK, the population store now supplies +all eligible members through a snapshot binding. Planner compiles the ranking +above that boundary; the QueryPlan persists that physical program before +candidate pricing and activation. Serving recovers its operators and supplies +full-label native rows without parsing or lowering the query. Automatic cost +estimates include sorting CPU and temporary workspace; provider quote manifests +include the physical program itself. Other population readouts retain their +existing paths. This path does not imply that spatial heap candidates or buffered +Rate-to-heap persistence are deployed yet. From 1518443664bee8d3d9771863af77b7e1fa5d08a7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 22:42:54 +0000 Subject: [PATCH 21/45] feat: trace installed current-series native execution stages --- control_plane/src/physical/maintained_population.rs | 2 ++ crates/asap_types/src/query_plan/native.rs | 2 ++ .../asap_query_engine/logical_dag/native_values.rs | 13 ++++++++++--- docs/runtime-debugging.md | 7 +++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/control_plane/src/physical/maintained_population.rs b/control_plane/src/physical/maintained_population.rs index 7bb955031..88792817e 100644 --- a/control_plane/src/physical/maintained_population.rs +++ b/control_plane/src/physical/maintained_population.rs @@ -112,6 +112,8 @@ pub(super) fn operator( /// The maintained population is a deployment source; ranking is compiled by /// Planner before this candidate is priced or installed. +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.compile_install", query_id = %entry.query_id, input_kind = "current_series_snapshot"), err)] pub(super) fn install_native_topk( entry: &mut asap_types::query_plan::QueryPlanEntry, selected: &std::rc::Rc, diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index 2e0ca4de2..90afcec5c 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -269,6 +269,8 @@ impl QueryPlanEntry { /// Recover an installed population readout; the source binds the complete /// maintained vector, while the physical program owns ranking and limiting. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.recover_validate", query_id = %self.query_id, input_kind = "current_series_snapshot"), err)] pub fn recover_population_physical_dag(&self) -> Result { let invalid = |message: &str| QueryPlanError::Invalid(message.into()); let population = self diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index c7471ea9e..3ab4325c4 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -130,6 +130,8 @@ pub(super) fn semi_join( output(values, result) } +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.execute", query_id = %entry.query_id, evaluation_time_ms = at, input_kind = "current_series_snapshot"), err)] pub(super) fn execute_population( entry: &asap_types::query_plan::QueryPlanEntry, at: u64, @@ -174,9 +176,14 @@ where Batch::try_new(input.schema.clone(), rows).map_err(|error| miss(error.to_string()))?; let source = Operator::source(input.schema.clone(), vec![batch]) .map_err(|error| miss(error.to_string()))?; - let graph = program - .instantiate(BTreeMap::from([(input_id, Box::new(source) as Source<'_>)])) - .map_err(|error| miss(error.to_string()))?; + let graph = { + let _binding = tracing::debug_span!(target: "asap_runtime_debug", "physical_input_binding", + stage = "physical.bind_inputs", input_count = 1, input_kind = "current_series_snapshot") + .entered(); + program + .instantiate(BTreeMap::from([(input_id, Box::new(source) as Source<'_>)])) + .map_err(|error| miss(error.to_string()))? + }; let context = dag::RunContext::new( dag::Scope::Query { evaluation_time_ms: at_signed, diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 3a5d97c90..1e5affb01 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -95,3 +95,10 @@ publication receipt remains authoritative. These spans instrument the native storage API, including its recovery E2E. They do not imply that all PromQL maintenance candidates use that API yet. + +Current-series TopK uses these same stages: installation compiles the Planner +readout program, recovery validates the persisted program, binding supplies the +complete eligible population, and native execution performs ranking and limiting. +These spans use `input_kind=current_series_snapshot`; label sets and sample +values are not recorded. They describe the exact ranking path, not heap storage +or request-time physical compilation. From f247110eab76b72444448545306e3d1749d24d2a Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 23:09:51 +0000 Subject: [PATCH 22/45] feat: admit price and execute Planner spatial heap candidates --- Cargo.lock | 14 +- Cargo.toml | 12 +- control_plane/src/main.rs | 2 +- control_plane/src/physical/compiler.rs | 83 +++++++- .../src/physical/maintained_population.rs | 47 ++++- control_plane/src/physical/workload_cost.rs | 14 +- .../src/physical/workload_cost/automatic.rs | 79 +++++++- control_plane/tests/native_snapshot_topk.rs | 180 +++++++++++++++++ .../asapquery_compatibility_process_e2e.rs | 3 + .../tests/support/spatial_heap_process.rs | 184 ++++++++++++++++++ .../evidence-dependent-candidates.md | 10 +- .../catalog-physical-plan-runtime.md | 22 ++- 12 files changed, 602 insertions(+), 48 deletions(-) create mode 100644 control_plane/tests/native_snapshot_topk.rs create mode 100644 data_plane/tests/support/spatial_heap_process.rs diff --git a/Cargo.lock b/Cargo.lock index ea20d207f..4913b5488 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab#a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index 8041344b7..aeb0dff27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a2bf21fd7c8e9eda344ff2fed7f9d8c66dd9b5ab" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 8655f6a8d..77c6887c8 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -986,7 +986,7 @@ mod api_tests { .cost_comparison .as_ref() .expect("HTTP must compare automatically priced candidates"); - assert_eq!(report.model_version, "backend-workload-resources-v1"); + assert_eq!(report.model_version, "backend-workload-resources-v2"); assert!(report .candidate_evaluations .iter() diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 79813ea41..16fe06908 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -237,6 +237,10 @@ pub struct ScopedAccuracyEvidence { pub hydra_shared_grid_collision_bound: Option, #[serde(default)] pub hydra_shared_grid_failure_probability: Option, + /// Enforced upper bound across all partition/item identities for this query + /// snapshot. An observed cardinality estimate is not this contract. + #[serde(default)] + pub topk_max_distinct_items: Option, #[serde(default)] pub topk_selected_lower_bound: Option, #[serde(default)] @@ -279,6 +283,9 @@ impl ScopedAccuracyEvidence { _ => false, }; let valid_stats = valid_topk + && self + .topk_max_distinct_items + .is_none_or(|n| n > 0 && n <= (1_u64 << 53)) && self .hll .as_ref() @@ -654,6 +661,10 @@ struct QueryEvidence<'a> { } impl AccuracyEvidenceProvider for QueryEvidence<'_> { + fn topk_max_distinct_items(&self, _: &QueryExpr) -> Option { + self.scoped + .and_then(|evidence| evidence.topk_max_distinct_items) + } fn quantile_input_domain( &self, operand: &QueryExpr, @@ -946,6 +957,56 @@ impl BackendLocalPlanningInput { self.environment.observed_at_unix_ms, Some(&mut candidate_roots), )?; + // Resolve physical row identity before asking Planner for snapshot heap + // candidates. Keep canonical query semantics and exact candidates intact. + for (index, (query, root)) in queries.iter().zip(&canonical_roots).enumerate() { + let Ok(typed) = + asap_physical_operators::physical_planner::promql_rows::with_series_identity(root) + else { + continue; + }; + let evidence = QueryEvidence { + topk: topk_evidence_by_id.get(&query.query_id), + scoped: scoped_evidence_by_id.get(&query.query_id), + now_ms: self.environment.observed_at_unix_ms, + }; + let strategy = + asap_aware_mapping::SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( + &asap_aware_mapping::cost_model::DefaultCostModel, + &asap_aware_mapping::accuracy::DefaultAccuracyModel, + &asap_aware_mapping::accuracy::EqualSplitAllocator, + &evidence, + ); + let proposed = + strategy.current_series_topk_candidates(&Rc::new(typed), &query.accuracy_target); + for candidate in proposed.candidates { + let asap_aware_mapping::Replacement::Summary(root) = candidate.replacement else { + continue; + }; + let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root); + match compiled { + Ok(program) => { + planner_selection_trace.push(serde_json::json!({ + "stage": "planner.physical_candidate", "query_id": query.query_id, + "logical_root_id": crate::planner_selection::explained_root_id(&root, &query.accuracy_target), + "rationale": candidate.rationale, "physical_dag": serde_json::from_slice::(&program.encode().map_err(|error| CompileError::Snapshot(error.to_string()))?).map_err(|error| CompileError::Snapshot(error.to_string()))?, + "guarantee": root.guarantee, + })); + candidate_roots.push(vec![(index, root)]); + } + Err(error) => planner_selection_trace.push(serde_json::json!({ + "stage": "planner.physical_candidate", "query_id": query.query_id, + "status": "unsupported", "reason": error.to_string(), + })), + } + } + for rejected in proposed.rejected { + planner_selection_trace.push(serde_json::json!({ + "stage": "planner.physical_candidate", "query_id": query.query_id, + "status": "rejected", "reason": rejected.error.to_string(), + })); + } + } for query in &mut queries { prepare_window_implementations( query, @@ -1344,14 +1405,18 @@ impl DeploymentPlanCompiler { } let node = query.selected_plan_root.clone(); reject_uncertified_readouts(&query.query_id, &node, environment.target)?; - let selected = collect_selected_materializations( - &node, - request.allow_mixed_summary_and_exact_execution, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; + let selected = if super::maintained_population::supported_node(&node) { + Vec::new() + } else { + collect_selected_materializations( + &node, + request.allow_mixed_summary_and_exact_execution, + ) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, + })? + }; if !selected.is_empty() { reject_uncertified_readouts(&query.query_id, &node, environment.target)?; } @@ -5473,6 +5538,7 @@ pub(crate) mod tests { input_row_count: Some(10), hydra_shared_grid_collision_bound: None, hydra_shared_grid_failure_probability: None, + topk_max_distinct_items: None, topk_selected_lower_bound: None, topk_excluded_upper_bound: None, topk_interval_failure_probability: None, @@ -5563,6 +5629,7 @@ pub(crate) mod tests { input_row_count: None, hydra_shared_grid_collision_bound: None, hydra_shared_grid_failure_probability: None, + topk_max_distinct_items: None, topk_selected_lower_bound: None, topk_excluded_upper_bound: None, topk_interval_failure_probability: None, diff --git a/control_plane/src/physical/maintained_population.rs b/control_plane/src/physical/maintained_population.rs index 7bb955031..a7b877428 100644 --- a/control_plane/src/physical/maintained_population.rs +++ b/control_plane/src/physical/maintained_population.rs @@ -8,23 +8,50 @@ use planner_types::post_asap::{ maintained_population::*, SummaryExpr, SummaryNode, ValueOperation, }; -fn selected(node: &SummaryNode) -> Option<(&MaintainedPopulation, &PopulationReadout)> { - let SummaryExpr::ValueOperation { +fn selected(node: &SummaryNode) -> Option<(MaintainedPopulation, PopulationReadout)> { + if let SummaryExpr::ValueOperation { child, operation: ValueOperation::ReadPopulation { readout }, .. } = &node.expr - else { - return None; - }; + { + if let SummaryExpr::ValueOperation { + operation: ValueOperation::MaintainPopulation { population }, + .. + } = &child.expr + { + return Some((population.clone(), readout.clone())); + } + } + // The source remains a maintained population when Planner places a heap, + // projection and ranking above it. Backend binds that source only. let SummaryExpr::ValueOperation { - operation: ValueOperation::MaintainPopulation { population }, + operation: ValueOperation::Limit { n, offset: 0, .. }, .. - } = &child.expr + } = &node.expr else { return None; }; - Some((population, readout)) + let dag = + planner_types::post_asap::compile_executable_dag(&std::rc::Rc::new(node.clone())).ok()?; + let populations = dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + planner_types::post_asap::ExecutableOperatorPayload::Value { + operation: ValueOperation::MaintainPopulation { population }, + } => Some(population), + _ => None, + }) + .collect::>(); + let [population] = populations.as_slice() else { + return None; + }; + asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout( + &std::rc::Rc::new(node.clone()), + ) + .ok()?; + Some(((*population).clone(), PopulationReadout::TopK { k: *n })) } pub(super) fn supported_node(node: &SummaryNode) -> bool { @@ -55,7 +82,7 @@ pub(super) fn operator( .iter() .filter_map(|q| { selected(&q.selected_plan_root) - .map(|(p, _)| serde_json::to_string(p).expect("typed population serializes")) + .map(|(p, _)| serde_json::to_string(&p).expect("typed population serializes")) }) .collect(); let max_bytes = request @@ -97,7 +124,7 @@ pub(super) fn operator( .min(input.lookback_ms), }; population.validate()?; - let readout = match readout { + let readout = match &readout { PopulationReadout::Quantile { q } => SeriesReadout::Quantile { q: *q }, PopulationReadout::TopK { k } => SeriesReadout::TopK { k: *k as u64 }, PopulationReadout::Sum => SeriesReadout::Sum, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 53bd8c978..bdb77a003 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -874,18 +874,26 @@ fn enumerate_frontier_candidates( // workload roots. Preserve each priced temporal candidate and mask. let maintained: Vec<_> = candidates .iter() - .map(|candidate| { + .filter_map(|candidate| { let mut candidate = candidate.clone(); + let mut changed = false; for (query, selected) in candidate.queries.iter_mut().zip(&maintained_roots) { if let Some(selected) = selected { - query.selected_plan_root = std::rc::Rc::clone(selected); + if !super::maintained_population::supported_node(&query.selected_plan_root) + { + query.selected_plan_root = std::rc::Rc::clone(selected); + changed = true; + } } } + if !changed { + return None; + } if maintained_roots.iter().all(Option::is_some) { candidate.allow_mixed_summary_and_exact_execution = false; candidate.enabled_materialization_keys = None; } - candidate + Some(candidate) }) .collect(); for candidate in maintained { diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index e6b3b634c..ff4dd3182 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -9,7 +9,7 @@ use asap_types::{ }; use planner_types::post_asap::SketchAlgorithm; -pub const MODEL_VERSION: &str = "backend-workload-resources-v1"; +pub const MODEL_VERSION: &str = "backend-workload-resources-v2"; const CPU_PER_ITEM: f64 = 1e-7; const CPU_PER_BYTE: f64 = 1e-9; const SAMPLE_BYTES: f64 = 24.0; @@ -72,6 +72,74 @@ fn resources( calculation, } } +// Price the installed program, including transient heap state. Group count is +// conservatively bounded by input cardinality until scoped group statistics exist. +fn snapshot_work( + entry: &asap_types::query_plan::QueryPlanEntry, + rows: f64, +) -> Result<(f64, f64), CompileError> { + use planner_types::post_asap::{SketchParams, SummaryFamilyType}; + let program = entry + .physical_dag + .as_ref() + .ok_or_else(|| invalid("missing snapshot physical program"))?; + let nodes = program["nodes"] + .as_object() + .ok_or_else(|| invalid("invalid snapshot program"))?; + let mut items = rows; // Bind complete input rows. + let mut bytes = rows * SERIES_BYTES * 3.0; + for node in nodes.values().filter_map(|node| node.get("Operator")) { + let kind = node["operator"]["kind"] + .as_object() + .ok_or_else(|| invalid("invalid native operator"))?; + let Some((name, parameters)) = kind.iter().next() else { + return Err(invalid("missing native operator kind")); + }; + match name.as_str() { + "Sort" => items += rows * rows.max(2.0).log2(), + "Limit" | "Project" | "CompiledProject" => items += rows, + "KeyedSummaryBuild" => { + let family: SummaryFamilyType = + serde_json::from_value(parameters["family"].clone()) + .map_err(|error| invalid(error.to_string()))?; + let SummaryFamilyType::Sketch(kind, _) = family else { + return Err(invalid("no snapshot cost model for non-sketch keyed build")); + }; + let (width, depth, capacity) = match kind.params() { + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } + | SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => (*width as f64, *depth as f64, *heap_size as f64), + _ => return Err(invalid("no snapshot heap cost model for this algorithm")), + }; + let groups = if parameters["groups"] + .as_array() + .is_some_and(|groups| groups.is_empty()) + { + rows.min(1.0) + } else { + rows + }; + items += rows * (depth + capacity.max(2.0).log2()); + bytes += groups * (width * depth * 8.0 + capacity * (SERIES_BYTES + 24.0)); + } + "KeyedReadout" => items += rows * rows.max(2.0).log2(), + _ => { + return Err(invalid(format!( + "no automatic snapshot cost model for {name}" + ))) + } + } + } + Ok((items * CPU_PER_ITEM, bytes)) +} + fn state( m: &PrecomputeMaterialization, request: &PhysicalCompilationRequest, @@ -506,12 +574,13 @@ pub(super) fn estimate( } => { output_rows = cardinality; if entry.population_snapshot().is_some() { + let (cpu, workspace) = snapshot_work(entry, cardinality)?; detail = json!({"input_rows": cardinality, "physical_program": entry.physical_dag, - "workspace_bytes_bound": cardinality * SERIES_BYTES * 3.0, - "formula": "rows * (log2(max(rows, 2)) + 2) * cpu_seconds_per_item", + "workspace_bytes_bound": workspace, + "formula": "sum of installed operator work; grouped heap count bounded by input rows", "cpu_seconds_per_item": CPU_PER_ITEM}); - cardinality * (cardinality.max(2.0).log2() + 2.0) * CPU_PER_ITEM + cpu } else { cardinality * CPU_PER_ITEM } @@ -582,7 +651,7 @@ pub(super) fn estimate( BTreeSet::new() }; let workspace_byte_seconds = if entry.population_snapshot().is_some() { - cardinality * SERIES_BYTES * 3.0 * cpu * evaluations + snapshot_work(entry, cardinality)?.1 * cpu * evaluations } else { 0.0 }; diff --git a/control_plane/tests/native_snapshot_topk.rs b/control_plane/tests/native_snapshot_topk.rs new file mode 100644 index 000000000..9eae454b1 --- /dev/null +++ b/control_plane/tests/native_snapshot_topk.rs @@ -0,0 +1,180 @@ +//! Planner owns the snapshot heap program; Backend admits and prices it. +use control_plane::physical::{ + compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + }, + workload_cost::{ + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, + }, +}; +use serde_json::{json, Value}; + +fn fixture(certified: bool) -> BackendLocalPlanningInput { + let mut wire: Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let query = "topk by (job) (1, spatial_value)"; + let mut entry = wire["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = + json!({"explicit":{"EpsilonDelta":{"epsilon":0.1,"delta":0.1}}}); + wire["query_workload"]["repeating_queries"] = json!([entry]); + wire["implementation"]["topk_evidence"] = json!({}); + wire["implementation"]["data_snapshot_id"] = "snapshot-topk-test".into(); + if certified { + wire["implementation"]["accuracy_evidence"] = json!({query: { + "query_string": query, "data_snapshot_id":"snapshot-topk-test", + "data_workload": wire["data_workload"], "source":"enforced-fixture-contract", + "observed_at_unix_ms":9500, "valid_for_ms":60000, + "topk_max_distinct_items":1000, + "topk_selected_lower_bound":101.0, "topk_excluded_upper_bound":100.0, + "topk_interval_failure_probability":0.001 + }}); + } + serde_json::from_value(wire).unwrap() +} + +fn heap(plan: &control_plane::physical::compiler::CompiledPhysicalPlan) -> bool { + plan.query_plan.entries.values().any(|entry| { + entry + .physical_dag + .as_ref() + .is_some_and(|program| program.to_string().contains("CountSketchWithHeap")) + }) +} + +// Same physical inventory, different deployment quotes: selection must reverse. +#[test] +fn snapshot_heap_and_exact_candidates_reach_deployment_cost_selection() { + for prefer_heap in [false, true] { + let mut input = fixture(true); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + assert!(request + .planner_selection_trace + .iter() + .any(|event| event["stage"] == "planner.physical_candidate" + && event.get("physical_dag").is_some())); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let mut saw_heap = false; + let mut saw_exact = false; + let quotes = candidates + .into_iter() + .filter_map(|candidate| { + let plan = DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + .ok()?; + let is_heap = heap(&plan); + let local = plan + .query_plan + .entries + .values() + .all(|entry| entry.population_snapshot().is_some()); + saw_heap |= is_heap; + saw_exact |= local && !is_heap; + if is_heap { + assert!(plan.precompute_plan.materializations.is_empty()); + let entry = plan.query_plan.entries.values().next().unwrap(); + let restored: asap_types::query_plan::QueryPlanEntry = + serde_json::from_value(serde_json::to_value(entry).unwrap()).unwrap(); + restored.recover_population_physical_dag().unwrap(); + } + let manifest = manifest(&plan, &candidate.queries).unwrap(); + Some(WorkloadQuote { + unit_costs: manifest + .components + .keys() + .map(|key| { + ( + key.clone(), + if local && is_heap == prefer_heap { + 1.0 + } else { + 1e12 + }, + ) + }) + .collect(), + manifest, + executable: true, + }) + }) + .collect(); + assert!( + saw_heap, + "certified signed heap candidate never reached pricing" + ); + assert!(saw_exact, "exact population candidate disappeared"); + input.workload_cost_evidence = Some(WorkloadCostEvidence { + backend_revision: BACKEND_REVISION.into(), + planner_revision: PLANNER_REVISION.into(), + data_snapshot_id: "snapshot-topk-test".into(), + model_version: "fixture-cost-reversal".into(), + observed_at_unix_ms: 10000, + valid_for_ms: 60000, + quotes, + }); + assert_eq!(heap(&input.compile_promql().unwrap()), prefer_heap); + } +} + +// A priced physical graph is not an accuracy proof. Missing population/gap +// evidence must leave the exact candidate available and the heap uninstalled. +#[test] +fn unknown_snapshot_heap_guarantee_cannot_be_installed() { + let (request, environment) = fixture(false).into_physical_compilation_request().unwrap(); + assert!(request + .planner_selection_trace + .iter() + .any(|event| event["stage"] == "planner.physical_candidate")); + for candidate in enumerate_exact_and_materialized_candidates(request).unwrap() { + if let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { + assert!(!heap(&plan)); + } + } +} + +// Analytical costing must account for transient heap state and operator work, +// rather than pricing every population program as the same Sort/Limit pair. +#[test] +fn automatic_costs_include_installed_heap_workspace() { + let input = fixture(true); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + let heap_ids = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .into_iter() + .filter_map(|candidate| { + DeploymentPlanCompiler + .compile_promql(candidate, environment.clone()) + .ok() + }) + .filter(heap) + .map(|plan| plan.envelope.plan_id) + .collect::>(); + assert!(!heap_ids.is_empty()); + let plan = input.compile_promql().unwrap(); + let report = plan.cost_comparison.unwrap(); + let evaluated = report + .candidate_evaluations + .iter() + .filter(|candidate| candidate.plan_id.is_some_and(|id| heap_ids.contains(&id))) + .collect::>(); + assert!(!evaluated.is_empty()); + for candidate in evaluated { + assert!( + candidate.total_cost.is_some(), + "{:?}", + candidate.unavailable_reason + ); + let resources = candidate.automatic_cost.as_ref().unwrap(); + assert_eq!(resources.model_version, "backend-workload-resources-v2"); + assert!(resources.components.values().any(|component| { + component.calculation.get("physical_program").is_some() + && component.calculation["workspace_bytes_bound"] + .as_f64() + .unwrap_or(0.) + > 0. + && component.memory_byte_seconds > 0. + })); + } +} diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 79ea86a03..a6322c8ef 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -1916,3 +1916,6 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() #[path = "support/univmon_erp_process.rs"] mod univmon_erp_process; + +#[path = "support/spatial_heap_process.rs"] +mod spatial_heap_process; diff --git a/data_plane/tests/support/spatial_heap_process.rs b/data_plane/tests/support/spatial_heap_process.rs new file mode 100644 index 000000000..5d805725d --- /dev/null +++ b/data_plane/tests/support/spatial_heap_process.rs @@ -0,0 +1,184 @@ +use super::*; +use control_plane::physical::{ + compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + }, + workload_cost::{ + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, + }, +}; + +// Actual installed heap program: Remote Write -> current snapshot -> native +// CountSketch build/readout -> HTTP result. Updates never accumulate old weights. +#[tokio::test] +async fn spatial_heap_installs_and_serves_replacements_staleness_and_expiry() { + let query = "topk by (job) (1, spatial_value)"; + let mut wire: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut entry = wire["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = + serde_json::json!({"explicit":{"EpsilonDelta":{"epsilon":0.1,"delta":0.1}}}); + wire["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + wire["implementation"]["topk_evidence"] = serde_json::json!({}); + wire["implementation"]["data_snapshot_id"] = "spatial-heap-process".into(); + // Across every tested snapshot, the winner is >=900, every excluded value + // is <=10, and there are at most three complete series identities. + wire["implementation"]["accuracy_evidence"] = serde_json::json!({query:{ + "query_string":query,"data_snapshot_id":"spatial-heap-process", + "data_workload":wire["data_workload"],"source":"enforced-test-population", + "observed_at_unix_ms":9500,"valid_for_ms":60000,"topk_max_distinct_items":3, + "topk_selected_lower_bound":900.0,"topk_excluded_upper_bound":10.0, + "topk_interval_failure_probability":0.001 + }}); + let mut snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let mut saw_heap = false; + let quotes = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .into_iter() + .filter_map(|candidate| { + let plan = DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + .ok()?; + let heap = plan.query_plan.entries.values().any(|entry| { + entry + .physical_dag + .as_ref() + .is_some_and(|program| program.to_string().contains("CountSketchWithHeap")) + }); + saw_heap |= heap; + let manifest = manifest(&plan, &candidate.queries).unwrap(); + Some(WorkloadQuote { + unit_costs: manifest + .components + .keys() + .map(|key| (key.clone(), if heap { 1.0 } else { 1e12 })) + .collect(), + manifest, + executable: true, + }) + }) + .collect(); + assert!(saw_heap); + snapshot.workload_cost_evidence = Some(WorkloadCostEvidence { + backend_revision: BACKEND_REVISION.into(), + planner_revision: PLANNER_REVISION.into(), + data_snapshot_id: "spatial-heap-process".into(), + model_version: "test-only-heap-selection".into(), + observed_at_unix_ms: 10000, + valid_for_ms: 60000, + quotes, + }); + let plan = snapshot.clone().compile_promql().unwrap(); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert!(entry + .physical_dag + .as_ref() + .unwrap() + .to_string() + .contains("CountSketchWithHeap")); + entry.recover_population_physical_dag().unwrap(); + assert!(plan.precompute_plan.materializations.is_empty()); + + let calls = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let observed = calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let fallback = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve(listener,Router::new().route("/-/healthy",get(||async{"healthy"})).route("/api/v1/query",get(move ||{ + let observed=observed.clone();async move { + observed.fetch_add(1,std::sync::atomic::Ordering::Relaxed); + Json(serde_json::json!({"status":"success","data":{"resultType":"vector","result":[]}})) + } + }))).await.unwrap(); + }); + let output = tempfile::tempdir().unwrap(); + let path = output.path().join("snapshot.json"); + std::fs::write(&path, serde_json::to_vec(&snapshot).unwrap()).unwrap(); + let port = unused_port(); + let mut child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .args([ + "--profile", + "asapquery", + "--forward-unsupported-queries", + "--planning-snapshot", + ]) + .arg(path) + .args([ + "--prometheus-server", + &fallback, + "--http-port", + &port.to_string(), + "--output-dir", + ]) + .arg(output.path()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{port}"); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut child.0).await; + let end = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let stale = + f64::from_bits(data_plane::drivers::ingest::prometheus_remote_write::STALE_NAN_BITS); + for (offset, samples, winner, score) in [ + (0, vec![("a", 1000.0), ("b", 10.0), ("c", 1.0)], "a", 1000.0), + (1000, vec![("a", -5.0), ("b", 900.0)], "b", 900.0), + (2000, vec![("b", stale), ("c", 1000.0)], "c", 1000.0), + (7000, vec![("a", 900.0)], "a", 900.0), + ] { + let request = WriteRequest { + timeseries: samples + .into_iter() + .map(|(instance, value)| { + series_with_labels( + "spatial_value", + &[("job", "api"), ("unreferenced_instance", instance)], + &if offset == 0 { + vec![(end - 5000, value), (end, value)] + } else { + vec![(end + offset, value)] + }, + ) + }) + .collect(), + }; + assert_eq!(remote_write(&client, &base, &request).await, 204); + let body: Value = client + .get(format!("{base}/api/v1/query")) + .query(&[ + ("query", query.to_owned()), + ("time", format!("{:.3}", (end + offset) as f64 / 1000.0)), + ]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(is_warm(&body), "{body}"); + let rows = body["data"]["result"].as_array().unwrap(); + assert_eq!(rows.len(), 1, "{body}"); + assert_eq!(rows[0]["metric"]["unreferenced_instance"], winner, "{body}"); + let actual = rows[0]["value"][1] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!((actual - score).abs() < 1e-8, "{body}"); + } + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 0); + task.abort(); +} diff --git a/docs/design_docs/evidence-dependent-candidates.md b/docs/design_docs/evidence-dependent-candidates.md index 83c57b87b..f7af388d3 100644 --- a/docs/design_docs/evidence-dependent-candidates.md +++ b/docs/design_docs/evidence-dependent-candidates.md @@ -211,7 +211,7 @@ flowchart LR Compare --> Select[Lowest-cost feasible physical candidate] ``` -`backend-workload-resources-v1` is an explicit analytical reference model, not a +`backend-workload-resources-v2` is an explicit analytical reference model, not a calibrated prediction or a monetary quote. It computes a resource vector before weighting it: `cost = CPU_seconds + 1e-9 * memory_byte_seconds + 1e-8 * network_bytes`. These fixed versioned weights express a default tradeoff; a deployment can still @@ -317,3 +317,11 @@ bound must not change the query's read interval. A temporal producer that cannot implement a fractional range remains unsupported for that binding, while exact execution preserves the original range. Level-3 acceptance uses the actual replay cadence and still requires a local executable plan. + +Spatial TopK admission keeps the exact population ranking and the Planner's +CountSketch-with-heap physical candidate separate. The heap requires scoped +score separation and an enforced `topk_max_distinct_items` bound; an estimated +workload cardinality is insufficient. Missing evidence leaves the candidate +visible but ineligible for installation. Cost model v2 prices the installed +native operators and transient heap workspace. A complete provider quote can +choose either candidate; neither is forced by operator name. diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 10dcb744b..6c294488a 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -56,9 +56,8 @@ an explicitly planned physical operation. The storage integration test covers durable per-series sums, native quantile state construction, publication, bound readout and restart of both the store and -resolver. Automatic PromQL physical-candidate installation, Rate-to-heap storage -execution and spatial latest-value heap execution remain pending. This test does -not establish those paths. +resolver. The full PromQL physical-candidate handoff and Rate-to-heap storage +execution remain pending. This storage test does not establish those paths. Bound frozen reads resolve the stored-output/group prefix through the persistent series resolver. Each cached part builds a sorted output/window index once on @@ -72,7 +71,16 @@ all eligible members through a snapshot binding. Planner compiles the ranking above that boundary; the QueryPlan persists that physical program before candidate pricing and activation. Serving recovers its operators and supplies full-label native rows without parsing or lowering the query. Automatic cost -estimates include sorting CPU and temporary workspace; provider quote manifests -include the physical program itself. Other population readouts retain their -existing paths. This path does not imply that spatial heap candidates or buffered -Rate-to-heap persistence are deployed yet. +estimates include native operator CPU and temporary workspace; provider quote +manifests include the physical program itself. Other population readouts retain +their existing paths. + +Planner also exposes a CountSketch-with-heap candidate over that same snapshot. +Backend requires scoped membership/score evidence, including the enforced +`topk_max_distinct_items` bound, before installation. Source cardinality estimates +do not supply that accuracy bound. Signed sample values cannot authorize CMS. +Backend compares the complete physical programs; it does not replace the selected +heap with Sort/Limit. The heap is rebuilt for each evaluation, so decreases, +staleness and expiry do not accumulate historical weights. Automated real-process +Remote Write/HTTP coverage verifies these changes with no exact-backend fallback. +This is a query-time heap candidate; buffered Rate-to-heap persistence is separate. From 3632c14170340b510d36260b8b42fd810b3a45fd Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 23:13:35 +0000 Subject: [PATCH 23/45] feat: trace Planner snapshot heap candidate compilation --- control_plane/src/physical/compiler.rs | 3 +++ docs/runtime-debugging.md | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 51b6f2a22..49199c042 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -987,6 +987,9 @@ impl BackendLocalPlanningInput { let asap_aware_mapping::Replacement::Summary(root) = candidate.replacement else { continue; }; + let _physical = tracing::debug_span!(target: "asap_runtime_debug", "physical_candidate_compile", + stage = "planner.physical_candidate", query_id = %query.query_id, + input_kind = "current_series_snapshot").entered(); let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root); match compiled { Ok(program) => { diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 1e5affb01..50b930e20 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -102,3 +102,9 @@ complete eligible population, and native execution performs ranking and limiting These spans use `input_kind=current_series_snapshot`; label sets and sample values are not recorded. They describe the exact ranking path, not heap storage or request-time physical compilation. + +Snapshot heap discovery records `planner.physical_candidate` while Planner +compiles the candidate above its population boundary. Deployment admission and +pricing follow separately; an emitted physical graph does not imply that its +accuracy evidence was admitted. The same recovery, binding and execution spans +cover the installed CountSketch heap and exact ranking programs. From c76fd8fcb650be3cf9b0f8039662cf2691c32580 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 27 Sep 2026 23:40:16 +0000 Subject: [PATCH 24/45] feat: execute Planner Rate heap candidates over durable counter SDS --- Cargo.lock | 14 +- Cargo.toml | 12 +- control_plane/src/physical/compiler.rs | 87 +++++- control_plane/src/physical/plan_dot.rs | 3 + .../src/physical/workload_cost/automatic.rs | 9 +- control_plane/src/query_plan.rs | 1 + control_plane/tests/native_rate_topk.rs | 168 ++++++++++++ crates/asap_types/src/query_plan.rs | 13 +- crates/asap_types/src/query_plan/native.rs | 111 ++++++++ .../asap_query_engine/logical_dag.rs | 4 +- .../logical_dag/native_values.rs | 72 +++-- .../asap_query_engine/post_asap_readout.rs | 3 +- .../types/hot_reload_config.rs | 3 + .../asapquery_compatibility_process_e2e.rs | 3 + data_plane/tests/support/rate_heap_process.rs | 251 ++++++++++++++++++ .../evidence-dependent-candidates.md | 7 + .../catalog-physical-plan-runtime.md | 18 +- 17 files changed, 732 insertions(+), 47 deletions(-) create mode 100644 control_plane/tests/native_rate_topk.rs create mode 100644 data_plane/tests/support/rate_heap_process.rs diff --git a/Cargo.lock b/Cargo.lock index 4913b5488..3dd033406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3ebe195260fa6e16e226183ee50938078bb8359f#3ebe195260fa6e16e226183ee50938078bb8359f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index aeb0dff27..efdbb69da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3ebe195260fa6e16e226183ee50938078bb8359f" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 16fe06908..ea4428e1a 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -977,13 +977,21 @@ impl BackendLocalPlanningInput { &asap_aware_mapping::accuracy::EqualSplitAllocator, &evidence, ); - let proposed = - strategy.current_series_topk_candidates(&Rc::new(typed), &query.accuracy_target); + let typed = Rc::new(typed); + let mut proposed = + strategy.current_series_topk_candidates(&typed, &query.accuracy_target); + let direct = asap_aware_mapping::ReplacementStrategy::propose( + &strategy, + &asap_aware_mapping::TargetSubDAG::new(&typed), + ); + proposed.candidates.extend(direct.candidates); + proposed.rejected.extend(direct.rejected); for candidate in proposed.candidates { let asap_aware_mapping::Replacement::Summary(root) = candidate.replacement else { continue; }; - let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root); + let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root) + .or_else(|_| asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(&root).map(|(_, program)| program)); match compiled { Ok(program) => { planner_selection_trace.push(serde_json::json!({ @@ -2102,8 +2110,65 @@ impl DeploymentPlanCompiler { } else { false }; - let mut entry = if let Some(operator) = - super::maintained_population::operator(&request, query)? + let native_rate = + asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking( + &query.selected_plan_root, + ) + .ok(); + let mut entry = if let Some((source, program)) = native_rate { + let mut entry = crate::query_plan::compile_bound_composable_mapped( + query.query_id.clone(), + canonical.clone(), + &source, + instant, + FallbackPolicy::ExactBackend, + binding, + |node, query_node| { + if let Some(id) = executable_dags[query_index] + .as_ref() + .and_then(|compiled| compiled.node_ids.node_id(node)) + { + query_node_bindings.insert((query_index, id), query_node); + } + }, + )?; + let root = crate::query_plan::QueryNodeId( + entry.nodes.keys().map(|id| id.0).max().unwrap_or(0) + 1, + ); + let sources = program + .input_contracts() + .map(|(id, _)| id) + .collect::>(); + if sources.len() != 1 { + return Err(CompileError::Snapshot( + "Rate physical candidate requires one source binding".into(), + )); + } + entry.nodes.insert( + root, + crate::query_plan::QueryPlanNode::Physical { + inputs: vec![entry.root], + source_nodes: sources, + max_bytes: request + .retained_summary_memory_budget_bytes + .unwrap_or(64 * 1024 * 1024), + }, + ); + entry.root = root; + entry.physical_dag = Some( + serde_json::from_slice( + &program + .encode() + .map_err(|e| CompileError::Snapshot(e.to_string()))?, + ) + .map_err(|e| CompileError::Snapshot(e.to_string()))?, + ); + entry.recover_vector_physical_dag()?; + if let Some(compiled) = &executable_dags[query_index] { + query_node_bindings.insert((query_index, compiled.dag.root), root); + } + Ok(entry) + } else if let Some(operator) = super::maintained_population::operator(&request, query)? { let root = crate::query_plan::QueryNodeId(0); let compiled = executable_dags[query_index] @@ -4215,7 +4280,17 @@ fn collect_selected_materializations( } let mut selected = Vec::new(); - walk(node, None, composable, None, &mut selected)?; + let physical_source = + asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(node) + .ok() + .map(|(source, _)| source); + walk( + physical_source.as_ref().unwrap_or(node), + None, + composable, + None, + &mut selected, + )?; if composable { selected .retain(|state| !has_unsafe_raw_entity_leaf(node, &[Rc::clone(&state.node)], false)); diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index 715aacf7b..b39ea0288 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -139,6 +139,9 @@ fn escape(value: &str) -> String { fn query_node_label(node: &QueryPlanNode) -> String { match node { + QueryPlanNode::Physical { source_nodes, .. } => { + format!("Planner Physical DAG\nbound sources {source_nodes:?}") + } QueryPlanNode::RelationalJoin { .. } => "RelationalJoin".into(), QueryPlanNode::Relational { .. } => "Relational".into(), QueryPlanNode::Logical { operator, .. } => format!("Logical\n{}", residual_label(operator)), diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index ff4dd3182..8a600ace5 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -499,6 +499,11 @@ pub(super) fn estimate( let mut detail = json!({"input_rows":input_rows}); let mut network = 0.0; let cpu = match node { + Node::Physical { .. } => { + let (cpu, workspace) = snapshot_work(entry, input_rows)?; + detail = json!({"input_rows":input_rows,"physical_program":entry.physical_dag,"workspace_bytes_bound":workspace}); + cpu + } Node::ReadMaterialization { binding } => { let s = states .get(&binding.materialization.fingerprint()) @@ -650,7 +655,9 @@ pub(super) fn estimate( } else { BTreeSet::new() }; - let workspace_byte_seconds = if entry.population_snapshot().is_some() { + let workspace_byte_seconds = if matches!(node, Node::Physical { .. }) { + snapshot_work(entry, input_rows)?.1 * cpu * evaluations + } else if entry.population_snapshot().is_some() { snapshot_work(entry, cardinality)?.1 * cpu * evaluations } else { 0.0 diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 003e7a5f9..282ab6bbf 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -178,6 +178,7 @@ where for (local, mut physical) in nodes { match &mut physical { QueryPlanNode::Logical { inputs, .. } + | QueryPlanNode::Physical { inputs, .. } | QueryPlanNode::SummaryMerge { inputs } | QueryPlanNode::ExternalExact { inputs, .. } => { for input in inputs { diff --git a/control_plane/tests/native_rate_topk.rs b/control_plane/tests/native_rate_topk.rs new file mode 100644 index 000000000..d1cbfb129 --- /dev/null +++ b/control_plane/tests/native_rate_topk.rs @@ -0,0 +1,168 @@ +//! Planner owns Rate ranking; Backend binds durable state and prices candidates. +use control_plane::physical::{ + compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}, + workload_cost::enumerate_exact_and_materialized_candidates, +}; +use serde_json::{json, Value}; + +fn fixture(certified: bool) -> BackendLocalPlanningInput { + let mut wire: Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let query = "topk by (job) (1, rate(requests_total[1m]))"; + let mut entry = wire["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = + json!({"explicit":{"EpsilonDelta":{"epsilon":0.1,"delta":0.1}}}); + wire["query_workload"]["repeating_queries"] = json!([entry]); + wire["implementation"]["topk_evidence"] = json!({}); + wire["implementation"]["data_snapshot_id"] = "snapshot-topk-test".into(); + if certified { + wire["implementation"]["accuracy_evidence"] = json!({query: { + "query_string": query, "data_snapshot_id":"snapshot-topk-test", + "data_workload": wire["data_workload"], "source":"enforced-fixture-contract", + "observed_at_unix_ms":9500, "valid_for_ms":60000, + "topk_max_distinct_items":1000, + "topk_selected_lower_bound":101.0, "topk_excluded_upper_bound":100.0, + "topk_interval_failure_probability":0.001 + }}); + } + serde_json::from_value(wire).unwrap() +} + +// An admitted Rate heap reads durable counter state; it must not become an +// external whole-query fallback or accumulate counter samples as heap weights. +#[test] +fn rate_heap_candidates_bind_durable_counter_windows() { + let (request, environment) = fixture(true).into_physical_compilation_request().unwrap(); + let mut families = std::collections::BTreeSet::new(); + for candidate in enumerate_exact_and_materialized_candidates(request).unwrap() { + let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) else { + continue; + }; + let entry = plan.query_plan.entries.values().next().unwrap(); + let Some(program) = &entry.physical_dag else { + continue; + }; + for family in ["CmsWithHeap", "CountSketchWithHeap"] { + if program.to_string().contains(family) { + assert_eq!(entry.instant.lookback_ms, 60_000); + assert!(entry.nodes.values().any(|node| matches!( + node, + asap_types::query_plan::QueryPlanNode::ExactReadout { + readout: asap_types::query_plan::ExactReadout::Rate, + .. + } + ))); + assert_eq!(entry.materialization_bindings().len(), 1); + assert_eq!( + entry.materialization_bindings()[0].readout_lookback_ms, + Some(60_000) + ); + assert!(!plan.precompute_plan.materializations.is_empty()); + let mut restored: asap_types::query_plan::QueryPlanEntry = + serde_json::from_value(serde_json::to_value(entry).unwrap()).unwrap(); + restored.recover_vector_physical_dag().unwrap(); + restored.physical_dag = None; + assert!(restored.recover_vector_physical_dag().is_err()); + let mut drifted = entry.clone(); + if let asap_types::query_plan::QueryPlanNode::Physical { source_nodes, .. } = + drifted.nodes.get_mut(&drifted.root).unwrap() + { + source_nodes[0] += 100; + } + assert!(drifted.recover_vector_physical_dag().is_err()); + let mut wrong_window = entry.clone(); + wrong_window.instant.lookback_ms = 5_000; + assert!(wrong_window.recover_vector_physical_dag().is_err()); + families.insert(family); + } + } + } + assert_eq!( + families, + std::collections::BTreeSet::from(["CmsWithHeap", "CountSketchWithHeap"]) + ); +} + +// Scoped evidence is required even when a physical heap can be compiled. +#[test] +fn missing_rate_heap_proof_leaves_exact_candidate_available() { + let (request, environment) = fixture(false).into_physical_compilation_request().unwrap(); + let mut exact = false; + for candidate in enumerate_exact_and_materialized_candidates(request).unwrap() { + if let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { + for entry in plan.query_plan.entries.values() { + if let Some(program) = &entry.physical_dag { + assert!(!program.to_string().contains("WithHeap")); + exact = true; + } + } + } + } + assert!(exact); +} + +// Provider costs may choose exact ranking, CMS or CountSketch over the same +// counter population; no family is selected before deployment costs exist. +#[test] +fn rate_heap_costs_can_select_each_compiled_candidate() { + use control_plane::physical::{ + compiler::{BACKEND_REVISION, PLANNER_REVISION}, + workload_cost::{manifest, WorkloadCostEvidence, WorkloadQuote}, + }; + for preferred in ["exact", "CmsWithHeap", "CountSketchWithHeap"] { + let mut input = fixture(true); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + let quotes = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .into_iter() + .filter_map(|candidate| { + let plan = DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + .ok()?; + let entry = plan.query_plan.entries.values().next().unwrap(); + let program = entry + .physical_dag + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(); + let preferred_plan = entry.physical_vector_binding().is_some() + && if preferred == "exact" { + !program.contains("WithHeap") + } else { + program.contains(preferred) + }; + let manifest = manifest(&plan, &candidate.queries).unwrap(); + Some(WorkloadQuote { + unit_costs: manifest + .components + .keys() + .map(|key| (key.clone(), if preferred_plan { 1.0 } else { 1e12 })) + .collect(), + manifest, + executable: true, + }) + }) + .collect(); + input.workload_cost_evidence = Some(WorkloadCostEvidence { + backend_revision: BACKEND_REVISION.into(), + planner_revision: PLANNER_REVISION.into(), + data_snapshot_id: "snapshot-topk-test".into(), + model_version: "controlled-rate-ranking-cost".into(), + observed_at_unix_ms: 10000, + valid_for_ms: 60000, + quotes, + }); + let plan = input.compile_promql().unwrap(); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert!(entry.physical_vector_binding().is_some()); + let program = entry.physical_dag.as_ref().unwrap().to_string(); + if preferred == "exact" { + assert!(!program.contains("WithHeap")); + } else { + assert!(program.contains(preferred)); + } + } +} diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index bbf0ceb33..9fcda7c3e 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -395,7 +395,9 @@ impl QueryPlanEntry { self.query_id, self.root.0 ))); } - if self.population_snapshot().is_some() { + if self.physical_vector_binding().is_some() { + self.recover_vector_physical_dag()?; + } else if self.population_snapshot().is_some() { self.recover_population_physical_dag()?; } else if self.relation_output_schema()?.is_some() || self.physical_dag.is_some() { self.recover_relational_physical_dag()?; @@ -603,6 +605,14 @@ pub struct ExternalExactRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryPlanNode { + /// Bind deployment-provided vectors to the already compiled physical DAG. + /// Input positions correspond to `source_nodes`; operators live only in + /// QueryPlanEntry.physical_dag, never in this binding. + Physical { + inputs: Vec, + source_nodes: Vec, + max_bytes: u64, + }, RelationalJoin { inputs: [QueryNodeId; 2], join_kind: planner_types::pre_asap::JoinKind, @@ -672,6 +682,7 @@ impl QueryPlanNode { | Self::SummaryEstimate { input, .. } | Self::ExactReadout { input, .. } => std::slice::from_ref(input), Self::SummaryMerge { inputs } + | Self::Physical { inputs, .. } | Self::Logical { inputs, .. } | Self::ExternalExact { inputs, .. } => inputs, } diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index da3637e0a..ecd2a414e 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -326,6 +326,117 @@ impl QueryPlanEntry { } } +impl QueryPlanEntry { + pub fn physical_vector_binding(&self) -> Option<(&[QueryNodeId], &[u64], u64)> { + match self.nodes.get(&self.root) { + Some(QueryPlanNode::Physical { + inputs, + source_nodes, + max_bytes, + }) => Some((inputs, source_nodes, *max_bytes)), + _ => None, + } + } + + /// The initial stored-vector adapter binds exact per-series counter reads. + /// Recovery validates source identities and schemas without logical lowering. + pub fn recover_vector_physical_dag(&self) -> Result { + let invalid = |message: &str| QueryPlanError::Invalid(message.into()); + let (inputs, source_nodes, max_bytes) = self + .physical_vector_binding() + .ok_or_else(|| invalid("missing physical vector binding"))?; + if max_bytes == 0 + || usize::try_from(max_bytes).is_err() + || inputs.is_empty() + || inputs.len() != source_nodes.len() + || source_nodes.iter().copied().collect::>().len() != source_nodes.len() + { + return Err(invalid("invalid physical vector source mapping or budget")); + } + for input in inputs { + let Some(QueryPlanNode::ExactReadout { + input: state, + readout: ExactReadout::Rate, + }) = self.nodes.get(input) + else { + return Err(invalid( + "physical vector requires an exact per-series Rate readout", + )); + }; + let Some(QueryPlanNode::ReadMaterialization { binding }) = self.nodes.get(state) else { + return Err(invalid( + "physical Rate input requires its installed stored output", + )); + }; + if binding + .readout_lookback_ms + .is_none_or(|window| window == 0 || window != self.instant.lookback_ms) + || !matches!(&binding.output_grouping, PhysicalGrouping::PerEntity) + { + return Err(invalid( + "physical Rate input must preserve every series and its window", + )); + } + } + let value = self + .physical_dag + .as_ref() + .ok_or_else(|| invalid("missing installed vector physical DAG"))?; + let dag = CompiledPhysicalDag::decode( + &serde_json::to_vec(value).map_err(|e| QueryPlanError::Invalid(e.to_string()))?, + ) + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + if dag + .input_contracts() + .map(|(id, _)| id) + .collect::>() + != source_nodes.iter().copied().collect() + || dag.roots().len() != 1 + { + return Err(invalid( + "physical vector program differs from installed source mapping", + )); + } + use planner_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; + let vector_schema = |schema: &SummarySchema| { + [ + ( + asap_physical_operators::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN, + DataType::Utf8, + ), + ("value", DataType::Float64), + ] + .into_iter() + .all(|(name, dtype)| { + schema.fields.iter().any(|f| { + f.name == name + && f.dtype == SummaryFamilyType::Plain(dtype.clone()) + && !f.nullable + }) + }) && schema.time_index.is_some_and(|i| { + schema + .fields + .get(i) + .is_some_and(|f| f.dtype == SummaryFamilyType::Plain(DataType::Timestamp)) + }) + }; + if dag + .input_contracts() + .any(|(_, input)| !vector_schema(&input.schema)) + || !vector_schema( + &dag.output_contract(dag.roots()[0]) + .map_err(|e| QueryPlanError::Invalid(e.to_string()))? + .schema, + ) + { + return Err(invalid( + "physical vector program loses complete identity, timestamp or value", + )); + } + Ok(dag) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 06b9af019..6340553d2 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -96,13 +96,13 @@ pub(crate) fn execute_installed( where F: FnMut(QueryNodeId, u64) -> Result, { - if entry.population_snapshot().is_some() { + if entry.population_snapshot().is_some() || entry.physical_vector_binding().is_some() { if !leaves.is_empty() { return Err(miss( "population physical input must use its installed source binding", )); } - return native_values::execute_population(entry, at, callback); + return native_values::execute_vectors(entry, at, callback); } execute_values(entry, leaves, at, callback) } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index c7471ea9e..2ca9cf2bd 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -130,7 +130,7 @@ pub(super) fn semi_join( output(values, result) } -pub(super) fn execute_population( +pub(super) fn execute_vectors( entry: &asap_types::query_plan::QueryPlanEntry, at: u64, mut callback: F, @@ -157,33 +157,63 @@ where }; use futures::{executor::block_on, StreamExt}; use std::collections::BTreeMap; - let program = entry - .recover_population_physical_dag() - .map_err(|error| miss(error.to_string()))?; - let (input_id, input) = program.input_contracts().next().unwrap(); + let (program, bindings, max_bytes) = + if let Some((inputs, source_nodes, budget)) = entry.physical_vector_binding() { + ( + entry + .recover_vector_physical_dag() + .map_err(|e| miss(e.to_string()))?, + source_nodes + .iter() + .copied() + .zip(inputs.iter().copied()) + .collect::>(), + budget, + ) + } else { + let program = entry + .recover_population_physical_dag() + .map_err(|e| miss(e.to_string()))?; + let source = program.input_contracts().next().unwrap().0; + ( + program, + BTreeMap::from([(source, entry.root)]), + entry.population_snapshot().unwrap().max_bytes, + ) + }; let at_signed = i64::try_from(at).map_err(|_| miss("evaluation timestamp overflow"))?; - let values = super::vector(super::from_result(callback(entry.root, at)?)?)?; - let rows = values - .into_iter() - .map(|(labels, value)| { - series_row(&input.schema, &labels, at_signed, value) - .map_err(|error| miss(error.to_string())) - }) - .collect::, _>>()?; - let batch = - Batch::try_new(input.schema.clone(), rows).map_err(|error| miss(error.to_string()))?; - let source = Operator::source(input.schema.clone(), vec![batch]) - .map_err(|error| miss(error.to_string()))?; + let mut sources = BTreeMap::new(); + let mut input_bytes = 0usize; + for (input_id, input) in program.input_contracts() { + let values = super::vector(super::from_result(callback(bindings[&input_id], at)?)?)?; + let rows = values + .into_iter() + .map(|(labels, value)| { + series_row(&input.schema, &labels, at_signed, value) + .map_err(|e| miss(e.to_string())) + }) + .collect::, _>>()?; + let batch = Batch::try_new(input.schema.clone(), rows).map_err(|e| miss(e.to_string()))?; + input_bytes = input_bytes + .checked_add(batch.bytes()) + .ok_or_else(|| miss("physical input size overflow"))?; + if input_bytes > max_bytes as usize { + return Err(miss("physical input exceeds run budget")); + } + let source = + Operator::source(input.schema.clone(), vec![batch]).map_err(|e| miss(e.to_string()))?; + sources.insert(input_id, Box::new(source) as Source<'_>); + } let graph = program - .instantiate(BTreeMap::from([(input_id, Box::new(source) as Source<'_>)])) - .map_err(|error| miss(error.to_string()))?; + .instantiate(sources) + .map_err(|e| miss(e.to_string()))?; let context = dag::RunContext::new( dag::Scope::Query { evaluation_time_ms: at_signed, revision: 0, }, dag::Limits { - max_bytes: entry.population_snapshot().unwrap().max_bytes as usize, + max_bytes: max_bytes as usize, ..dag::Limits::default() }, ) @@ -229,7 +259,7 @@ where Ok(( QueryResult::vector(values, at), super::ExecutionStats { - summary_readout_evaluations: 1, + summary_readout_evaluations: bindings.len(), ..Default::default() }, )) diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index e4805b193..1511ed8c9 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -321,7 +321,8 @@ impl PhysicalQueryRuntime<'_> { item_labels: merged_item_labels.unwrap_or_default(), }) } - QueryPlanNode::Logical { .. } + QueryPlanNode::Physical { .. } + | QueryPlanNode::Logical { .. } | QueryPlanNode::Relational { .. } | QueryPlanNode::ExternalExact { .. } | QueryPlanNode::RelationalJoin { .. } => Err(PhysicalNodeError::Fallback( diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 096d2f41e..a18216de4 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -80,6 +80,9 @@ impl RuntimePhysicalPlan { .map_err(|error| error.to_string())?; let mut program = entry.clone(); program.root = root; + // A bound readout is an input to the physical graph, not a second + // invocation of that graph. Keep only its storage/readout contract. + program.physical_dag = None; program.nodes = reachable .into_iter() .map(|id| (id, entry.nodes[&id].clone())) diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index a6322c8ef..e01a9bf95 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -1919,3 +1919,6 @@ mod univmon_erp_process; #[path = "support/spatial_heap_process.rs"] mod spatial_heap_process; + +#[path = "support/rate_heap_process.rs"] +mod rate_heap_process; diff --git a/data_plane/tests/support/rate_heap_process.rs b/data_plane/tests/support/rate_heap_process.rs new file mode 100644 index 000000000..4c023a1b2 --- /dev/null +++ b/data_plane/tests/support/rate_heap_process.rs @@ -0,0 +1,251 @@ +use super::*; +use control_plane::physical::{ + compiler::{ + BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + }, + workload_cost::{ + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, + }, +}; + +// Real Remote Write -> persisted counter windows -> exact Rate -> Planner heap +// -> HTTP. Restart reads the same bound SDS and retained physical program. +#[tokio::test] +async fn rate_countsketch_heap_survives_counter_reset_and_durable_restart() { + run("CountSketchWithHeap").await; +} + +#[tokio::test] +async fn rate_cms_heap_survives_counter_reset_and_durable_restart() { + run("CmsWithHeap").await; +} + +async fn run(algorithm: &str) { + let query = "topk by (job) (1, rate(requests_total[1m]))"; + let mut wire: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut entry = wire["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = + serde_json::json!({"explicit":{"EpsilonDelta":{"epsilon":0.1,"delta":0.1}}}); + entry["demand"]["fixed_interval_at"]["interval"] = 60_000.into(); + entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); + wire["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + wire["implementation"]["topk_evidence"] = serde_json::json!({}); + wire["implementation"]["data_snapshot_id"] = "rate-heap-process".into(); + // Fixture rates: winner >=500, excluded rates <=10, at most three series. + wire["implementation"]["accuracy_evidence"] = serde_json::json!({query:{ + "query_string":query,"data_snapshot_id":"rate-heap-process", + "data_workload":wire["data_workload"],"source":"enforced-test-population", + "observed_at_unix_ms":9500,"valid_for_ms":60000,"topk_max_distinct_items":3, + "topk_selected_lower_bound":500.0,"topk_excluded_upper_bound":10.0, + "topk_interval_failure_probability":0.001 + }}); + let mut snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let mut saw_heap = false; + let quotes = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .into_iter() + .filter_map(|candidate| { + let plan = DeploymentPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) + .ok()?; + let heap = plan.query_plan.entries.values().any(|entry| { + entry + .physical_dag + .as_ref() + .is_some_and(|program| program.to_string().contains(algorithm)) + }); + saw_heap |= heap; + let manifest = manifest(&plan, &candidate.queries).unwrap(); + Some(WorkloadQuote { + unit_costs: manifest + .components + .keys() + .map(|key| (key.clone(), if heap { 1.0 } else { 1e12 })) + .collect(), + manifest, + executable: true, + }) + }) + .collect(); + assert!(saw_heap); + snapshot.workload_cost_evidence = Some(WorkloadCostEvidence { + backend_revision: BACKEND_REVISION.into(), + planner_revision: PLANNER_REVISION.into(), + data_snapshot_id: "rate-heap-process".into(), + model_version: "test-only-heap-selection".into(), + observed_at_unix_ms: 10000, + valid_for_ms: 60000, + quotes, + }); + let plan = snapshot.clone().compile_promql().unwrap(); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert!(entry + .physical_dag + .as_ref() + .unwrap() + .to_string() + .contains(algorithm)); + entry.recover_vector_physical_dag().unwrap(); + assert_eq!(plan.precompute_plan.materializations.len(), 1); + + let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, + precompute_plan: plan.precompute_plan, + transmission_plan: plan.transmission_plan, + query_plan: plan.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + }; + let directory = tempfile::tempdir().unwrap(); + let artifact = directory.path().join("plan.json"); + let disk = directory.path().join("disk"); + std::fs::write(&artifact, serde_json::to_vec(&install).unwrap()).unwrap(); + let spawn = |port: u16| { + ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--physical-plan") + .arg(&artifact) + .args(["--http-port", &port.to_string(), "--output-dir"]) + .arg(directory.path()) + .arg("--enable-remote-write") + .arg("--persistence-enabled") + .arg("--persistence-dir") + .arg(&disk) + .args([ + "--persistence-memory-limit-mb", + "1", + "--persistence-hot-window-secs", + "1", + "--persistence-delete-older-than-secs", + "0", + "--persistence-seal-window-count", + "1", + "--persistence-flush-interval-ms", + "10", + "--precompute-allowed-lateness-ms", + "0", + "--precompute-flush-interval-ms", + "25", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ) + }; + let client = reqwest::Client::new(); + let mut expected = Vec::new(); + for restart in [false, true] { + let port = unused_port(); + let backend = format!("http://127.0.0.1:{port}"); + let mut child = spawn(port); + wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; + if !restart { + let request = WriteRequest { + timeseries: [ + ( + "a", + vec![ + (1000, 1000.), + (20000, 20000.), + (40000, 40000.), + (60000, 60000.), + (61000, 61000.), + (80000, 61000.), + (100000, 61000.), + (120000, 61000.), + ], + ), + ( + "b", + vec![ + (1000, 1.), + (20000, 20.), + (40000, 40.), + (60000, 60.), + (61000, 1000.), + (80000, 20000.), + (100000, 50.), + (120000, 20050.), + ], + ), + ( + "c", + vec![ + (1000, 0.1), + (20000, 2.), + (40000, 4.), + (60000, 6.), + (61000, 6.1), + (80000, 8.), + (100000, 10.), + (120000, 12.), + ], + ), + ] + .into_iter() + .map(|(id, samples)| { + series_with_labels( + "requests_total", + &[("job", "api"), ("unreferenced_instance", id)], + &samples, + ) + }) + .collect(), + }; + assert_eq!(remote_write(&client, &backend, &request).await, 204); + drain_precompute(&client, &backend).await; + } + for (index, (at, winner, score)) in [(60., "a", 1000.), (120., "b", 39050. / 59.)] + .into_iter() + .enumerate() + { + let body = wait_for_warm_instant( + &client, + &backend, + query, + at, + &directory.path().join("query_engine.log"), + ) + .await; + let rows = body["data"]["result"].as_array().unwrap(); + assert_eq!(rows.len(), 1, "{body}"); + assert_eq!(rows[0]["metric"]["unreferenced_instance"], winner, "{body}"); + let value = rows[0]["value"][1] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!((value - score).abs() < 1e-8, "{body}, expected {score}"); + if restart { + assert_eq!(body["data"]["result"], expected[index]); + } else { + expected.push(body["data"]["result"].clone()); + } + } + let missing: Value = client + .get(format!("{backend}/api/v1/query")) + .query(&[("query", query), ("time", "180")]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!( + !is_warm(&missing), + "missing counter window was served as complete: {missing}" + ); + // The drained source cohort is durable before the first process exits. + assert!(disk.join("sketch_index/parts").is_dir()); + } +} diff --git a/docs/design_docs/evidence-dependent-candidates.md b/docs/design_docs/evidence-dependent-candidates.md index f7af388d3..0931f5f03 100644 --- a/docs/design_docs/evidence-dependent-candidates.md +++ b/docs/design_docs/evidence-dependent-candidates.md @@ -325,3 +325,10 @@ workload cardinality is insufficient. Missing evidence leaves the candidate visible but ineligible for installation. Cost model v2 prices the installed native operators and transient heap workspace. A complete provider quote can choose either candidate; neither is forced by operator name. + +Rate TopK follows the same admission and pricing boundary. Nonnegative finalized +counter rates admit CMS as well as CountSketch when the scoped accuracy evidence +is sufficient. Exact ranking remains available. The installed physical program +consumes the complete per-series Rate vector from bound counter SDS; Backend +quotes can select any of the three programs, and automatic costs charge the +actual native workspace. Operator support alone does not supply accuracy proof. diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 6c294488a..1c28eb6d2 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -56,8 +56,8 @@ an explicitly planned physical operation. The storage integration test covers durable per-series sums, native quantile state construction, publication, bound readout and restart of both the store and -resolver. The full PromQL physical-candidate handoff and Rate-to-heap storage -execution remain pending. This storage test does not establish those paths. +resolver. That test does not establish the full PromQL physical-candidate handoff or +precomputed heap publication. Bound frozen reads resolve the stored-output/group prefix through the persistent series resolver. Each cached part builds a sorted output/window index once on @@ -84,3 +84,17 @@ heap with Sort/Limit. The heap is rebuilt for each evaluation, so decreases, staleness and expiry do not accumulate historical weights. Automated real-process Remote Write/HTTP coverage verifies these changes with no exact-backend fallback. This is a query-time heap candidate; buffered Rate-to-heap persistence is separate. + +For `topk by (...) (..., rate(metric[1m]))`, Planner also compiles exact ranking, +CMS heap and CountSketch heap above the exact per-series Rate boundary. Backend +binds that input to the installed counter SDS. `QueryPlanNode::Physical` maps +source node IDs to deployment readouts and supplies the run budget; it contains +no second operator representation. Recovery requires the persisted physical +program, matching source IDs, complete identity and the same readout window. + +The process E2E tests ingest raw counters, publish durable counter windows, run +native ranking, restart the process and repeat both window queries. Both heap +families preserve hidden series labels, account for counter resets and rebuild +ranking independently for each window. Missing windows cannot serve a warm +result. This path stores counter state and builds the heap at query time; +publishing the heap itself during precompute requires a separate placement. From 5d571e6b7d621d2bc68b8821a33d88696c1e6e79 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 00:14:40 +0000 Subject: [PATCH 25/45] Bind fixed-window Rate heap candidates to native maintenance and durable SDS --- Cargo.lock | 14 +- Cargo.toml | 12 +- control_plane/src/physical/compiler.rs | 110 ++++++++++++++- .../src/physical/executable_binding.rs | 1 + control_plane/src/physical/plan_dot.rs | 44 ++++++ control_plane/src/physical/workload_cost.rs | 22 +++ .../src/physical/workload_cost/automatic.rs | 31 ++++- control_plane/tests/native_rate_topk.rs | 116 ++++++++++++++++ crates/asap_types/src/executable_plan.rs | 64 ++++++++- crates/asap_types/src/precompute_plan.rs | 26 +++- crates/asap_types/src/query_plan.rs | 2 +- crates/asap_types/src/query_plan/native.rs | 18 ++- .../precompute_engine/maintenance_runtime.rs | 36 ++++- data_plane/src/precompute_engine/mod.rs | 1 + .../precompute_engine/native_maintenance.rs | 130 ++++++++++++++++++ .../query_engines/asap_query_engine/engine.rs | 31 ++++- .../asap_query_engine/logical_dag.rs | 2 +- .../logical_dag/native_values.rs | 123 ++++++++++++++--- .../storage_engines/sketch_db/index/native.rs | 121 ++++++++++++++-- data_plane/tests/support/rate_heap_process.rs | 76 ++++++++-- .../evidence-dependent-candidates.md | 7 + .../summary-catalog-sds-architecture.md | 8 ++ .../catalog-physical-plan-runtime.md | 23 ++++ 23 files changed, 938 insertions(+), 80 deletions(-) create mode 100644 data_plane/src/precompute_engine/native_maintenance.rs diff --git a/Cargo.lock b/Cargo.lock index 3dd033406..6c7f3f4d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=c899084019580eac55e6502eb641d42e748c6ff2#c899084019580eac55e6502eb641d42e748c6ff2" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index efdbb69da..d583e3c88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "c899084019580eac55e6502eb641d42e748c6ff2" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index ea4428e1a..49d837c8c 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -984,6 +984,11 @@ impl BackendLocalPlanningInput { &strategy, &asap_aware_mapping::TargetSubDAG::new(&typed), ); + proposed.candidates.extend( + strategy + .fixed_window_rate_topk_candidates(&typed) + .candidates, + ); proposed.candidates.extend(direct.candidates); proposed.rejected.extend(direct.rejected); for candidate in proposed.candidates { @@ -992,6 +997,16 @@ impl BackendLocalPlanningInput { }; let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root) .or_else(|_| asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(&root).map(|(_, program)| program)); + if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&root) { + planner_selection_trace.push(serde_json::json!({ + "stage":"planner.physical_candidate", "query_id":query.query_id, + "logical_root_id":crate::planner_selection::explained_root_id(&root, &query.accuracy_target), + "rationale":candidate.rationale, "physical_candidate":serde_json::from_slice::(&physical.encode().map_err(|e| CompileError::Snapshot(e.to_string()))?).map_err(|e| CompileError::Snapshot(e.to_string()))?, + "guarantee":root.guarantee, + })); + candidate_roots.push(vec![(index, root)]); + continue; + } match compiled { Ok(program) => { planner_selection_trace.push(serde_json::json!({ @@ -1533,7 +1548,7 @@ impl DeploymentPlanCompiler { for state in &selected { if matches!(&state.node.expr, SummaryExpr::SummaryAgg { reduction: planner_types::pre_asap::Reduction::Reduce(keys), .. - } if keys.is_empty()) + } if keys.is_empty()) || asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&query.selected_plan_root).is_ok() { if let Some(sources) = immutable_materialization_sources(&state.node) { canonical_nodes.insert(Rc::as_ptr(&state.node) as usize); @@ -2115,7 +2130,31 @@ impl DeploymentPlanCompiler { &query.selected_plan_root, ) .ok(); + let native_rate = native_rate.or_else(|| { + let physical = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&query.selected_plan_root).ok()?; + fn heap(node: &Rc) -> Option> { + match &node.expr { + SummaryExpr::SummaryAgg { family: SummaryFamilyType::Sketch(..), .. } => Some(node.clone()), + SummaryExpr::ValueOperation { child, .. } => heap(child), + SummaryExpr::SummaryEstimate { summary_input, .. } => heap(summary_input), + _ => None, + } + } + Some((heap(&query.selected_plan_root)?, physical.query)) + }); let mut entry = if let Some((source, program)) = native_rate { + let native_state_binding = if let SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(..), + .. + } = &source.expr + { + let SummaryExpr::SummaryAgg { family, .. } = &source.expr else { + unreachable!() + }; + Some(binding(&source, family)?) + } else { + None + }; let mut entry = crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), @@ -2132,6 +2171,12 @@ impl DeploymentPlanCompiler { } }, )?; + if let Some(binding) = native_state_binding { + entry.nodes.insert( + entry.root, + crate::query_plan::QueryPlanNode::ReadMaterialization { binding }, + ); + } let root = crate::query_plan::QueryNodeId( entry.nodes.keys().map(|id| id.0).max().unwrap_or(0) + 1, ); @@ -2297,7 +2342,7 @@ impl DeploymentPlanCompiler { .find(|entry| entry.query_id == query_id) .expect("compiled query entry exists") .root; - let installed = super::executable_binding::install_selected_dag( + let mut installed = super::executable_binding::install_selected_dag( query_id.clone(), &compiled.dag, query_plan_sink, @@ -2313,11 +2358,35 @@ impl DeploymentPlanCompiler { query_id: query_id.clone(), reason, })?; + if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&request.queries[query_index].selected_plan_root) { + let program = physical.precompute.ok_or_else(|| CompileError::Snapshot("fixed-window candidate has no maintenance program".into()))?; + let sink = PostAsapNodeId(u32::try_from(program.roots()[0]).map_err(|_| CompileError::Snapshot("maintenance sink overflow".into()))?); + installed.native_programs.insert(sink, serde_json::from_slice(&program.encode().map_err(|e| CompileError::Snapshot(e.to_string()))?).map_err(|e| CompileError::Snapshot(e.to_string()))?); + } query_plan .selected_dags .insert(query_id.clone(), installed.document.clone()); installed_dags.insert(query_id, installed); } + let maintenance_lookbacks = materializations + .iter() + .filter_map(|target| { + target + .derived_input + .as_ref() + .map(|input| (input, target.stored_window_ms())) + }) + .flat_map(|(input, window)| input.inputs.iter().map(move |source| (*source, window))) + .fold( + BTreeMap::::new(), + |mut windows, (source, window)| { + windows + .entry(source) + .and_modify(|old| *old = (*old).max(window)) + .or_insert(window); + windows + }, + ); for materialization in &mut materializations { let fingerprint = materialization.policy_fingerprint(); let max_lookback_ms = query_plan @@ -2326,6 +2395,7 @@ impl DeploymentPlanCompiler { .flat_map(QueryPlanEntry::materialization_bindings) .filter(|binding| binding.materialization.fingerprint() == fingerprint) .filter_map(|binding| binding.readout_lookback_ms) + .chain(maintenance_lookbacks.get(&fingerprint.into()).copied()) .max(); if let Some(lookback_ms) = max_lookback_ms { materialization.num_aggregates_to_retain = Some(retained_state_count( @@ -3434,6 +3504,11 @@ pub(crate) fn retained_partition_count( A::Increase | A::Rate | A::Min | A::Max ) || !materialization.grouping_labels.names().is_empty() + || (materialization.derived_input.is_some() + && matches!( + materialization.aggregation_type, + A::CountMinSketchWithHeap | A::CountSketchWithHeap + )) { u128::from(input_cardinality.unwrap_or(1).max(1)) } else { @@ -3761,6 +3836,21 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option { asap_types::sds::PopulationPartitioning::PerEntity diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 93056ad3a..9c09bf26b 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -55,6 +55,7 @@ pub fn install_selected_dag( } precompute_sinks.sort(); let installed = InstalledPostAsapDag { + native_programs: std::collections::BTreeMap::new(), document: OwnedPostAsapDag::from_executable(query_id, dag)?, binding: BackendExecutableBinding { nodes, diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index b39ea0288..112674a6b 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -66,6 +66,14 @@ pub fn render(plan: &CompiledPhysicalPlan) -> String { edge.role )); } + for (sink, program) in &installed.native_programs { + render_native( + &mut dot, + &format!("maintenance_{dag_index}_{}", sink.0), + "Planner maintenance operators", + program, + ); + } dot.push_str(" }\n"); } dot.push_str(" }\n"); @@ -104,12 +112,48 @@ pub fn render(plan: &CompiledPhysicalPlan) -> String { )); } } + if let Some(program) = &entry.physical_dag { + render_native( + &mut dot, + &format!("native_query_{query_index}"), + "Planner query operators", + program, + ); + } dot.push_str(" }\n"); } dot.push_str("}\n"); dot } +fn render_native(dot: &mut String, prefix: &str, label: &str, program: &serde_json::Value) { + let Some(nodes) = program["nodes"].as_object() else { + return; + }; + dot.push_str(&format!( + " subgraph cluster_{prefix} {{\n label=\"{}\";\n", + escape(label) + )); + for (id, node) in nodes { + let operator = node.get("Operator"); + let label = operator + .map(|op| op["operator"]["kind"].to_string()) + .unwrap_or_else(|| "Bound physical input".into()); + emit_node( + dot, + &format!("{prefix}_{id}"), + &format!("#{id}\n{label}"), + "", + ); + if let Some(inputs) = operator.and_then(|op| op["inputs"].as_array()) { + for input in inputs { + dot.push_str(&format!(" {prefix}_{input} -> {prefix}_{id};\n")); + } + } + } + dot.push_str(" }\n"); +} + fn materialization_node(id: u64) -> String { format!("materialization_{id:016x}") } diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index bdb77a003..f321ce3ec 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -215,6 +215,28 @@ pub fn manifest( } } } + for installed in plan.precompute_plan.executable_dags.values() { + for (sink, program) in &installed.native_programs { + let Some(asap_types::executable_plan::BackendNodeBinding::Materialization { + stored_output, + }) = installed.binding.node(*sink) + else { + return Err(invalid("native maintenance sink is unbound")); + }; + let config = plan + .precompute_plan + .materializations + .iter() + .find(|m| m.policy_fingerprint() == stored_output.fingerprint()) + .ok_or_else(|| invalid("native maintenance config is absent"))?; + add( + format!("maintenance:{}", stored_output.0), + json!({"physical_program":program,"stored_output":stored_output,"window_ms":config.stored_window_ms(),"interval_ms":config.slide_interval.saturating_mul(1000)}), + "horizon", + 1.0, + ); + } + } for rule in &plan.transmission_plan.rules { add( format!("transport:{}:{}", rule.producer_id, rule.materialization.0), diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index 8a600ace5..d0d146d03 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -78,11 +78,14 @@ fn snapshot_work( entry: &asap_types::query_plan::QueryPlanEntry, rows: f64, ) -> Result<(f64, f64), CompileError> { - use planner_types::post_asap::{SketchParams, SummaryFamilyType}; let program = entry .physical_dag .as_ref() .ok_or_else(|| invalid("missing snapshot physical program"))?; + physical_work(program, rows) +} +fn physical_work(program: &Value, rows: f64) -> Result<(f64, f64), CompileError> { + use planner_types::post_asap::{SketchParams, SummaryFamilyType}; let nodes = program["nodes"] .as_object() .ok_or_else(|| invalid("invalid snapshot program"))?; @@ -129,6 +132,7 @@ fn snapshot_work( items += rows * (depth + capacity.max(2.0).log2()); bytes += groups * (width * depth * 8.0 + capacity * (SERIES_BYTES + 24.0)); } + "Readout" => items += rows, "KeyedReadout" => items += rows * rows.max(2.0).log2(), _ => { return Err(invalid(format!( @@ -357,6 +361,31 @@ pub(super) fn estimate( "backend_summary_decode_charged_in_transport": remote_backend}), ), )?; + } else if id.starts_with("maintenance:") { + let binding = &demand.implementation; + let interval = binding["interval_ms"] + .as_u64() + .filter(|value| *value > 0) + .ok_or_else(|| invalid("invalid native maintenance interval"))? + as f64 + / 1000.0; + let window = binding["window_ms"] + .as_u64() + .ok_or_else(|| invalid("missing native window"))? as f64 + / 1000.0; + let (cpu, workspace) = physical_work(&binding["physical_program"], cardinality)?; + let runs = (horizon / interval).ceil(); + let counter_scan = cardinality * (window / cadence).ceil() * CPU_PER_ITEM; + insert( + id.clone(), + resources( + runs * (cpu + counter_scan), + workspace * horizon, + 0.0, + vec![], + json!({"runs":runs,"input_series":cardinality,"window_seconds":window,"workspace_bytes_bound":workspace,"physical_program":binding["physical_program"]}), + ), + )?; } else if id.starts_with("state:") { let binding = &demand.implementation["binding"]; let materialization: asap_types::PolicyFingerprint = diff --git a/control_plane/tests/native_rate_topk.rs b/control_plane/tests/native_rate_topk.rs index d1cbfb129..b66566e67 100644 --- a/control_plane/tests/native_rate_topk.rs +++ b/control_plane/tests/native_rate_topk.rs @@ -41,6 +41,14 @@ fn rate_heap_candidates_bind_durable_counter_windows() { let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) else { continue; }; + if plan + .precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()) + { + continue; + } let entry = plan.query_plan.entries.values().next().unwrap(); let Some(program) = &entry.physical_dag else { continue; @@ -129,6 +137,11 @@ fn rate_heap_costs_can_select_each_compiled_candidate() { .map(ToString::to_string) .unwrap_or_default(); let preferred_plan = entry.physical_vector_binding().is_some() + && !plan + .precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()) && if preferred == "exact" { !program.contains("WithHeap") } else { @@ -166,3 +179,106 @@ fn rate_heap_costs_can_select_each_compiled_candidate() { } } } + +// Fixed-window candidates retain a native maintenance graph and read its heap +// state directly; removing that graph must make recovered installation invalid. +#[test] +fn fixed_window_rate_heap_candidates_install_both_physical_graphs() { + let mut input = fixture(true); + let mut wire = serde_json::to_value(&input).unwrap(); + wire["query_workload"]["repeating_queries"][0]["demand"]["fixed_interval_at"]["interval"] = + 60_000.into(); + wire["query_workload"]["repeating_queries"][0]["demand"]["fixed_interval_at"] + ["evaluation_phase"] = 0.into(); + input = serde_json::from_value(wire).unwrap(); + let (request, environment) = input.into_physical_compilation_request().unwrap(); + let mut families = std::collections::BTreeSet::new(); + let mut errors = Vec::new(); + for candidate in enumerate_exact_and_materialized_candidates(request).unwrap() { + let plan = match DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { + Ok(plan) => plan, + Err(error) => { + errors.push(error.to_string()); + continue; + } + }; + for installed in plan.precompute_plan.executable_dags.values() { + for sink in installed.native_programs.keys() { + let program = installed.native_program(*sink).unwrap().unwrap(); + let encoded = String::from_utf8(program.encode().unwrap()).unwrap(); + for family in ["CmsWithHeap", "CountSketchWithHeap"] { + if encoded.contains(family) { + families.insert(family); + } + } + assert!(encoded.contains("Rate")); + let entry = plan.query_plan.entries.values().next().unwrap(); + let query = entry.recover_vector_physical_dag().unwrap(); + assert!(!String::from_utf8(query.encode().unwrap()) + .unwrap() + .contains("KeyedSummaryBuild")); + let mut broken = plan.precompute_plan.clone(); + for installed in broken.executable_dags.values_mut() { + installed.native_programs.clear(); + } + assert!(broken.validate().is_err()); + } + } + } + assert_eq!( + families, + std::collections::BTreeSet::from(["CmsWithHeap", "CountSketchWithHeap"]), + "{errors:#?}" + ); +} + +// Candidate costs include the persisted maintenance graph, not just its cheap readout. +#[test] +fn fixed_window_heap_costs_include_native_maintenance() { + let mut wire = serde_json::to_value(fixture(true)).unwrap(); + wire["query_workload"]["repeating_queries"][0]["demand"]["fixed_interval_at"]["interval"] = + 60_000.into(); + wire["query_workload"]["repeating_queries"][0]["demand"]["fixed_interval_at"] + ["evaluation_phase"] = 0.into(); + let input: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + let ids = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .into_iter() + .filter_map(|candidate| { + DeploymentPlanCompiler + .compile_promql(candidate, environment.clone()) + .ok() + }) + .filter(|plan| { + plan.precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()) + }) + .map(|plan| plan.envelope.plan_id) + .collect::>(); + assert!(!ids.is_empty()); + let report = input.compile_promql().unwrap().cost_comparison.unwrap(); + let candidates = report + .candidate_evaluations + .iter() + .filter(|candidate| candidate.plan_id.is_some_and(|id| ids.contains(&id))) + .collect::>(); + assert!(!candidates.is_empty()); + for candidate in candidates { + assert!( + candidate.total_cost.is_some(), + "{:?}", + candidate.unavailable_reason + ); + let resources = candidate.automatic_cost.as_ref().unwrap(); + assert!(resources + .components + .iter() + .any(|(id, resource)| id.starts_with("maintenance:") + && resource.cpu_seconds > 0. + && resource.memory_byte_seconds > 0. + && resource.calculation.get("physical_program").is_some())); + } +} diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index 1c89096c6..8427b9942 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -178,6 +178,10 @@ impl OwnedPostAsapDag { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct InstalledPostAsapDag { + /// Planner-compiled bounded programs keyed by persisted output node. + /// Deployment bindings below identify their stored input/output instances. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub native_programs: BTreeMap, pub document: OwnedPostAsapDag, pub binding: BackendExecutableBinding, } @@ -190,7 +194,64 @@ impl InstalledPostAsapDag { if self.document.schema_version != MAINTENANCE_DAG_SCHEMA_VERSION { return Err("unsupported maintenance DAG document version".into()); } - self.binding.validate_maintenance(&self.document.decode()?) + self.binding + .validate_maintenance(&self.document.decode()?)?; + for sink in self.native_programs.keys() { + self.native_program(*sink)?; + } + Ok(()) + } + + /// Recovery validates the physical producer's typed storage boundaries; + /// it never lowers the semantic provenance document again. + pub fn native_program( + &self, + sink: PostAsapNodeId, + ) -> Result, String> + { + let Some(value) = self.native_programs.get(&sink) else { + return Ok(None); + }; + let program = asap_physical_operators::physical_planner::CompiledPhysicalDag::decode( + &serde_json::to_vec(value).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + if program.roots() != [u64::from(sink.0)] || !self.binding.precompute_sinks.contains(&sink) + { + return Err("native maintenance program differs from its installed sink".into()); + } + let dag = self.document.decode()?; + for (id, contract) in program.input_contracts() { + let id = PostAsapNodeId(u32::try_from(id).map_err(|_| "physical source id overflow")?); + if id == sink + || !matches!( + self.binding.node(id), + Some(BackendNodeBinding::Materialization { .. }) + ) + || dag + .nodes + .iter() + .find(|n| n.id == id) + .is_none_or(|n| n.output_schema != *contract.schema) + { + return Err( + "native maintenance source differs from installed state boundary".into(), + ); + } + } + let output = program + .output_contract(u64::from(sink.0)) + .map_err(|e| e.to_string())?; + if program.input_contracts().count() == 0 + || dag + .nodes + .iter() + .find(|n| n.id == sink) + .is_none_or(|n| n.output_schema != *output.schema) + { + return Err("native maintenance output differs from semantic schema".into()); + } + Ok(Some(program)) } /// Project the selected semantic DAG onto the maintenance ancestors of its @@ -379,6 +440,7 @@ mod tests { #[test] fn installed_maintenance_dag_rejects_complete_dag_version() { let installed = InstalledPostAsapDag { + native_programs: std::collections::BTreeMap::new(), document: OwnedPostAsapDag { schema_version: OWNED_POST_ASAP_DAG_SCHEMA_VERSION, query_id: "q".into(), diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 96c558ce9..40573bb17 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -549,12 +549,6 @@ impl PrecomputePlan { return Err(invalid()); } validated_source_window_cohort(config, &sources)?; - if sources - .iter() - .any(|source| !matches!(source.aggregation_type, crate::AggregationType::Sum)) - { - return Err(invalid()); - } if config.window_size != config.slide_interval { return Err(invalid()); } @@ -576,8 +570,26 @@ impl PrecomputePlan { .iter() .find(|node| node.id == *sink) .ok_or_else(invalid)?; - validate_maintenance_reduction(config, target_node) + let native = installed + .native_program(*sink) .map_err(PrecomputePlanError::CatalogContract)?; + if sources.iter().any(|source| { + !matches!(source.aggregation_type, crate::AggregationType::Sum) + && !(native.is_some() + && matches!(source.aggregation_type, crate::AggregationType::Rate)) + }) { + return Err(invalid()); + } + if native.is_none() { + validate_maintenance_reduction(config, target_node) + .map_err(PrecomputePlanError::CatalogContract)?; + } else if !matches!( + config.aggregation_type, + crate::AggregationType::CountMinSketchWithHeap + | crate::AggregationType::CountSketchWithHeap + ) { + return Err(invalid()); + } let inputs: Vec<_> = dag .edges .iter() diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 9fcda7c3e..b509908ca 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -605,7 +605,7 @@ pub struct ExternalExactRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryPlanNode { - /// Bind deployment-provided vectors to the already compiled physical DAG. + /// Bind deployment-provided vectors or stored batches to the compiled physical DAG. /// Input positions correspond to `source_nodes`; operators live only in /// QueryPlanEntry.physical_dag, never in this binding. Physical { diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index ecd2a414e..8e49e3212 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -354,6 +354,16 @@ impl QueryPlanEntry { return Err(invalid("invalid physical vector source mapping or budget")); } for input in inputs { + if let Some(QueryPlanNode::ReadMaterialization { binding }) = self.nodes.get(input) { + if binding.readout_lookback_ms != Some(self.instant.lookback_ms) + || binding.window_ms != self.instant.lookback_ms + || binding.window_ms == 0 + || !matches!(&binding.output_grouping, PhysicalGrouping::Reduce(labels) if labels.is_empty()) + { + return Err(invalid(&format!("stored native batch requires one complete bound window: {binding:?}, query lookback {}", self.instant.lookback_ms))); + } + continue; + } let Some(QueryPlanNode::ExactReadout { input: state, readout: ExactReadout::Rate, @@ -422,7 +432,13 @@ impl QueryPlanEntry { }; if dag .input_contracts() - .any(|(_, input)| !vector_schema(&input.schema)) + .any(|(id, input)| { + let position = source_nodes.iter().position(|source| *source == id).unwrap(); + if matches!(self.nodes.get(&inputs[position]), Some(QueryPlanNode::ReadMaterialization { .. })) { + let summaries = input.schema.fields.iter().filter(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))).collect::>(); + !matches!(summaries.as_slice(), [field] if matches!(&field.dtype, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), planner_types::post_asap::SketchAlgorithm::CmsWithHeap | planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap))) + } else { !vector_schema(&input.schema) } + }) || !vector_schema( &dag.output_contract(dag.roots()[0]) .map_err(|e| QueryPlanError::Invalid(e.to_string()))? diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 3206809e0..c3b868ac2 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -1460,8 +1460,12 @@ fn execute_finite_complete_populations( .iter() .find(|node| node.id == sink) .ok_or("complete target node is absent")?; - asap_types::precompute_plan::validate_maintenance_reduction(config, target_node)?; - if !matches!(&target_node.payload, ExecutableOperatorPayload::SummaryAgg { + let native_program = installed.native_program(sink)?; + if native_program.is_none() { + asap_types::precompute_plan::validate_maintenance_reduction(config, target_node)?; + } + if native_program.is_none() + && !matches!(&target_node.payload, ExecutableOperatorPayload::SummaryAgg { reduction: planner_types::pre_asap::Reduction::Reduce(keys), .. } if keys.is_empty()) { @@ -1511,6 +1515,26 @@ fn execute_finite_complete_populations( for window in common_windows.unwrap_or_default() { let cohort = store.read_complete_raw_maintenance_cohort(generation, &derived.inputs, window)?; + if let Some(program) = &native_program { + let max_bytes = asap_physical_operators::runtime::Limits::default().max_bytes; + let batch = super::native_maintenance::execute( + installed, + program, + cohort.inputs(), + window, + max_bytes, + )?; + let mut output = crate::storage_engines::types::PrecomputedOutput::new( + window.0, + window.1, + None, + target.fingerprint(), + ); + output.population_labels = Some(Population::new()); + output.catalog_generation = Some(Arc::clone(generation)); + store.publish_native_summary_output(resolver, config, &output, batch, max_bytes)?; + continue; + } let (dag, key) = prepare_frozen_maintenance_sink( installed, &plan.materializations, @@ -2797,6 +2821,7 @@ mod tests { }, ); let installed = InstalledPostAsapDag { + native_programs: std::collections::BTreeMap::new(), document, binding: durable_binding, }; @@ -3181,7 +3206,11 @@ mod tests { binding .nodes .insert(PostAsapNodeId(6), BackendNodeBinding::MaintenanceInput); - let installed = InstalledPostAsapDag { document, binding }; + let installed = InstalledPostAsapDag { + native_programs: BTreeMap::new(), + document, + binding, + }; let configs = [first, second, target]; let catalog = Arc::new( asap_types::summary_catalog::SummaryCatalog::from_materializations(2, 1, &configs) @@ -4142,6 +4171,7 @@ mod tests { bundle.precompute_plan.executable_dags = BTreeMap::from([( "retry".into(), InstalledPostAsapDag { + native_programs: BTreeMap::new(), document: { let mut document = OwnedPostAsapDag::from_executable("retry".into(), &dag).unwrap(); diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 726801a29..8019d2568 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -8,6 +8,7 @@ pub mod ingest_handler; pub mod maintenance_runtime; pub(crate) mod metrics; pub mod multisource_coordinator; +mod native_maintenance; pub mod output_sink; pub mod raw_dag; pub mod series_buffer; diff --git a/data_plane/src/precompute_engine/native_maintenance.rs b/data_plane/src/precompute_engine/native_maintenance.rs new file mode 100644 index 000000000..dba59db09 --- /dev/null +++ b/data_plane/src/precompute_engine/native_maintenance.rs @@ -0,0 +1,130 @@ +//! Bind a complete durable counter cohort to a Planner-owned maintenance graph. +use std::{collections::BTreeMap, sync::Arc}; + +use asap_physical_operators::{ + operators::Operator, + physical_planner::{CompiledPhysicalDag, Source}, + runtime::{Limits, RunContext, Scope}, + values::{Batch, Value}, +}; +use asap_types::executable_plan::{BackendNodeBinding, InstalledPostAsapDag}; +use futures::{executor::block_on, StreamExt}; +use planner_types::{ + post_asap::{PostAsapNodeId, SummaryFamilyType}, + pre_asap::DataType, +}; + +pub(super) fn execute( + installed: &InstalledPostAsapDag, + program: &CompiledPhysicalDag, + inputs: &[crate::storage_engines::sketch_db::index::FrozenExactWindows], + window: (u64, u64), + max_bytes: usize, +) -> Result { + let mut sources = BTreeMap::new(); + let mut input_bytes = 0usize; + for (id, contract) in program.input_contracts() { + let node = PostAsapNodeId(u32::try_from(id).map_err(|_| "native source id overflow")?); + let Some(BackendNodeBinding::Materialization { stored_output }) = + installed.binding.node(node) + else { + return Err("native maintenance input has no stored binding".into()); + }; + let mut rows = Vec::new(); + for input in inputs + .iter() + .filter(|input| input.stored_output_reference.stored_output_id == *stored_output) + { + let state = input + .windows + .get(&window) + .ok_or("native maintenance requires an exact complete counter window")?; + let row = contract + .schema + .fields + .iter() + .map(|field| match &field.dtype { + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Rate, + _, + ) => Ok(Value::Summary { + family: field.dtype.clone(), + state: Arc::clone(state), + }), + SummaryFamilyType::Plain(DataType::Timestamp) => Ok(Value::Timestamp( + i64::try_from(window.1).map_err(|_| "native window overflow")?, + )), + SummaryFamilyType::Plain(DataType::Utf8) + if field.name == "$promql_series_identity" => + { + Ok(Value::Utf8( + serde_json::to_string(&input.group) + .map_err(|e| e.to_string())? + .into(), + )) + } + SummaryFamilyType::Plain(DataType::Utf8) => Ok(Value::Utf8( + input + .group + .get(&field.name) + .cloned() + .unwrap_or_default() + .into(), + )), + _ => Err("unsupported native maintenance stored input field".to_string()), + }) + .collect::, String>>()?; + rows.push(row); + } + let batch = Batch::try_new(contract.schema.clone(), rows).map_err(|e| e.to_string())?; + input_bytes = input_bytes + .checked_add(batch.bytes()) + .ok_or("native input size overflow")?; + if input_bytes > max_bytes { + return Err("native maintenance input exceeds run budget".into()); + } + sources.insert( + id, + Box::new( + Operator::source(contract.schema.clone(), vec![batch]) + .map_err(|e| e.to_string())?, + ) as Source<'_>, + ); + } + let graph = program.instantiate(sources).map_err(|e| e.to_string())?; + let context = RunContext::new( + Scope::Ingestion { + window_start_ms: i64::try_from(window.0).map_err(|_| "native window overflow")?, + window_end_ms: i64::try_from(window.1).map_err(|_| "native window overflow")?, + revision: 0, + }, + Limits { + max_bytes, + ..Limits::default() + }, + ) + .map_err(|e| e.to_string())?; + let schema = program + .output_contract(program.roots()[0]) + .map_err(|e| e.to_string())? + .schema; + let mut stream = graph + .execute(program.roots(), context) + .map_err(|e| e.to_string())? + .remove(0); + block_on(async { + let mut rows = Vec::new(); + let mut bytes = 0usize; + while let Some(batch) = stream.next().await { + let batch = batch.map_err(|e| e.to_string())?; + bytes = bytes + .checked_add(batch.bytes()) + .ok_or("native output size overflow")?; + if bytes > max_bytes { + return Err("native maintenance output exceeds publication budget".into()); + } + rows.extend(batch.rows().iter().cloned()); + } + Batch::try_new(schema, rows).map_err(|e| e.to_string()) + }) +} diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index ede83919d..afb3e3e12 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -400,11 +400,28 @@ impl ASAPQueryEngine { .summary_store .as_ref() .map(|index| index.summary_update_revision()); - let result = super::logical_dag::execute_installed( - entry, - leaves, - at, - |root, evaluation_ms| { + let result = if entry + .physical_vector_binding() + .is_some_and(|(inputs, _, _)| { + inputs.iter().all(|input| { + matches!( + entry.nodes.get(input), + Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { .. }) + ) + }) + }) { + let store = self.summary_store.as_ref().ok_or_else(|| { + EngineError::capability_miss("native_stored", "summary store unavailable") + })?; + super::logical_dag::native_values::execute_stored( + entry, + physical.query_plan.plan_id, + physical.query_plan.plan_version, + store, + at, + ) + } else { + super::logical_dag::execute_installed(entry, leaves, at, |root, evaluation_ms| { if let Some(asap_types::query_plan::QueryPlanNode::Logical { operator: asap_types::query_plan::residual::ResidualQueryOperator::CurrentSeries { @@ -521,8 +538,8 @@ impl ASAPQueryEngine { evaluation_ms, false, )) - }, - ); + }) + }; let current = self .summary_store .as_ref() diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 6340553d2..13d5b3fae 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -1,5 +1,5 @@ //! Executes the installed typed logical DAG. No serving-time PromQL parsing. -mod native_values; +pub(super) mod native_values; use crate::query_engines::{ query_result::{InstantVectorElement, QueryResult}, EngineError, diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 2ca9cf2bd..b77029d47 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -147,15 +147,7 @@ where u64, ) -> Result, { - use crate::{ - query_engines::query_result::{InstantVectorElement, QueryResult}, - storage_engines::types::KeyByLabelValues, - }; - use asap_physical_operators::physical_planner::{ - promql_rows::{decode_series_identity, series_row, SERIES_IDENTITY_COLUMN}, - Source, - }; - use futures::{executor::block_on, StreamExt}; + use asap_physical_operators::physical_planner::promql_rows::series_row; use std::collections::BTreeMap; let (program, bindings, max_bytes) = if let Some((inputs, source_nodes, budget)) = entry.physical_vector_binding() { @@ -181,19 +173,112 @@ where entry.population_snapshot().unwrap().max_bytes, ) }; + execute_batches( + &program, + max_bytes, + bindings.len(), + at, + |input_id, schema| { + let values = super::vector(super::from_result(callback(bindings[&input_id], at)?)?)?; + let rows = values + .into_iter() + .map(|(labels, value)| { + series_row( + schema, + &labels, + i64::try_from(at).map_err(|_| miss("evaluation timestamp overflow"))?, + value, + ) + .map_err(|e| miss(e.to_string())) + }) + .collect::, _>>()?; + Batch::try_new(schema.clone(), rows).map_err(|e| miss(e.to_string())) + }, + ) +} + +pub(in crate::query_engines::asap_query_engine) fn execute_stored( + entry: &asap_types::query_plan::QueryPlanEntry, + plan_id: u64, + plan_version: u64, + store: &crate::storage_engines::sketch_db::index::SketchStore, + at: u64, +) -> Result< + ( + crate::query_engines::query_result::QueryResult, + super::ExecutionStats, + ), + EngineError, +> { + let (inputs, sources, max_bytes) = entry + .physical_vector_binding() + .ok_or_else(|| miss("missing native stored binding"))?; + let program = entry + .recover_vector_physical_dag() + .map_err(|e| miss(e.to_string()))?; + execute_batches(&program, max_bytes, inputs.len(), at, |id, schema| { + let index = sources + .iter() + .position(|source| *source == id) + .ok_or_else(|| miss("native source is unbound"))?; + let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = + entry.nodes.get(&inputs[index]) + else { + return Err(miss("native stored source is not a bound heap")); + }; + let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; + let start = at + .checked_sub(binding.window_ms) + .ok_or_else(|| miss("native window underflow"))?; + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + stored_output_id: binding.stored_output_reference.stored_output_id, + population: std::collections::BTreeMap::new(), + window: asap_types::sds::HalfOpenTimeRange { + start_ms: start as i64, + end_ms: end, + }, + }; + store + .read_bound_native_summary( + &address, + &binding.stored_output_reference, + schema.clone(), + max_bytes as usize, + ) + .map_err(miss) + }) +} + +fn execute_batches( + program: &asap_physical_operators::physical_planner::CompiledPhysicalDag, + max_bytes: u64, + input_count: usize, + at: u64, + mut input_batch: impl FnMut(u64, &Schema) -> Result, +) -> Result< + ( + crate::query_engines::query_result::QueryResult, + super::ExecutionStats, + ), + EngineError, +> { + use crate::{ + query_engines::query_result::{InstantVectorElement, QueryResult}, + storage_engines::types::KeyByLabelValues, + }; + use asap_physical_operators::physical_planner::{ + promql_rows::{decode_series_identity, SERIES_IDENTITY_COLUMN}, + Source, + }; + use futures::{executor::block_on, StreamExt}; + use std::collections::BTreeMap; let at_signed = i64::try_from(at).map_err(|_| miss("evaluation timestamp overflow"))?; let mut sources = BTreeMap::new(); let mut input_bytes = 0usize; for (input_id, input) in program.input_contracts() { - let values = super::vector(super::from_result(callback(bindings[&input_id], at)?)?)?; - let rows = values - .into_iter() - .map(|(labels, value)| { - series_row(&input.schema, &labels, at_signed, value) - .map_err(|e| miss(e.to_string())) - }) - .collect::, _>>()?; - let batch = Batch::try_new(input.schema.clone(), rows).map_err(|e| miss(e.to_string()))?; + let batch = input_batch(input_id, &input.schema)?; input_bytes = input_bytes .checked_add(batch.bytes()) .ok_or_else(|| miss("physical input size overflow"))?; @@ -259,7 +344,7 @@ where Ok(( QueryResult::vector(values, at), super::ExecutionStats { - summary_readout_evaluations: bindings.len(), + summary_readout_evaluations: input_count, ..Default::default() }, )) diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index 1493cddaf..0fa3bb97b 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -19,20 +19,49 @@ struct NativeSummaryOutput { } impl NativeSummaryOutput { fn new(batch: Batch, max_bytes: usize) -> Result { - let [row] = batch.rows() else { - return Err("one stored summary record requires one output row".into()); - }; - let states: Vec<_> = row + let families = batch + .schema() + .fields .iter() - .filter_map(|value| match value { - Value::Summary { state, .. } => Some(state), - _ => None, + .filter_map(|field| { + (!matches!( + field.dtype, + planner_types::post_asap::SummaryFamilyType::Plain(_) + )) + .then_some(&field.dtype) }) - .collect(); - let [state] = states.as_slice() else { - return Err("stored native summary requires exactly one summary state".into()); + .collect::>(); + let [family] = families.as_slice() else { + return Err("native stored batch requires one summary column".into()); + }; + use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; + let schema_kind = match family { + SummaryFamilyType::Sketch(sketch, _) => match sketch.algorithm() { + SketchAlgorithm::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), + SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), + _ => None, + }, + _ => None, }; - let kind = state.get_accumulator_type(); + let mut kind = schema_kind; + for row in batch.rows() { + let states = row + .iter() + .filter_map(|value| match value { + Value::Summary { state, .. } => Some(state), + _ => None, + }) + .collect::>(); + let [state] = states.as_slice() else { + return Err("native stored row requires one summary state".into()); + }; + let row_kind = state.get_accumulator_type(); + if kind.is_some_and(|kind| kind != row_kind) { + return Err("native stored rows have different summary families".into()); + } + kind = Some(row_kind); + } + let kind = kind.ok_or("empty native batch has no supported summary family")?; let bytes = encode_batch(&batch).map_err(|error| error.to_string())?; if bytes.len() > max_bytes || batch.bytes() > max_bytes { return Err("native summary exceeds publication/read budget".into()); @@ -48,7 +77,11 @@ impl NativeSummaryOutput { .iter() .position(|field| &field.name == key) .ok_or("native output is missing its stored group key")?; - if !matches!(&self.batch.rows()[0][column], Value::Utf8(actual) if actual.as_ref() == value) + if self + .batch + .rows() + .iter() + .any(|row| !matches!(&row[column], Value::Utf8(actual) if actual.as_ref() == value)) { return Err("native output group differs from stored address".into()); } @@ -128,7 +161,22 @@ impl SketchStore { let (population_key, group) = build_attrs_fp_and_label_map(config, output)?; let state = NativeSummaryOutput::new(batch, max_bytes)?; let family = config.accumulator_spec().map_err(|e| e.to_string())?.family; - for value in &state.batch.rows()[0] { + if state + .batch + .schema() + .fields + .iter() + .filter(|field| { + !matches!( + field.dtype, + planner_types::post_asap::SummaryFamilyType::Plain(_) + ) + }) + .any(|field| field.dtype != family) + { + return Err("native output schema differs from installed definition".into()); + } + for value in state.batch.rows().iter().flatten() { if let Value::Summary { family: actual, .. } = value { if actual != &family { return Err("native output family differs from installed definition".into()); @@ -206,6 +254,46 @@ impl SketchStore { let sid = resolver .lookup("stored-output", &population_key, &identity) .ok_or("native stored output/group is unavailable")?; + self.read_native_summary_handle(sid, address, reference, expected_schema, max_bytes) + } + + /// A complete native batch is stored atomically under one global address. + /// Logical grouping remains in the batch; reads never allocate a resolver ID. + pub fn read_bound_native_summary( + &self, + address: &asap_types::sds::StoredSummaryKey, + reference: &StoredOutputReference, + expected_schema: Schema, + max_bytes: usize, + ) -> Result { + if !address.population.is_empty() { + return Err("native batch binding requires the complete stored output".into()); + } + let handles = self.storage_handles_for_output(reference); + let [sid] = handles.as_slice() else { + return Err("native stored output is absent or ambiguous".into()); + }; + self.read_native_summary_handle(*sid, address, reference, expected_schema, max_bytes) + } + + fn read_native_summary_handle( + &self, + sid: u64, + address: &asap_types::sds::StoredSummaryKey, + reference: &StoredOutputReference, + expected_schema: Schema, + max_bytes: usize, + ) -> Result { + address.validate().map_err(|e| e.to_string())?; + let generation = self + .active_catalog_generation() + .ok_or("native read has no active generation")?; + if (address.plan_id, address.plan_version) != (generation.plan_id, generation.plan_version) + || address.stored_output_id != reference.stored_output_id + || self.stored_output_for_handle(sid).as_ref() != Some(reference) + { + return Err("native read differs from its installed output binding".into()); + } let window = ( u64::try_from(address.window.start_ms).map_err(|_| "negative native window")?, u64::try_from(address.window.end_ms).map_err(|_| "negative native window")?, @@ -483,6 +571,13 @@ mod tests { 1 << 20, ) .unwrap(); + let direct = store + .read_bound_native_summary(&address, &reference, expected_schema.clone(), 1 << 20) + .unwrap(); + assert_eq!( + encode_batch(&direct).unwrap(), + encode_batch(&batch).unwrap() + ); let handles = store.storage_handles_for_output(&reference); assert_eq!(handles.len(), 1); let frames = store.query_range(handles[0], 0, 60_000); diff --git a/data_plane/tests/support/rate_heap_process.rs b/data_plane/tests/support/rate_heap_process.rs index 4c023a1b2..dc8681ca9 100644 --- a/data_plane/tests/support/rate_heap_process.rs +++ b/data_plane/tests/support/rate_heap_process.rs @@ -12,15 +12,25 @@ use control_plane::physical::{ // -> HTTP. Restart reads the same bound SDS and retained physical program. #[tokio::test] async fn rate_countsketch_heap_survives_counter_reset_and_durable_restart() { - run("CountSketchWithHeap").await; + run("CountSketchWithHeap", false).await; } #[tokio::test] async fn rate_cms_heap_survives_counter_reset_and_durable_restart() { - run("CmsWithHeap").await; + run("CmsWithHeap", false).await; } -async fn run(algorithm: &str) { +// The maintenance graph publishes the heap itself before HTTP readout and restart. +#[tokio::test] +async fn precomputed_rate_countsketch_heap_survives_durable_restart() { + run("CountSketchWithHeap", true).await; +} +#[tokio::test] +async fn precomputed_rate_cms_heap_survives_durable_restart() { + run("CmsWithHeap", true).await; +} + +async fn run(algorithm: &str, precomputed: bool) { let query = "topk by (job) (1, rate(requests_total[1m]))"; let mut wire: Value = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" @@ -35,7 +45,7 @@ async fn run(algorithm: &str) { wire["query_workload"]["repeating_queries"] = serde_json::json!([entry]); wire["implementation"]["topk_evidence"] = serde_json::json!({}); wire["implementation"]["data_snapshot_id"] = "rate-heap-process".into(); - // Fixture rates: winner >=500, excluded rates <=10, at most three series. + // Fixture rates: winner >=500, excluded rates <=10, at most three series per ranking group. wire["implementation"]["accuracy_evidence"] = serde_json::json!({query:{ "query_string":query,"data_snapshot_id":"rate-heap-process", "data_workload":wire["data_workload"],"source":"enforced-test-population", @@ -62,6 +72,13 @@ async fn run(algorithm: &str) { .as_ref() .is_some_and(|program| program.to_string().contains(algorithm)) }); + let heap = heap + && plan + .precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()) + == precomputed; saw_heap |= heap; let manifest = manifest(&plan, &candidate.queries).unwrap(); Some(WorkloadQuote { @@ -94,7 +111,17 @@ async fn run(algorithm: &str) { .to_string() .contains(algorithm)); entry.recover_vector_physical_dag().unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!( + plan.precompute_plan.materializations.len(), + if precomputed { 2 } else { 1 } + ); + assert_eq!( + plan.precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()), + precomputed + ); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -200,6 +227,20 @@ async fn run(algorithm: &str) { &samples, ) }) + .chain(std::iter::once(series_with_labels( + "requests_total", + &[("job", "worker"), ("unreferenced_instance", "d")], + &[ + (1000, 2000.), + (20000, 40000.), + (40000, 80000.), + (60000, 120000.), + (61000, 122000.), + (80000, 160000.), + (100000, 200000.), + (120000, 240000.), + ], + ))) .collect(), }; assert_eq!(remote_write(&client, &backend, &request).await, 204); @@ -218,13 +259,26 @@ async fn run(algorithm: &str) { ) .await; let rows = body["data"]["result"].as_array().unwrap(); - assert_eq!(rows.len(), 1, "{body}"); - assert_eq!(rows[0]["metric"]["unreferenced_instance"], winner, "{body}"); - let value = rows[0]["value"][1] - .as_str() - .unwrap() - .parse::() + assert_eq!(rows.len(), 2, "{body}"); + assert!( + rows.iter() + .all(|row| row["metric"].get("__name__").is_none()), + "Rate must drop the metric name: {body}" + ); + let api = rows + .iter() + .find(|row| row["metric"]["job"] == "api") + .unwrap(); + let worker = rows + .iter() + .find(|row| row["metric"]["job"] == "worker") .unwrap(); + assert_eq!(worker["metric"]["unreferenced_instance"], "d"); + assert!( + (worker["value"][1].as_str().unwrap().parse::().unwrap() - 2000.).abs() < 1e-8 + ); + assert_eq!(api["metric"]["unreferenced_instance"], winner, "{body}"); + let value = api["value"][1].as_str().unwrap().parse::().unwrap(); assert!((value - score).abs() < 1e-8, "{body}, expected {score}"); if restart { assert_eq!(body["data"]["result"], expected[index]); diff --git a/docs/design_docs/evidence-dependent-candidates.md b/docs/design_docs/evidence-dependent-candidates.md index 0931f5f03..69977e239 100644 --- a/docs/design_docs/evidence-dependent-candidates.md +++ b/docs/design_docs/evidence-dependent-candidates.md @@ -332,3 +332,10 @@ is sufficient. Exact ranking remains available. The installed physical program consumes the complete per-series Rate vector from bound counter SDS; Backend quotes can select any of the three programs, and automatic costs charge the actual native workspace. Operator support alone does not supply accuracy proof. + +A fixed-window Rate heap candidate places Rate finalization and heap construction +in precompute and persists the heap. Query-time construction remains a separate +candidate. This placement requires a complete, durable counter population for the +same window and does not authorize merging rates across windows. The initial +runtime realization uses the finite-source completion barrier. Candidate admission +must reject deployments unable to satisfy that completion requirement. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index c406a7925..e757eabb1 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -176,6 +176,14 @@ the producer's input contract, not inferred from interval endpoints alone. A replacement snapshot replaces a record's revision; readers must not mix its old metadata with new bytes or count both snapshots as separate inputs. +The stored output also fixes its publication granularity. A native grouped heap +may publish all groups as one typed batch for a complete window, with an empty +outer `group_key`. Group columns remain inside that batch and in its semantic +definition. The installed query reads the batch and runs Planner's grouped +readout. This makes the complete group set atomic across publication and recovery; +it does not claim that the logical computation has no grouping. A per-group record +layout and a complete-batch layout cannot silently substitute for each other. + ## 5. Semantic identity vs. deployed-output identity SDS uses two identities because they answer different questions: diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 1c28eb6d2..6ca558968 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -98,3 +98,26 @@ families preserve hidden series labels, account for counter resets and rebuild ranking independently for each window. Missing windows cannot serve a warm result. This path stores counter state and builds the heap at query time; publishing the heap itself during precompute requires a separate placement. + +Fixed-window Rate ranking has a second placement: Planner finalizes the complete +per-series counter window and builds the heap during maintenance. The deployment +binds the resulting heap to SDS; the query graph contains readout and ranking +projection only. Counter states are the bounded input buffer, including timestamps +and resets. Rates from different windows are never added as heap updates. + +This placement runs after the finite input cohort is closed and durable. It does +not infer population completeness from a timer or a missing series. Continuous +maintenance needs an explicit population/window completion contract before it can +use this path. Binding requires matching nonoverlapping complete source windows. + +Each native heap output is one atomic batch record per window. Logical groups +and series identities remain in the typed batch; its outer storage address uses +an empty group. Publishing groups independently would permit recovery to expose +an incomplete group set. The canonical SummaryDefinition still describes the +logical grouping and expressions. This storage granularity does not erase them. + +Installed maintenance programs retain Planner operator definitions and typed +input/output node identities. Recovery validates those boundaries against the +installed semantic document and stored-output bindings. The query resolves its +exact deployed output through the store index, checks the definition, generation, +window, encoding and batch schema, then executes the retained query graph. From 292406b7337333afcb3e119a95f8507074871247 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 00:19:02 +0000 Subject: [PATCH 26/45] Trace native maintenance, bound heap reads and shared physical execution --- crates/asap_types/src/query_plan/native.rs | 2 +- data_plane/src/precompute_engine/native_maintenance.rs | 2 ++ .../asap_query_engine/logical_dag/native_values.rs | 8 ++++++-- data_plane/src/storage_engines/sketch_db/index/native.rs | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index 2734e7225..c15952bde 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -347,7 +347,7 @@ impl QueryPlanEntry { /// The initial stored-vector adapter binds exact per-series counter reads. /// Recovery validates source identities and schemas without logical lowering. #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, - fields(stage = "physical.recover_validate", query_id = %self.query_id, input_kind = "stored_counter_readout"), err)] + fields(stage = "physical.recover_validate", query_id = %self.query_id, input_kind = "bound_native_input"), err)] pub fn recover_vector_physical_dag(&self) -> Result { let invalid = |message: &str| QueryPlanError::Invalid(message.into()); let (inputs, source_nodes, max_bytes) = self diff --git a/data_plane/src/precompute_engine/native_maintenance.rs b/data_plane/src/precompute_engine/native_maintenance.rs index dba59db09..0e7ed0d0e 100644 --- a/data_plane/src/precompute_engine/native_maintenance.rs +++ b/data_plane/src/precompute_engine/native_maintenance.rs @@ -14,6 +14,8 @@ use planner_types::{ pre_asap::DataType, }; +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.maintenance.execute", query_id = %installed.document.query_id, window_start_ms = window.0, window_end_ms = window.1, max_bytes, input_populations = inputs.len()), err)] pub(super) fn execute( installed: &InstalledPostAsapDag, program: &CompiledPhysicalDag, diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 8ce8c5e63..b2b22d07f 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -131,7 +131,7 @@ pub(super) fn semi_join( } #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, - fields(stage = "physical.execute", query_id = %entry.query_id, evaluation_time_ms = at, + fields(stage = "physical.bind_vectors", query_id = %entry.query_id, evaluation_time_ms = at, input_kind = "bound_promql_vector", bound_counter_state = entry.physical_vector_binding().is_some()), err)] pub(super) fn execute_vectors( entry: &asap_types::query_plan::QueryPlanEntry, @@ -200,6 +200,8 @@ where ) } +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.bind_stored_summary", query_id = %entry.query_id, evaluation_time_ms = at, plan_id, plan_version), err)] pub(in crate::query_engines::asap_query_engine) fn execute_stored( entry: &asap_types::query_plan::QueryPlanEntry, plan_id: u64, @@ -254,6 +256,8 @@ pub(in crate::query_engines::asap_query_engine) fn execute_stored( }) } +#[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "physical.execute", input_count, evaluation_time_ms = at, max_bytes), err)] fn execute_batches( program: &asap_physical_operators::physical_planner::CompiledPhysicalDag, max_bytes: u64, @@ -294,7 +298,7 @@ fn execute_batches( } let graph = { let _binding = tracing::debug_span!(target: "asap_runtime_debug", "physical_input_binding", - stage = "physical.bind_inputs", input_count = sources.len(), input_bytes, input_kind = "bound_promql_vector").entered(); + stage = "physical.bind_inputs", input_count = sources.len(), input_bytes, input_kind = "native_batch").entered(); program .instantiate(sources) .map_err(|e| miss(e.to_string()))? diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index a60dd821a..ab1bb9670 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -266,6 +266,8 @@ impl SketchStore { /// A complete native batch is stored atomically under one global address. /// Logical grouping remains in the batch; reads never allocate a resolver ID. + #[tracing::instrument(level = "debug", target = "asap_runtime_debug", skip_all, + fields(stage = "sds.read_bound_native", plan_version = address.plan_version, stored_output_id = ?reference.stored_output_id, start_ms = address.window.start_ms, end_ms = address.window.end_ms, max_bytes), err)] pub fn read_bound_native_summary( &self, address: &asap_types::sds::StoredSummaryKey, From 1b41bbd2b222cd03b9af36c777a37d19f4210500 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 00:56:39 +0000 Subject: [PATCH 27/45] Execute grouped Rate Sum candidates over complete overlapping SDS windows --- Cargo.lock | 14 +- Cargo.toml | 12 +- control_plane/src/physical/compiler.rs | 226 ++++++++++++++---- .../src/physical/compiler/windows.rs | 13 +- .../src/physical/workload_cost/automatic.rs | 13 + control_plane/src/query_plan/residual.rs | 6 +- control_plane/tests/native_rate_topk.rs | 59 +++++ crates/asap_types/src/precompute_plan.rs | 7 +- crates/asap_types/src/query_plan/native.rs | 24 +- .../precompute_engine/maintenance_runtime.rs | 14 +- .../src/precompute_engine/partitioning.rs | 20 +- .../logical_dag/native_values.rs | 35 ++- .../sketch_db/index/maintenance.rs | 5 + .../storage_engines/sketch_db/index/native.rs | 3 + data_plane/tests/support/rate_heap_process.rs | 112 ++++++++- .../evidence-dependent-candidates.md | 6 + .../summary-catalog-sds-architecture.md | 8 +- .../catalog-physical-plan-runtime.md | 11 +- 18 files changed, 478 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c7f3f4d6..86341a533 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "asap-physical-operators" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "asap-types", "asap_sketch_codec", @@ -416,12 +416,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "serde", "serde_json", @@ -443,7 +443,7 @@ dependencies = [ [[package]] name = "asap_sketch_codec" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=574bef34c3792c9ebc76773eeff159ffd14194fd#574bef34c3792c9ebc76773eeff159ffd14194fd" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=176c1bd565e0c400f9a35996c2bd65de32475e72#176c1bd565e0c400f9a35996c2bd65de32475e72" dependencies = [ "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "prost", diff --git a/Cargo.toml b/Cargo.toml index d583e3c88..152043148 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ version = "0.1.0" [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -37,8 +37,8 @@ arc-swap = "1.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "574bef34c3792c9ebc76773eeff159ffd14194fd" } +asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } +asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "176c1bd565e0c400f9a35996c2bd65de32475e72" } asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 49d837c8c..3f8e58f1a 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -984,9 +984,12 @@ impl BackendLocalPlanningInput { &strategy, &asap_aware_mapping::TargetSubDAG::new(&typed), ); + proposed + .candidates + .extend(strategy.fixed_window_rate_candidates(&typed).candidates); proposed.candidates.extend( strategy - .fixed_window_rate_topk_candidates(&typed) + .query_time_rate_aggregation_candidates(&typed) .candidates, ); proposed.candidates.extend(direct.candidates); @@ -997,7 +1000,7 @@ impl BackendLocalPlanningInput { }; let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root) .or_else(|_| asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(&root).map(|(_, program)| program)); - if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&root) { + if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&root) { planner_selection_trace.push(serde_json::json!({ "stage":"planner.physical_candidate", "query_id":query.query_id, "logical_root_id":crate::planner_selection::explained_root_id(&root, &query.accuracy_target), @@ -1548,7 +1551,7 @@ impl DeploymentPlanCompiler { for state in &selected { if matches!(&state.node.expr, SummaryExpr::SummaryAgg { reduction: planner_types::pre_asap::Reduction::Reduce(keys), .. - } if keys.is_empty()) || asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&query.selected_plan_root).is_ok() + } if keys.is_empty()) || asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&query.selected_plan_root).is_ok() { if let Some(sources) = immutable_materialization_sources(&state.node) { canonical_nodes.insert(Rc::as_ptr(&state.node) as usize); @@ -1559,6 +1562,7 @@ impl DeploymentPlanCompiler { } } let cohort_nodes = windows::cohort_nodes(&selected); + let native_cohort = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&query.selected_plan_root).is_ok(); for (ordinal, selected) in selected.into_iter().enumerate() { let mut branch_query = query.clone(); branch_query.query_lookback_ms = selected @@ -1575,7 +1579,17 @@ impl DeploymentPlanCompiler { candidate.window_secs.saturating_mul(1_000) == branch_query.query_lookback_ms && if cohort_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { - windows::is_full_cohort(candidate) + if native_cohort { + windows::is_complete_window(candidate) + && candidate.slide_secs.saturating_mul(1000) + == u64::from( + query + .summary_lifecycle_inputs + .evaluation_interval_ms, + ) + } else { + windows::is_full_cohort(candidate) + } } else { !candidate.cohort_only } @@ -1791,7 +1805,9 @@ impl DeploymentPlanCompiler { query_id: query.query_id.clone(), reason: error.to_string(), })?; - if runtime_materialization.window_size != runtime_materialization.slide_interval + if !native_cohort + && runtime_materialization.window_size + != runtime_materialization.slide_interval { return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "immutable scalar composition runtime requires nonoverlapping windows".into() }); @@ -2131,10 +2147,10 @@ impl DeploymentPlanCompiler { ) .ok(); let native_rate = native_rate.or_else(|| { - let physical = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&query.selected_plan_root).ok()?; + let physical = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&query.selected_plan_root).ok()?; fn heap(node: &Rc) -> Option> { match &node.expr { - SummaryExpr::SummaryAgg { family: SummaryFamilyType::Sketch(..), .. } => Some(node.clone()), + SummaryExpr::SummaryAgg { family: SummaryFamilyType::Sketch(..) | SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), .. } => Some(node.clone()), SummaryExpr::ValueOperation { child, .. } => heap(child), SummaryExpr::SummaryEstimate { summary_input, .. } => heap(summary_input), _ => None, @@ -2144,7 +2160,9 @@ impl DeploymentPlanCompiler { }); let mut entry = if let Some((source, program)) = native_rate { let native_state_binding = if let SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(..), + family: + SummaryFamilyType::Sketch(..) + | SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), .. } = &source.expr { @@ -2155,28 +2173,46 @@ impl DeploymentPlanCompiler { } else { None }; - let mut entry = crate::query_plan::compile_bound_composable_mapped( - query.query_id.clone(), - canonical.clone(), - &source, - instant, - FallbackPolicy::ExactBackend, - binding, - |node, query_node| { - if let Some(id) = executable_dags[query_index] - .as_ref() - .and_then(|compiled| compiled.node_ids.node_id(node)) - { - query_node_bindings.insert((query_index, id), query_node); - } - }, - )?; - if let Some(binding) = native_state_binding { - entry.nodes.insert( - entry.root, - crate::query_plan::QueryPlanNode::ReadMaterialization { binding }, - ); - } + let mut entry = if let Some(binding) = native_state_binding { + let root = crate::query_plan::QueryNodeId(0); + if let Some(id) = executable_dags[query_index] + .as_ref() + .and_then(|compiled| compiled.node_ids.node_id(&source)) + { + query_node_bindings.insert((query_index, id), root); + } + crate::query_plan::QueryPlanEntry { + physical_dag: None, + language: crate::query_plan::QueryLanguage::PromQl, + query_id: query.query_id.clone(), + canonical_query: canonical.clone(), + fixed_evaluation: None, + root, + nodes: BTreeMap::from([( + root, + crate::query_plan::QueryPlanNode::ReadMaterialization { binding }, + )]), + instant, + fallback: FallbackPolicy::ExactBackend, + } + } else { + crate::query_plan::compile_bound_composable_mapped( + query.query_id.clone(), + canonical.clone(), + &source, + instant, + FallbackPolicy::ExactBackend, + binding, + |node, query_node| { + if let Some(id) = executable_dags[query_index] + .as_ref() + .and_then(|compiled| compiled.node_ids.node_id(node)) + { + query_node_bindings.insert((query_index, id), query_node); + } + }, + )? + }; let root = crate::query_plan::QueryNodeId( entry.nodes.keys().map(|id| id.0).max().unwrap_or(0) + 1, ); @@ -2358,7 +2394,7 @@ impl DeploymentPlanCompiler { query_id: query_id.clone(), reason, })?; - if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_ranking(&request.queries[query_index].selected_plan_root) { + if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&request.queries[query_index].selected_plan_root) { let program = physical.precompute.ok_or_else(|| CompileError::Snapshot("fixed-window candidate has no maintenance program".into()))?; let sink = PostAsapNodeId(u32::try_from(program.roots()[0]).map_err(|_| CompileError::Snapshot("maintenance sink overflow".into()))?); installed.native_programs.insert(sink, serde_json::from_slice(&program.encode().map_err(|e| CompileError::Snapshot(e.to_string()))?).map_err(|e| CompileError::Snapshot(e.to_string()))?); @@ -3507,7 +3543,7 @@ pub(crate) fn retained_partition_count( || (materialization.derived_input.is_some() && matches!( materialization.aggregation_type, - A::CountMinSketchWithHeap | A::CountSketchWithHeap + A::CountMinSketchWithHeap | A::CountSketchWithHeap | A::Sum )) { u128::from(input_cardinality.unwrap_or(1).max(1)) @@ -3827,7 +3863,7 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option Option Option, readout: Option<&SketchQuery>, composable: bool, + native_maintenance: bool, inherited_grouping: Option>, selected: &mut Vec, ) -> Result<(), String> { @@ -4225,9 +4278,26 @@ fn collect_selected_materializations( } else { None }; - if let Some(source) = immutable_materialization_sources(node) { + // A nested legacy Sum is a query reduction unless Planner supplied a + // complete native maintenance graph for this selected root. + let immutable_sources = if !native_maintenance + && matches!( + &node.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + _ + ), + .. + } + ) { + None + } else { + immutable_materialization_sources(node) + }; + if let Some(source) = &immutable_sources { for source in source { - walk(&source, None, composable, None, selected)?; + walk(source, None, composable, native_maintenance, None, selected)?; } } match &node.expr { @@ -4238,24 +4308,73 @@ fn collect_selected_materializations( pruning: Some(_), .. } => { - walk(candidates, readout, composable, grouping.clone(), selected)?; + walk( + candidates, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; // Explicit external authoritative values do not need duplicate local state. if !composable { - walk(values, readout, composable, grouping.clone(), selected)?; + walk( + values, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } } SummaryExpr::ValueOperation { child, .. } => { - walk(child, readout, composable, grouping.clone(), selected)?; + walk( + child, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::RelationalJoin { left, right, .. } => { - walk(left, readout, composable, grouping.clone(), selected)?; - walk(right, readout, composable, grouping.clone(), selected)?; + walk( + left, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; + walk( + right, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::BinaryOp { lhs, rhs, .. } if composable || crate::query_plan::exact_value_executable(node) => { - walk(lhs, readout, composable, grouping.clone(), selected)?; - walk(rhs, readout, composable, grouping.clone(), selected)?; + walk( + lhs, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; + walk( + rhs, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::SummaryAgg { child, @@ -4263,15 +4382,23 @@ fn collect_selected_materializations( SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), .. } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + && immutable_sources.is_none() && ((composable && crate::query_plan::exact_accumulator_value_source(child).is_some()) || crate::query_plan::exact_value_executable(node)) => { - walk(child, readout, composable, grouping.clone(), selected)?; + walk( + child, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::SummaryAgg { child, .. } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && immutable_materialization_sources(node).is_none() => {} + && immutable_sources.is_none() => {} SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _), @@ -4284,12 +4411,20 @@ fn collect_selected_materializations( summary_input, Some(query), composable, + native_maintenance, grouping.clone(), selected, )?, SummaryExpr::SummaryMerge { children, .. } => { for child in children { - walk(child, readout, composable, grouping.clone(), selected)?; + walk( + child, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } } SummaryExpr::SummaryAgg { @@ -4394,6 +4529,7 @@ fn collect_selected_materializations( physical_source.as_ref().unwrap_or(node), None, composable, + asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(node).is_ok(), None, &mut selected, )?; diff --git a/control_plane/src/physical/compiler/windows.rs b/control_plane/src/physical/compiler/windows.rs index 709273322..fcf2987ae 100644 --- a/control_plane/src/physical/compiler/windows.rs +++ b/control_plane/src/physical/compiler/windows.rs @@ -28,6 +28,10 @@ pub(super) fn is_full_cohort(candidate: &WindowRealizationCandidate) -> bool { } } +pub(super) fn is_complete_window(candidate: &WindowRealizationCandidate) -> bool { + matches!(candidate.layout, WindowMaterializationLayout::FullWindow) || is_full_cohort(candidate) +} + pub(super) fn cohort_nodes(states: &[SelectedMaterialization]) -> BTreeSet { let mut nodes = BTreeSet::new(); for state in states { @@ -153,6 +157,7 @@ pub fn prepare_window_implementations( reason, })?; let cohorts = cohort_nodes(&states); + let native_cohort = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&query.selected_plan_root).is_ok(); let requirements = states .iter() .map(|state| { @@ -171,7 +176,7 @@ pub fn prepare_window_implementations( &model, &query.summary_lifecycle_inputs, window, - cohort, + cohort && !native_cohort, target, staleness_margin_ms, ) @@ -196,7 +201,11 @@ pub fn prepare_window_implementations( } let applicable = requirements.iter().any(|&(window, cohort)| { quote.window_secs == window - && if cohort { + && if cohort && native_cohort { + is_complete_window(quote) + && quote.slide_secs.saturating_mul(1000) + == u64::from(query.summary_lifecycle_inputs.evaluation_interval_ms) + } else if cohort { is_full_cohort(quote) } else { quote.slide_secs.saturating_mul(1_000) diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index d0d146d03..0198a3da3 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -132,6 +132,19 @@ fn physical_work(program: &Value, rows: f64) -> Result<(f64, f64), CompileError> items += rows * (depth + capacity.max(2.0).log2()); bytes += groups * (width * depth * 8.0 + capacity * (SERIES_BYTES + 24.0)); } + "SummaryBuild" => { + let family: SummaryFamilyType = + serde_json::from_value(parameters["family"].clone()) + .map_err(|e| invalid(e.to_string()))?; + if !matches!( + family, + SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _) + ) { + return Err(invalid("no native aggregate cost for this family")); + } + items += rows; + bytes += rows * 64.0; + } "Readout" => items += rows, "KeyedReadout" => items += rows * rows.max(2.0).log2(), _ => { diff --git a/control_plane/src/query_plan/residual.rs b/control_plane/src/query_plan/residual.rs index 0083566d8..3d392724a 100644 --- a/control_plane/src/query_plan/residual.rs +++ b/control_plane/src/query_plan/residual.rs @@ -422,7 +422,7 @@ pub(super) fn residual_nodes( } } Err(invalid( - "Planner residual does not match any original query subtree", + "Planner logical fragment does not match any original query subtree", )) } @@ -451,7 +451,7 @@ pub(super) fn binary_operator( }); } if operator.vector_match.is_some() { - return Err(invalid("explicit residual vector matching unsupported")); + return Err(invalid("explicit logical vector matching unsupported")); } let operation = match operator.kind.to_string().as_str() { "+" => BinaryOperation::Add, @@ -501,7 +501,7 @@ pub(super) fn selected_native_expression( ) -> Result { if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { return Err(invalid( - "native residual substitution requires an exact selected value", + "native subtree substitution requires an exact selected value", )); } let selected = match &selected.expr { diff --git a/control_plane/tests/native_rate_topk.rs b/control_plane/tests/native_rate_topk.rs index b66566e67..eafc353b8 100644 --- a/control_plane/tests/native_rate_topk.rs +++ b/control_plane/tests/native_rate_topk.rs @@ -282,3 +282,62 @@ fn fixed_window_heap_costs_include_native_maintenance() { && resource.calculation.get("physical_program").is_some())); } } + +// Rate must precede grouped Sum in both placements; deployment chooses ownership. +#[test] +fn grouped_rate_has_native_query_and_maintenance_candidates() { + let mut wire = serde_json::to_value(fixture(false)).unwrap(); + let entry = &mut wire["query_workload"]["repeating_queries"][0]; + entry["query"] = "sum by(job)(rate(requests_total[1m]))".into(); + entry["requirements"]["accuracy"] = json!({"explicit":"Exact"}); + entry["demand"]["fixed_interval_at"]["interval"] = 5_000.into(); + entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); + wire["implementation"]["accuracy_evidence"] = json!({}); + let input: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); + let mut placements = std::collections::BTreeSet::new(); + let mut errors = Vec::new(); + let mut ids = std::collections::BTreeSet::new(); + for candidate in enumerate_exact_and_materialized_candidates(request).unwrap() { + let plan = match DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { + Ok(plan) => plan, + Err(error) => { + errors.push(error.to_string()); + continue; + } + }; + let entry = plan.query_plan.entries.values().next().unwrap(); + let Some(program) = &entry.physical_dag else { + continue; + }; + if entry.physical_vector_binding().is_none() { + continue; + } + entry.recover_vector_physical_dag().unwrap(); + let stored = plan + .precompute_plan + .executable_dags + .values() + .any(|dag| !dag.native_programs.is_empty()); + assert_eq!(program.to_string().contains("SummaryBuild"), !stored); + placements.insert(stored); + ids.insert(plan.envelope.plan_id); + } + assert_eq!( + placements, + std::collections::BTreeSet::from([false, true]), + "{errors:#?}" + ); + let report = input.compile_promql().unwrap().cost_comparison.unwrap(); + for candidate in report + .candidate_evaluations + .iter() + .filter(|candidate| candidate.plan_id.is_some_and(|id| ids.contains(&id))) + { + assert!( + candidate.total_cost.is_some(), + "{:?}", + candidate.unavailable_reason + ); + } +} diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 40573bb17..ec590b54c 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -549,9 +549,6 @@ impl PrecomputePlan { return Err(invalid()); } validated_source_window_cohort(config, &sources)?; - if config.window_size != config.slide_interval { - return Err(invalid()); - } let mut matched = false; for installed in self.executable_dags.values() { let dag = installed @@ -581,12 +578,16 @@ impl PrecomputePlan { return Err(invalid()); } if native.is_none() { + if config.window_size != config.slide_interval { + return Err(invalid()); + } validate_maintenance_reduction(config, target_node) .map_err(PrecomputePlanError::CatalogContract)?; } else if !matches!( config.aggregation_type, crate::AggregationType::CountMinSketchWithHeap | crate::AggregationType::CountSketchWithHeap + | crate::AggregationType::Sum ) { return Err(invalid()); } diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index 8e49e3212..bf5108282 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -1,4 +1,4 @@ -//! Installation of native SQL fragments; recovery only decodes physical operators. +//! Retained physical programs for SQL relations and PromQL vectors. use super::*; use asap_physical_operators::{dag, physical_planner::CompiledPhysicalDag}; use planner_types::post_asap::{ @@ -338,8 +338,8 @@ impl QueryPlanEntry { } } - /// The initial stored-vector adapter binds exact per-series counter reads. - /// Recovery validates source identities and schemas without logical lowering. + /// Validate bound counter vectors or stored aggregate batches against the + /// retained physical program; recovery never lowers logical operators. pub fn recover_vector_physical_dag(&self) -> Result { let invalid = |message: &str| QueryPlanError::Invalid(message.into()); let (inputs, source_nodes, max_bytes) = self @@ -436,17 +436,21 @@ impl QueryPlanEntry { let position = source_nodes.iter().position(|source| *source == id).unwrap(); if matches!(self.nodes.get(&inputs[position]), Some(QueryPlanNode::ReadMaterialization { .. })) { let summaries = input.schema.fields.iter().filter(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))).collect::>(); - !matches!(summaries.as_slice(), [field] if matches!(&field.dtype, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), planner_types::post_asap::SketchAlgorithm::CmsWithHeap | planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap))) + !matches!(summaries.as_slice(), [field] if match &field.dtype { + SummaryFamilyType::Sketch(kind, _) => matches!(kind.algorithm(), planner_types::post_asap::SketchAlgorithm::CmsWithHeap | planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap), + SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _) => true, + _ => false, + }) } else { !vector_schema(&input.schema) } }) - || !vector_schema( - &dag.output_contract(dag.roots()[0]) - .map_err(|e| QueryPlanError::Invalid(e.to_string()))? - .schema, - ) + || { + let output = dag.output_contract(dag.roots()[0]).map_err(|e| QueryPlanError::Invalid(e.to_string()))?; + output.schema.fields.iter().filter(|field| field.dtype == SummaryFamilyType::Plain(DataType::Float64)).count() != 1 + || output.schema.fields.iter().any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(DataType::Utf8 | DataType::Float64 | DataType::Timestamp))) + } { return Err(invalid( - "physical vector program loses complete identity, timestamp or value", + "physical program has incompatible input or result schema", )); } Ok(dag) diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index c3b868ac2..a5719c471 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -1443,15 +1443,17 @@ fn execute_finite_complete_populations( .collect::, _>>()?; asap_types::precompute_plan::validated_source_window_cohort(config, &sources) .map_err(|error| error.to_string())?; + let native_program = installed.native_program(sink)?; if std::iter::once(config) .chain(sources.iter().copied()) .any(|config| { config.population_key_encoding != asap_types::PopulationKeyEncoding::CanonicalLabelsV1 - || config.slide_interval.checked_mul(1000) != Some(config.stored_window_ms()) + || (native_program.is_none() + && config.slide_interval.checked_mul(1000) != Some(config.stored_window_ms())) }) { return Err( - "complete population execution requires canonical nonoverlapping full windows".into(), + "complete population execution requires canonical windows supported by the bound program".into(), ); } let dag = installed.document.decode()?; @@ -1460,7 +1462,6 @@ fn execute_finite_complete_populations( .iter() .find(|node| node.id == sink) .ok_or("complete target node is absent")?; - let native_program = installed.native_program(sink)?; if native_program.is_none() { asap_types::precompute_plan::validate_maintenance_reduction(config, target_node)?; } @@ -1486,11 +1487,16 @@ fn execute_finite_complete_populations( } for (start, end) in windows { let width = source.stored_window_ms(); + let stride = source + .slide_interval + .checked_mul(1000) + .ok_or("source cadence overflow")?; if width == 0 + || stride == 0 || end.checked_sub(*start) != Some(width) || *end > i64::MAX as u64 || (*start as i128 - source.pane_origin_ms.unwrap_or(0) as i128) - .rem_euclid(width as i128) + .rem_euclid(stride as i128) != 0 { return Err( diff --git a/data_plane/src/precompute_engine/partitioning.rs b/data_plane/src/precompute_engine/partitioning.rs index 09ef174fd..fe29a12a7 100644 --- a/data_plane/src/precompute_engine/partitioning.rs +++ b/data_plane/src/precompute_engine/partitioning.rs @@ -68,15 +68,15 @@ impl DagPartitioning { Self::Labels(names) => { let pairs = names .iter() + // Missing and empty PromQL grouping labels denote the same + // group. Ownership must colocate them without adding a label. .map(|name| { - population - .get(name) - .map(|value| (name.as_str(), value.as_str())) - .ok_or_else(|| { - format!("DAG partition label {name} is absent from population") - }) + ( + name.as_str(), + population.get(name).map(String::as_str).unwrap_or(""), + ) }) - .collect::, _>>()?; + .collect::>(); let key = super::group_key::GroupKey::new(pairs); Ok(xxh64(key.canonical_bytes(), 0) as usize % workers) } @@ -99,7 +99,11 @@ mod tests { series.insert("instance".into(), instance.into()); assert_eq!(rule.owner(&series, 4).unwrap(), expected); } - assert!(rule.owner(&BTreeMap::new(), 4).is_err()); + assert_eq!( + rule.owner(&BTreeMap::new(), 4).unwrap(), + rule.owner(&BTreeMap::from([("service".into(), "".into())]), 4) + .unwrap() + ); assert!(rule.owner(&group, 0).is_err()); let owners = (0..64) .map(|i| { diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index b77029d47..ef25c4a56 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -1,4 +1,4 @@ -//! Bind protocol vectors to native batch operators; computation stays in Planner. +//! Bind deployment inputs to retained native programs and decode PromQL results. use super::{grouping_key, miss, EngineError, Grouping, Labels, Vector}; use asap_physical_operators::dag::{ self, batch_execution, @@ -224,7 +224,7 @@ pub(in crate::query_engines::asap_query_engine) fn execute_stored( let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(&inputs[index]) else { - return Err(miss("native stored source is not a bound heap")); + return Err(miss("native stored source has no deployed summary binding")); }; let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; let start = at @@ -315,21 +315,36 @@ fn execute_batches( .schema() .fields .iter() - .position(|field| field.name == SERIES_IDENTITY_COLUMN) - .ok_or_else(|| miss("physical output loses series identity"))?; + .position(|field| field.name == SERIES_IDENTITY_COLUMN); let value = batch .schema() .fields .iter() - .position(|field| field.name == "value") + .position(|field| field.dtype == SummaryFamilyType::Plain(DataType::Float64)) .ok_or_else(|| miss("physical output loses sample value"))?; for row in batch.rows() { - let (Value::Utf8(encoded), Value::Float64(sample)) = (&row[identity], &row[value]) - else { - return Err(miss("invalid population physical output")); + let Value::Float64(sample) = &row[value] else { + return Err(miss("invalid physical result value")); + }; + let labels = if let Some(identity) = identity { + let Value::Utf8(encoded) = &row[identity] else { + return Err(miss("invalid physical series identity")); + }; + decode_series_identity(encoded).map_err(|error| miss(error.to_string()))? + } else { + batch + .schema() + .fields + .iter() + .zip(row) + .filter_map(|(field, value)| match value { + Value::Utf8(label) if !label.is_empty() => { + Some((field.name.clone(), label.to_string())) + } + _ => None, + }) + .collect() }; - let labels = - decode_series_identity(encoded).map_err(|error| miss(error.to_string()))?; values.push( InstantVectorElement::new( KeyByLabelValues::new_with_labels(labels.values().cloned().collect()), diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index 7e355c417..ec53f5575 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -387,6 +387,11 @@ impl SketchStore { .get_or_load(part.part_id) .map_err(|e| e.to_string())?; for record in reader.window_records(sid, start_ms, end_ms) { + // A bound full-window read selects its exact coordinates; other + // overlapping snapshots do not contribute to this computation. + if !expected_windows.contains(&(record.start_ts, record.end_ts)) { + continue; + } let entry = reader.load_entry(&record).map_err(|e| e.to_string())?; if entry.label.as_ref().map_or(0, |label| label.labels.len()) != keys.len() { return Err("immutable input label arity differs from its descriptor".into()); diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index 0fa3bb97b..a93bef09a 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -41,6 +41,9 @@ impl NativeSummaryOutput { SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), _ => None, }, + SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _) => { + Some(AggregationType::Sum) + } _ => None, }; let mut kind = schema_kind; diff --git a/data_plane/tests/support/rate_heap_process.rs b/data_plane/tests/support/rate_heap_process.rs index dc8681ca9..9862601f6 100644 --- a/data_plane/tests/support/rate_heap_process.rs +++ b/data_plane/tests/support/rate_heap_process.rs @@ -30,8 +30,21 @@ async fn precomputed_rate_cms_heap_survives_durable_restart() { run("CmsWithHeap", true).await; } +#[tokio::test] +async fn precomputed_grouped_rate_sum_survives_durable_restart() { + run("Sum", true).await; +} +#[tokio::test] +async fn query_time_grouped_rate_sum_survives_durable_restart() { + run("Sum", false).await; +} + async fn run(algorithm: &str, precomputed: bool) { - let query = "topk by (job) (1, rate(requests_total[1m]))"; + let query = if algorithm == "Sum" { + "sum by (job) (rate(requests_total[1m]))" + } else { + "topk by (job) (1, rate(requests_total[1m]))" + }; let mut wire: Value = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) @@ -40,7 +53,14 @@ async fn run(algorithm: &str, precomputed: bool) { entry["query"] = query.into(); entry["requirements"]["accuracy"] = serde_json::json!({"explicit":{"EpsilonDelta":{"epsilon":0.1,"delta":0.1}}}); - entry["demand"]["fixed_interval_at"]["interval"] = 60_000.into(); + entry["demand"]["fixed_interval_at"]["interval"] = if algorithm == "Sum" { + 5_000.into() + } else { + 60_000.into() + }; + if algorithm == "Sum" { + entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); + } entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); wire["query_workload"]["repeating_queries"] = serde_json::json!([entry]); wire["implementation"]["topk_evidence"] = serde_json::json!({}); @@ -53,6 +73,9 @@ async fn run(algorithm: &str, precomputed: bool) { "topk_selected_lower_bound":500.0,"topk_excluded_upper_bound":10.0, "topk_interval_failure_probability":0.001 }}); + if algorithm == "Sum" { + wire["implementation"]["accuracy_evidence"] = serde_json::json!({}); + } let mut snapshot: BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); let (request, environment) = snapshot .clone() @@ -123,6 +146,19 @@ async fn run(algorithm: &str, precomputed: bool) { precomputed ); + if precomputed { + assert!( + plan.precompute_plan + .materializations + .iter() + .all( + |state| state.slide_interval == if algorithm == "Sum" { 5 } else { 60 } + && state.window_size == 60 + ), + "maintenance must match the query cadence, not publish only every 60 seconds" + ); + } + let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -241,15 +277,37 @@ async fn run(algorithm: &str, precomputed: bool) { (120000, 240000.), ], ))) + .chain((algorithm == "Sum").then(|| { + series_with_labels( + "requests_total", + &[("unreferenced_instance", "no-job")], + &[ + (1000, 3.), + (20000, 60.), + (40000, 120.), + (60000, 180.), + (61000, 183.), + (80000, 240.), + (100000, 300.), + (120000, 360.), + ], + ) + })) .collect(), }; assert_eq!(remote_write(&client, &backend, &request).await, 204); drain_precompute(&client, &backend).await; } - for (index, (at, winner, score)) in [(60., "a", 1000.), (120., "b", 39050. / 59.)] - .into_iter() - .enumerate() - { + let evaluations = if algorithm == "Sum" { + vec![ + (60., "a", 1000.), + (65., "a", 1000.), + (120., "b", 39050. / 59.), + ] + } else { + vec![(60., "a", 1000.), (120., "b", 39050. / 59.)] + }; + for (index, (at, winner, score)) in evaluations.into_iter().enumerate() { let body = wait_for_warm_instant( &client, &backend, @@ -259,7 +317,20 @@ async fn run(algorithm: &str, precomputed: bool) { ) .await; let rows = body["data"]["result"].as_array().unwrap(); - assert_eq!(rows.len(), 2, "{body}"); + assert_eq!(rows.len(), if algorithm == "Sum" { 3 } else { 2 }, "{body}"); + if algorithm == "Sum" { + let empty = rows + .iter() + .find(|row| { + row["metric"] + .as_object() + .is_some_and(|labels| labels.is_empty()) + }) + .unwrap_or_else(|| panic!("missing grouping labels must be omitted: {body}")); + assert!( + (empty["value"][1].as_str().unwrap().parse::().unwrap() - 3.).abs() < 1e-8 + ); + } assert!( rows.iter() .all(|row| row["metric"].get("__name__").is_none()), @@ -273,12 +344,35 @@ async fn run(algorithm: &str, precomputed: bool) { .iter() .find(|row| row["metric"]["job"] == "worker") .unwrap(); - assert_eq!(worker["metric"]["unreferenced_instance"], "d"); + if algorithm != "Sum" { + assert_eq!(worker["metric"]["unreferenced_instance"], "d"); + } assert!( (worker["value"][1].as_str().unwrap().parse::().unwrap() - 2000.).abs() < 1e-8 ); - assert_eq!(api["metric"]["unreferenced_instance"], winner, "{body}"); + if algorithm == "Sum" { + assert!( + api["metric"].get("unreferenced_instance").is_none(), + "grouped Sum must drop ungrouped labels: {body}" + ); + } else { + assert_eq!(api["metric"]["unreferenced_instance"], winner, "{body}"); + } let value = api["value"][1].as_str().unwrap().parse::().unwrap(); + let score = score + + if algorithm == "Sum" { + if at == 60. { + 1.1 + } else if at == 65. { + // b's zero-point extrapolation truncates the left edge: + // (980 * 45 / 41 + 20) / 60, plus c's constant 0.1. + (980. * 45. / 41. + 20.) / 60. + 0.1 + } else { + 0.1 + } + } else { + 0. + }; assert!((value - score).abs() < 1e-8, "{body}, expected {score}"); if restart { assert_eq!(body["data"]["result"], expected[index]); diff --git a/docs/design_docs/evidence-dependent-candidates.md b/docs/design_docs/evidence-dependent-candidates.md index 69977e239..d4baf36cc 100644 --- a/docs/design_docs/evidence-dependent-candidates.md +++ b/docs/design_docs/evidence-dependent-candidates.md @@ -339,3 +339,9 @@ candidate. This placement requires a complete, durable counter population for th same window and does not authorize merging rates across windows. The initial runtime realization uses the finite-source completion barrier. Candidate admission must reject deployments unable to satisfy that completion requirement. + +For grouped Rate, the exact grouped Sum also has query-time and precompute +physical candidates. A stored complete-window Sum does not combine raw counters +across series before Rate. Its producer cadence must satisfy the query recurrence; +selecting a 60-second lookback cannot silently reduce five-second evaluations to +one output per minute. Overlapping full windows remain independent stored results. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index e757eabb1..3160315cb 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -176,12 +176,16 @@ the producer's input contract, not inferred from interval endpoints alone. A replacement snapshot replaces a record's revision; readers must not mix its old metadata with new bytes or count both snapshots as separate inputs. -The stored output also fixes its publication granularity. A native grouped heap +The stored output also fixes its publication granularity. A native grouped heap or grouped Sum may publish all groups as one typed batch for a complete window, with an empty outer `group_key`. Group columns remain inside that batch and in its semantic definition. The installed query reads the batch and runs Planner's grouped readout. This makes the complete group set atomic across publication and recovery; -it does not claim that the logical computation has no grouping. A per-group record +it does not claim that the logical computation has no grouping. Full-window +snapshots may overlap when the evaluation cadence is shorter than the lookback. +An installed full-window read selects its exact endpoints; other overlapping +snapshots are neither combined nor mistaken for ambiguous versions of that record. +A per-group record layout and a complete-batch layout cannot silently substitute for each other. ## 5. Semantic identity vs. deployed-output identity diff --git a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md index 6ca558968..d6cd16363 100644 --- a/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md +++ b/docs/developer_docs/query-engine/catalog-physical-plan-runtime.md @@ -108,7 +108,10 @@ and resets. Rates from different windows are never added as heap updates. This placement runs after the finite input cohort is closed and durable. It does not infer population completeness from a timer or a missing series. Continuous maintenance needs an explicit population/window completion contract before it can -use this path. Binding requires matching nonoverlapping complete source windows. +use this path. Binding requires matching complete source windows at the query's +evaluation cadence. Full windows may overlap: a 60-second lookback evaluated +every five seconds stores independent `(t-60s,t]` outputs every five seconds. +A bound read selects exact window endpoints and never adds neighboring snapshots. Each native heap output is one atomic batch record per window. Logical groups and series identities remain in the typed batch; its outer storage address uses @@ -121,3 +124,9 @@ input/output node identities. Recovery validates those boundaries against the installed semantic document and stored-output bindings. The query resolves its exact deployed output through the store index, checks the definition, generation, window, encoding and batch schema, then executes the retained query graph. + +Grouped Rate supports the same placement choice. Planner can emit per-series +Rate readout → grouped Sum → Sum readout entirely at query time, or persist the +complete grouped Sum batch during maintenance. Both preserve grouping and drop +ungrouped series labels in the result. Backend prices both physical programs; +it does not move Sum across Rate or pool raw counters before calculating Rate. From 1d903970d6c323559264f728538462f2c8feff06 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 00:57:18 +0000 Subject: [PATCH 28/45] Document native Rate aggregation diagnostics across maintenance and query execution --- docs/runtime-debugging.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/runtime-debugging.md b/docs/runtime-debugging.md index 50b930e20..731ca3684 100644 --- a/docs/runtime-debugging.md +++ b/docs/runtime-debugging.md @@ -63,12 +63,27 @@ JSON API. Missing a completion event can indicate an interrupted process; it is not proof that work committed. Publication receipts and recovered state remain the source of truth. -A missing candidate is not evidence that its placement costs more. For example, -the retained #728 grouped-rate fixture exports per-series Rate state plus -query-time Sum and an exact fallback; it does not export a precomputed grouped -Rate-result candidate. The selected plan therefore cannot establish that query-time -Sum beats that absent candidate. Search coverage is scoped to the declared -inventory, never a claim of exhaustive physical optimization. +A missing candidate is not evidence that its placement costs more. The #728 +inventory now includes grouped Rate with query-time Sum and precomputed Sum; +compare their admission and resource costs before interpreting selection. Search +coverage is scoped to the declared inventory, never a claim of exhaustive +physical optimization. + +### Native Rate aggregation stages + +`physical.maintenance.execute` runs the retained Planner maintenance program over +complete bound counter windows. It covers per-series Rate followed by grouped +Sum or CMS/CountSketch heap construction. `sds.publish_native` publishes one +atomic grouped batch for that window. Overlapping complete windows are separate +executions and records. + +On a bound query, `sds.read_bound_native` resolves the installed stored output; +`physical.bind_stored_summary` binds the recovered batch. Query-time candidates +instead use `physical.bind_vectors` for the per-series Rate vector. Both call +`physical.execute` for the retained query graph. These spans expose the selected +placement; they never move an operator between query and maintenance phases. +The finite complete-input barrier is required for precomputed grouped outputs; +an elapsed-time event does not prove the population is complete. ### Installed SQL physical DAG stages From f17c87fce5814d2e12cb105d30d69ced88d6a366 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 01:15:57 +0000 Subject: [PATCH 29/45] Resolve shared population bindings once per deployment candidate --- control_plane/src/physical/compiler.rs | 8 ++-- .../src/physical/maintained_population.rs | 48 +++++++++++++------ 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 3f8e58f1a..cd7ddb84f 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1394,6 +1394,7 @@ impl DeploymentPlanCompiler { for (id, root) in planner_types::post_asap::share_common_summary_subtrees(roots) { request.queries[id].selected_plan_root = root; } + let population_operators = super::maintained_population::operators(&request)?; let mut compiled_materializations = Vec::with_capacity(request.queries.len()); let mut collector_materializations = Vec::with_capacity(request.queries.len()); let mut plan_materializations = Vec::with_capacity(request.queries.len()); @@ -1500,9 +1501,7 @@ impl DeploymentPlanCompiler { .collect::>(); // An exact native fallback has no maintained state and must not // depend on evidence for unused window/state implementations. - if selected.is_empty() - && super::maintained_population::operator(&request, query)?.is_none() - { + if selected.is_empty() && population_operators[query_index].is_none() { continue; } let executable = planner_types::post_asap::compile_executable_dag_with_node_ids( @@ -2249,8 +2248,7 @@ impl DeploymentPlanCompiler { query_node_bindings.insert((query_index, compiled.dag.root), root); } Ok(entry) - } else if let Some(operator) = super::maintained_population::operator(&request, query)? - { + } else if let Some(operator) = population_operators[query_index].clone() { let root = crate::query_plan::QueryNodeId(0); let compiled = executable_dags[query_index] .as_ref() diff --git a/control_plane/src/physical/maintained_population.rs b/control_plane/src/physical/maintained_population.rs index a7b877428..7844c5665 100644 --- a/control_plane/src/physical/maintained_population.rs +++ b/control_plane/src/physical/maintained_population.rs @@ -1,5 +1,7 @@ //! Lower typed population operators according to executor membership capabilities. -use super::compiler::{CompileError, PhysicalCompilationRequest, QueryCompilationInput}; +#[cfg(test)] +use super::compiler::QueryCompilationInput; +use super::compiler::{CompileError, PhysicalCompilationRequest}; use asap_types::query_plan::{ current_series::{SeriesPopulation, SeriesReadout}, residual::{Grouping, LabelMatch, LabelMatcher, ResidualQueryOperator}, @@ -67,22 +69,21 @@ pub(super) fn supported(request: &PhysicalCompilationRequest) -> bool { .any(|q| supported_node(&q.selected_plan_root)) } -pub(super) fn operator( +/// Resolve population bindings once per candidate. Scanning every workload +/// root for each consumer recompiles the same native ranking graphs quadratically. +pub(super) fn operators( request: &PhysicalCompilationRequest, - query: &QueryCompilationInput, -) -> Result, CompileError> { - let Some((spec, readout)) = selected(&query.selected_plan_root) else { - return Ok(None); - }; - let PopulationInput::CurrentSeries(input) = &spec.input else { - return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "maintained table-row populations require a row-update executor; remote-write current-series state is incompatible".into() }); - }; - let populations: std::collections::BTreeSet<_> = request +) -> Result>, CompileError> { + let selected = request .queries .iter() - .filter_map(|q| { - selected(&q.selected_plan_root) - .map(|(p, _)| serde_json::to_string(&p).expect("typed population serializes")) + .map(|query| selected(&query.selected_plan_root)) + .collect::>(); + let populations: std::collections::BTreeSet<_> = selected + .iter() + .flatten() + .map(|(population, _)| { + serde_json::to_string(population).expect("typed population serializes") }) .collect(); let max_bytes = request @@ -90,6 +91,11 @@ pub(super) fn operator( .unwrap_or(64 * 1024 * 1024) .min(1_073_741_824) / populations.len().max(1) as u64; + selected.into_iter().zip(&request.queries).map(|(selected, query)| { + let Some((spec, readout)) = selected else { return Ok(None); }; + let PopulationInput::CurrentSeries(input) = &spec.input else { + return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "maintained table-row populations require a row-update executor; remote-write current-series state is incompatible".into() }); + }; let population = SeriesPopulation { metric: input.metric.clone(), matchers: input @@ -135,6 +141,20 @@ pub(super) fn operator( population, readout, })) + }).collect() +} + +#[cfg(test)] +pub(super) fn operator( + request: &PhysicalCompilationRequest, + query: &QueryCompilationInput, +) -> Result, CompileError> { + let index = request + .queries + .iter() + .position(|q| q.query_id == query.query_id) + .expect("query belongs to the compilation request"); + Ok(operators(request)?.remove(index)) } /// The maintained population is a deployment source; ranking is compiled by From 320958556d014a2409aec26951a69690f3e4b3c6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 01:23:40 +0000 Subject: [PATCH 30/45] Share immutable candidate diagnostics and reuse unchanged window preparation --- control_plane/Cargo.toml | 2 +- control_plane/src/main.rs | 6 +++--- control_plane/src/physical/compiler.rs | 16 +++++++++------- control_plane/src/physical/workload_cost.rs | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 63b866090..4004000bc 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" tokio = { version = "1", features = ["full"] } axum = { version = "0.7", features = ["ws"] } futures-util = "0.3" -serde = { version = "1", features = ["derive"] } +serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" sha2 = "0.10" serde_yaml = "0.9" diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 77c6887c8..462e70ade 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -381,7 +381,7 @@ async fn compile_and_publish_physical_plan( Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, - planner_selection_trace: bundle.planner_selection_trace, + planner_selection_trace: bundle.planner_selection_trace.as_ref().clone(), plan_id: bundle.envelope.plan_id, plan_version: bundle.envelope.plan_version, status: "active", @@ -656,7 +656,7 @@ fn compile_physical_plan_request( } let compilation_request = physical::compiler::PhysicalCompilationRequest { planner_candidate_forests: Vec::new(), - planner_selection_trace, + planner_selection_trace: planner_selection_trace.into(), query_workload: Some(query_workload), data_workload: Some(request.data_workload), canonical_roots, @@ -688,7 +688,7 @@ fn compile_physical_plan_request( compilation_request.clone(), ) .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - let planner_selection_trace = compilation_request.planner_selection_trace.clone(); + let planner_selection_trace = compilation_request.planner_selection_trace.as_ref().clone(); let (manifests, candidate_evaluations) = physical::workload_cost::compile_candidates_for_pricing( candidates.clone(), diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index cd7ddb84f..31626cb21 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -157,8 +157,8 @@ pub struct PhysicalCompilationRequest { /// Complete Planner candidates retained until deployment admission/pricing. /// Empty for callers that explicitly supply one already-selected forest. pub planner_candidate_forests: Vec>, - /// Diagnostic projections of the original Planner search; never consumed by selection. - pub planner_selection_trace: Vec, + /// Immutable search diagnostics shared across candidates; never consumed by selection. + pub planner_selection_trace: std::sync::Arc>, /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. pub allow_mixed_summary_and_exact_execution: bool, /// Deployment feasibility: external exact dependencies cannot be bound. @@ -616,7 +616,7 @@ pub struct CompiledPhysicalPlan { /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, pub cost_comparison: Option, - pub planner_selection_trace: Vec, + pub planner_selection_trace: std::sync::Arc>, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -1044,12 +1044,14 @@ impl BackendLocalPlanningInput { let mut planner_candidate_forests = Vec::new(); for roots in candidate_roots { let mut forest = queries.clone(); + let changed: BTreeSet<_> = roots.iter().map(|(index, _)| *index).collect(); for (index, root) in roots { forest[index].selected_plan_root = root; } - let preparation = forest.iter_mut().try_for_each(|query| { + // Unchanged roots already have prepared windows in `queries`. + let preparation = changed.into_iter().try_for_each(|index| { prepare_window_implementations( - query, + &mut forest[index], &self.physical_inputs.window_cost_model, self.environment.target, self.physical_inputs.query_retention_margin_ms, @@ -1071,7 +1073,7 @@ impl BackendLocalPlanningInput { Ok(( PhysicalCompilationRequest { planner_candidate_forests, - planner_selection_trace, + planner_selection_trace: planner_selection_trace.into(), allow_mixed_summary_and_exact_execution: true, require_backend_local_execution: self .physical_inputs @@ -5477,7 +5479,7 @@ pub(crate) mod tests { } Ok(PhysicalCompilationRequest { planner_candidate_forests: Vec::new(), - planner_selection_trace: Vec::new(), + planner_selection_trace: Default::default(), canonical_roots: Vec::new(), allow_mixed_summary_and_exact_execution: false, require_backend_local_execution: false, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index f321ce3ec..aa53ad861 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -112,7 +112,7 @@ pub struct MaterializationSearchCoverage { pub struct CandidatePlanSelectionReport { #[serde(default)] #[serde(rename = "logical_selection")] - pub planner_selection_trace: Vec, + pub planner_selection_trace: std::sync::Arc>, #[serde(default)] pub materialization_search_coverage: Option, pub data_snapshot_id: String, From d0352787f9b358d412ede90938b9dd4136051892 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 14:12:44 +0000 Subject: [PATCH 31/45] fix: scope candidate cost manifests to the logical dataset --- control_plane/src/physical/workload_cost.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index aa53ad861..6193dced7 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -40,6 +40,7 @@ pub struct CostComponentDemand { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct WorkloadCostManifest { + pub dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, pub plan_id: u64, pub plan_version: u64, pub planner_revision: String, @@ -344,6 +345,8 @@ pub fn manifest( ); } Ok(WorkloadCostManifest { + dataset_identity: plan.precompute_plan.ingest.dataset_identity.clone() + .ok_or_else(|| CompileError::Snapshot("cost manifest lacks dataset identity".into()))?, plan_id: plan.envelope.plan_id, plan_version: plan.envelope.plan_version, planner_revision: plan.envelope.planner_revision.clone(), From 916ff3946ef9fe609dffd8770fa64d74e453bd80 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 14:16:01 +0000 Subject: [PATCH 32/45] style: format dataset-scoped cost manifest --- control_plane/src/physical/workload_cost.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 6193dced7..eea4d3532 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -345,7 +345,11 @@ pub fn manifest( ); } Ok(WorkloadCostManifest { - dataset_identity: plan.precompute_plan.ingest.dataset_identity.clone() + dataset_identity: plan + .precompute_plan + .ingest + .dataset_identity + .clone() .ok_or_else(|| CompileError::Snapshot("cost manifest lacks dataset identity".into()))?, plan_id: plan.envelope.plan_id, plan_version: plan.envelope.plan_version, From 3ca7d4126eb2f6ec64177e8cb418c4d7641ddbfe Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 28 Sep 2026 17:27:21 +0000 Subject: [PATCH 33/45] fix(control-plane): supply dataset_identity in the API test fixtures CompileAndPublishPhysicalPlanRequest gained a required `dataset_identity` field in this branch, but two api_tests fixtures build their request body as JSON by hand and were never updated, so both failed deserialization with `missing field dataset_identity` before reaching the handler. Take the value from the planning snapshot's own `environment.dataset_identity` rather than inventing one, so the fixture keeps describing the same dataset the rest of the snapshot describes. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit e627dbb329dabcff5a8d18e8796da2b0111efc01) --- control_plane/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index a6f66ac38..ad12ed794 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -898,6 +898,7 @@ mod api_tests { "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": snapshot.physical_inputs.window_cost_model }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, @@ -975,6 +976,7 @@ mod api_tests { "metric": metric, "window_secs": query.query_lookback_ms / 1_000, "accuracy": query.accuracy_target, "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } }], + "dataset_identity": snapshot.environment.dataset_identity, "collector_ids": [], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, From 40c4b2f9d1069a457359e21ddc11f671bd20d4aa Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 20:21:48 +0000 Subject: [PATCH 34/45] fix: terminate selected candidate execution on resource exhaustion --- control_plane/src/query_plan.rs | 1 + .../logical_dag/native_values.rs | 20 +++++++++---------- .../asap_query_engine/post_asap_readout.rs | 1 + 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index c3c34ab43..bedd02e3c 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1379,6 +1379,7 @@ mod tests { ) .unwrap(); let entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "topk(1, m)".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 12ddd9d18..e5c416173 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -545,7 +545,7 @@ where .map_err(|e| miss(e.to_string())) }) .collect::, _>>()?; - Batch::try_new(schema.clone(), rows).map_err(|e| miss(e.to_string())) + Batch::try_new(schema.clone(), rows).map_err(EngineError::from) }, ) } @@ -634,17 +634,15 @@ fn execute_batches( let batch = input_batch(input_id, &input.schema)?; input_bytes = input_bytes .checked_add(batch.bytes()) - .ok_or_else(|| miss("physical input size overflow"))?; + .ok_or(asap_physical_operators::Error::MemoryLimit)?; if input_bytes > max_bytes as usize { - return Err(miss("physical input exceeds run budget")); + return Err(asap_physical_operators::Error::MemoryLimit.into()); } let source = - Operator::source(input.schema.clone(), vec![batch]).map_err(|e| miss(e.to_string()))?; + Operator::source(input.schema.clone(), vec![batch]).map_err(EngineError::from)?; sources.insert(input_id, Box::new(source) as Source<'_>); } - let graph = program - .instantiate(sources) - .map_err(|e| miss(e.to_string()))?; + let graph = program.instantiate(sources).map_err(EngineError::from)?; let context = dag::RunContext::new( dag::Scope::Query { evaluation_time_ms: at_signed, @@ -655,15 +653,15 @@ fn execute_batches( ..dag::Limits::default() }, ) - .map_err(|error| miss(error.to_string()))?; + .map_err(EngineError::from)?; let mut stream = graph .execute(program.roots(), context) - .map_err(|error| miss(error.to_string()))? + .map_err(EngineError::from)? .remove(0); let values = block_on(async { let mut values = Vec::new(); while let Some(batch) = stream.next().await { - let batch = batch.map_err(|error| miss(error.to_string()))?; + let batch = batch.map_err(EngineError::from)?; let identity = batch .schema() .fields @@ -683,7 +681,7 @@ fn execute_batches( let Value::Utf8(encoded) = &row[identity] else { return Err(miss("invalid physical series identity")); }; - decode_series_identity(encoded).map_err(|error| miss(error.to_string()))? + decode_series_identity(encoded).map_err(EngineError::from)? } else { batch .schema() diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 838efc08b..1027b49ce 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -956,6 +956,7 @@ mod tests { }, }; let entry = QueryPlanEntry { + physical_dag: None, language: QueryLanguage::PromQl, query_id: "resource-test".into(), canonical_query: "1".into(), From 9c3af3e5f8d32f383028b3ab6a446c0fc832bc64 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 20:22:45 +0000 Subject: [PATCH 35/45] test: reject exhausted candidate inputs without fallback --- .../logical_dag/native_values.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index e5c416173..a41d00fc6 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -428,6 +428,24 @@ mod tests { } } + // The complete selected-candidate adapter must not classify a byte budget as capability. + #[test] + fn selected_candidate_input_budget_is_a_terminal_error() { + let plan = CompiledPhysicalDag::decode(&sorted()).unwrap(); + let error = execute_batches(&plan, 1, 1, 42, |_, schema| { + Ok(Batch::try_new( + schema.clone(), + vec![vec![Value::Float64(1.)]], + )?) + }) + .err() + .expect("input must exceed the byte budget"); + assert!( + matches!(error, EngineError::Physical(Error::MemoryLimit)), + "{error}" + ); + } + // Count-like values are bound as integers only when the protocol sample is exact. #[test] fn integer_input_binding_preserves_type_and_rejects_rounding() { From 265ed24b15578553e0af79f910903040477c3c1f Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 20:24:31 +0000 Subject: [PATCH 36/45] fix: price compiled fragments using the native physical cost model --- control_plane/src/physical/workload_cost/automatic.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/control_plane/src/physical/workload_cost/automatic.rs b/control_plane/src/physical/workload_cost/automatic.rs index 0198a3da3..6f0be5883 100644 --- a/control_plane/src/physical/workload_cost/automatic.rs +++ b/control_plane/src/physical/workload_cost/automatic.rs @@ -541,6 +541,13 @@ pub(super) fn estimate( let mut detail = json!({"input_rows":input_rows}); let mut network = 0.0; let cpu = match node { + Node::PhysicalFragment { dag, .. } => { + let program: Value = + serde_json::from_slice(dag).map_err(|error| invalid(error.to_string()))?; + let (cpu, workspace) = physical_work(&program, input_rows)?; + detail = json!({"input_rows":input_rows,"physical_program":program,"workspace_bytes_bound":workspace}); + cpu + } Node::Physical { .. } => { let (cpu, workspace) = snapshot_work(entry, input_rows)?; detail = json!({"input_rows":input_rows,"physical_program":entry.physical_dag,"workspace_bytes_bound":workspace}); From ba48bf1b3899e554b3d88bc10ed3d76bf07e5af9 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 20:28:59 +0000 Subject: [PATCH 37/45] fix: identify the incompatible physical binding schema --- crates/asap_types/src/query_plan/native.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index bf5108282..06ec40294 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -449,9 +449,11 @@ impl QueryPlanEntry { || output.schema.fields.iter().any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(DataType::Utf8 | DataType::Float64 | DataType::Timestamp))) } { - return Err(invalid( - "physical program has incompatible input or result schema", - )); + return Err(invalid(&format!( + "physical program has incompatible input or result schema: inputs {:?}; output {:?}", + dag.input_contracts().map(|(id, contract)| (id, &contract.schema)).collect::>(), + dag.output_contract(dag.roots()[0]).map_err(|error| QueryPlanError::Invalid(error.to_string()))?.schema, + ))); } Ok(dag) } From 8a3525ff1afef1fdb49f64b88f497cb97102e533 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 20:54:46 +0000 Subject: [PATCH 38/45] test: assert physical input budget error directly --- .../asap_query_engine/logical_dag/native_values.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index facb54465..b0dd8febd 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -522,8 +522,7 @@ mod tests { vec![vec![Value::Float64(1.)]], )?) }) - .err() - .expect("input must exceed the byte budget"); + .expect_err("input must exceed the byte budget"); assert!( matches!(error, EngineError::Physical(Error::MemoryLimit)), "{error}" From c73365ea315e829d8c6748da1258ea2d942a3bab Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:08:02 +0000 Subject: [PATCH 39/45] fix: finalize direct Planner proposals before physical binding --- control_plane/src/physical/compiler.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8d756986f..8b71ac958 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1000,6 +1000,8 @@ impl BackendLocalPlanningInput { let asap_aware_mapping::Replacement::Summary(root) = candidate.replacement else { continue; }; + let root = asap_aware_mapping::replacement::finalize_query_candidate(root, &typed) + .map_err(|error| CompileError::Snapshot(error.to_string()))?; let compiled = asap_physical_operators::physical_planner::promql_rows::compile_current_series_readout(&root) .or_else(|_| asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(&root).map(|(_, program)| program)); if let Ok(physical) = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&root) { From 4129a50105ef83eaf8aed432698b8c61080b85f0 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:23:36 +0000 Subject: [PATCH 40/45] style: format traced native input binding --- .../asap_query_engine/logical_dag/native_values.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 14ccfabfa..bda5f42e9 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -753,9 +753,7 @@ fn execute_batches( let graph = { let _binding = tracing::debug_span!(target: "asap_runtime_debug", "physical_input_binding", stage = "physical.bind_inputs", input_count = sources.len(), input_bytes, input_kind = "native_batch").entered(); - program - .instantiate(sources) - .map_err(EngineError::from)? + program.instantiate(sources).map_err(EngineError::from)? }; let context = dag::RunContext::new( dag::Scope::Query { From 372350b3ff8e8c2bd47e0d88aea36357332fa1cc Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:43:06 +0000 Subject: [PATCH 41/45] fix: validate composed query accuracy before deployment admission --- control_plane/src/physical/compiler.rs | 34 ++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8b71ac958..2e365eb01 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1441,7 +1441,12 @@ impl DeploymentPlanCompiler { validate_evidence(&query.query_id, e, &environment)?; } let node = query.selected_plan_root.clone(); - reject_uncertified_readouts(&query.query_id, &node, environment.target)?; + reject_uncertified_readouts( + &query.query_id, + &node, + &query.accuracy_target, + environment.target, + )?; let selected = if super::maintained_population::supported_node(&node) { Vec::new() } else { @@ -1455,7 +1460,12 @@ impl DeploymentPlanCompiler { })? }; if !selected.is_empty() { - reject_uncertified_readouts(&query.query_id, &node, environment.target)?; + reject_uncertified_readouts( + &query.query_id, + &node, + &query.accuracy_target, + environment.target, + )?; } let selected = selected .into_iter() @@ -3995,6 +4005,7 @@ fn validate_executable_subdag(node: &Rc) -> Result<(), String> { fn reject_uncertified_readouts( query_id: &str, root: &Rc, + accuracy: &AccuracyTarget, target: PhysicalDeploymentTarget, ) -> Result<(), CompileError> { let dag = planner_types::post_asap::compile_executable_dag(root).map_err(|error| { @@ -4003,6 +4014,25 @@ fn reject_uncertified_readouts( reason: format!("invalid executable subDAG: {error}"), } })?; + // Leaf guarantees do not certify a composition (for example, division of + // two approximate quantiles). Admission checks the complete query result. + if dag.nodes.iter().any(|node| { + matches!( + node.payload, + planner_types::post_asap::ExecutableOperatorPayload::SummaryEstimate { .. } + ) + }) && root.guarantee.as_ref().is_none_or(|guarantee| { + !asap_aware_mapping::accuracy::AccuracyModel::satisfies( + &DefaultAccuracyModel, + guarantee, + accuracy, + ) + }) { + return Err(CompileError::Query { + query_id: query_id.into(), + reason: "selected query result has no certified accuracy guarantee satisfying the requested accuracy".into(), + }); + } for node in &dag.nodes { if target != PhysicalDeploymentTarget::BackendLocalRemoteWrite && node.guarantee.as_ref().is_some_and(|guarantee| { From cc1e8c6a27121dc016855a816469c92d1f71ea8f Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:43:14 +0000 Subject: [PATCH 42/45] fix: describe Planner physical fragments in runtime diagnostics --- crates/asap_types/src/query_plan.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 8d70965b9..8924aaa84 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -790,6 +790,7 @@ impl QueryPlanNode { R::Subquery { .. } => "logical/subquery", }, Self::Physical { .. } => "physical_dag", + Self::PhysicalFragment { .. } => "physical_fragment", Self::Scalar { .. } => "scalar", Self::Binary { .. } => "binary", Self::ReduceSum { .. } => "reduce_sum", @@ -867,6 +868,9 @@ impl QueryPlanNode { offset_ms, } => format!("range_ms={range_ms} step_ms={step_ms} offset_ms={offset_ms}"), }, + Self::PhysicalFragment { + inputs, row_input, .. + } => format!("input_count={} row_input={row_input}", inputs.len()), Self::Physical { inputs, source_nodes, From bb757f5bab501ee6c11da43cd7e59a11ae9ec905 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:44:32 +0000 Subject: [PATCH 43/45] fix: retain leaf diagnostics before checking composed accuracy --- control_plane/src/physical/compiler.rs | 39 +++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 2e365eb01..8c1723399 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4014,25 +4014,6 @@ fn reject_uncertified_readouts( reason: format!("invalid executable subDAG: {error}"), } })?; - // Leaf guarantees do not certify a composition (for example, division of - // two approximate quantiles). Admission checks the complete query result. - if dag.nodes.iter().any(|node| { - matches!( - node.payload, - planner_types::post_asap::ExecutableOperatorPayload::SummaryEstimate { .. } - ) - }) && root.guarantee.as_ref().is_none_or(|guarantee| { - !asap_aware_mapping::accuracy::AccuracyModel::satisfies( - &DefaultAccuracyModel, - guarantee, - accuracy, - ) - }) { - return Err(CompileError::Query { - query_id: query_id.into(), - reason: "selected query result has no certified accuracy guarantee satisfying the requested accuracy".into(), - }); - } for node in &dag.nodes { if target != PhysicalDeploymentTarget::BackendLocalRemoteWrite && node.guarantee.as_ref().is_some_and(|guarantee| { @@ -4063,6 +4044,25 @@ fn reject_uncertified_readouts( }); } } + // Leaf guarantees do not certify a composition (for example, division of + // two approximate quantiles). Admission checks the complete query result. + if dag.nodes.iter().any(|node| { + matches!( + node.payload, + planner_types::post_asap::ExecutableOperatorPayload::SummaryEstimate { .. } + ) + }) && root.guarantee.as_ref().is_none_or(|guarantee| { + !asap_aware_mapping::accuracy::AccuracyModel::satisfies( + &asap_aware_mapping::DefaultAccuracyModel, + guarantee, + accuracy, + ) + }) { + return Err(CompileError::Query { + query_id: query_id.into(), + reason: "selected query result has no certified accuracy guarantee satisfying the requested accuracy".into(), + }); + } Ok(()) } @@ -5870,6 +5870,7 @@ pub(crate) mod tests { let error = reject_uncertified_readouts( "q", &request.queries[0].selected_plan_root, + &request.queries[0].accuracy_target, PhysicalDeploymentTarget::DistributedCollectors, ) .unwrap_err(); From f865488fd4f1da1203e4e39646c207474cd4518d Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:48:25 +0000 Subject: [PATCH 44/45] test: keep cadence fixture outputs independent and accuracy-valid --- control_plane/src/physical/compiler.rs | 37 ++++++++++---------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8c1723399..a573d452c 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -7065,35 +7065,26 @@ pub(crate) mod tests { Query("sum_over_time(n[1m])".into()); let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); let (right, _) = right.into_physical_compilation_request().unwrap(); - let left = request.queries[0].selected_plan_root.clone(); - let right = right.queries[0].selected_plan_root.clone(); - request.queries[0].selected_plan_root = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - lhs: left.clone(), - rhs: right, - operator: planner_types::post_asap::BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Add, - ), - vector_match: None, - }, - }, - schema: left.schema.clone(), - guarantee: None, - }); - let text = "quantile(0.9, sum_over_time(m[1m])) + sum_over_time(n[1m])"; - request.queries[0].query_string = text.into(); + // These are independent query outputs. Adding an approximate quantile + // to an exact sum would require a separate composed accuracy proof. + let mut raw_query = right.queries[0].clone(); + raw_query.query_id = "raw-query".into(); + request.queries.push(raw_query); request .query_workload .as_mut() .unwrap() .repeating_queries .as_mut() - .unwrap()[0] - .query = Query(text.into()); + .unwrap() + .push( + right + .query_workload + .unwrap() + .repeating_queries + .unwrap() + .remove(0), + ); prepare_window_implementations(&mut request.queries[0], &model, env.target, 0).unwrap(); request.queries[0] .window_realization_candidates From 84b8cf4b42659fe034b5124c13bae25f388ec01b Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 28 Sep 2026 21:49:58 +0000 Subject: [PATCH 45/45] test: construct both cadence queries through workload planning --- control_plane/src/physical/compiler.rs | 36 ++++++++------------------ 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index a573d452c..1abb20168 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -7060,40 +7060,26 @@ pub(crate) mod tests { interval: RepetitionInterval(45_000), evaluation_phase: planner_types::workload::TimestampMs(0), }; - let mut right = snapshot.clone(); - right.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("sum_over_time(n[1m])".into()); - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - let (right, _) = right.into_physical_compilation_request().unwrap(); - // These are independent query outputs. Adding an approximate quantile - // to an exact sum would require a separate composed accuracy proof. - let mut raw_query = right.queries[0].clone(); - raw_query.query_id = "raw-query".into(); - request.queries.push(raw_query); - request + // Keep both outputs independent: composing a quantile with a sum would + // require an additional accuracy proof unrelated to cadence selection. + let mut raw_entry = entry.clone(); + raw_entry.query = Query("sum_over_time(n[1m])".into()); + snapshot .query_workload - .as_mut() - .unwrap() .repeating_queries .as_mut() .unwrap() - .push( - right - .query_workload - .unwrap() - .repeating_queries - .unwrap() - .remove(0), - ); - prepare_window_implementations(&mut request.queries[0], &model, env.target, 0).unwrap(); - request.queries[0] - .window_realization_candidates - .retain(|candidate| { + .push(raw_entry); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); + for query in &mut request.queries { + prepare_window_implementations(query, &model, env.target, 0).unwrap(); + query.window_realization_candidates.retain(|candidate| { matches!( candidate.layout, asap_types::WindowMaterializationLayout::Pane { .. } ) }); + } let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan