diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 575d8609f..4be16cfd4 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1309,6 +1309,33 @@ fn preserve_invalid_exact_fallback_roots( Ok(()) } +/// Planner compiles query-time computation over stored readouts. A selected +/// root whose computation it cannot compile keeps no summary state; the +/// external exact engine evaluates the original query. +fn preserve_uncompiled_computation_roots( + queries: &mut [QueryCompilationInput], + canonical_roots: &[Rc], +) -> Result<(), CompileError> { + for (index, query) in queries.iter_mut().enumerate() { + let root = &query.selected_plan_root; + if super::maintained_population::supported_node(root) + || !crate::query_plan::is_query_computation(root) + || crate::query_plan::compile_query_computation(root).is_ok() + { + continue; + } + let parsed = original_root(query, index, canonical_roots)?; + query.selected_plan_root = + crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { + CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + } + })?; + } + Ok(()) +} + /// A MetricsQL query whose only selected states are Prometheus-specific /// counter readouts has no backend materialization to bind. Keep the original /// query as one native exact root. Mixed queries retain their other selected @@ -1445,6 +1472,7 @@ impl DeploymentPlanCompiler { &request.canonical_roots, request.allow_mixed_summary_and_exact_execution, )?; + preserve_uncompiled_computation_roots(&mut request.queries, &request.canonical_roots)?; if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite && !request.allow_mixed_summary_and_exact_execution @@ -1539,13 +1567,9 @@ impl DeploymentPlanCompiler { planner_types::post_asap::ExactKind::Max, _ ) - ) || crate::query_plan::query_time::selected_range_max_materialization( - &query.query_string, + ) || crate::query_plan::query_time::is_range_max_materialization( &state.node, - ) - .ok() - .flatten() - .is_some()) + )) // An Ephemeral state is rebuilt at query time, not maintained. && !placement.is_ephemeral(query_index, &state.node) }) @@ -2180,23 +2204,10 @@ impl DeploymentPlanCompiler { full_history: false, cumulative_readout: true, }; - // A whole-query native fallback need not be expressible in the local - // query-time algebra (for example an ERP-rejected entropy readout). - // Retain its native boundary without discarding other workload roots. - let native_root = request.allow_mixed_summary_and_exact_execution - && if let SummaryExpr::KeepPreAsap(expr) = &query.selected_plan_root.expr { - let original = original_root(query, query_index, &request.canonical_roots)?; - expr.as_ref() == &original - && crate::query_plan::query_time::compile_logical( - query.query_id.clone(), - canonical.clone(), - instant, - FallbackPolicy::ExactBackend, - ) - .is_err() - } else { - false - }; + // A query Planner keeps pre-ASAP has no stored input; Prometheus + // evaluates it whole. + let exact_root = request.allow_mixed_summary_and_exact_execution + && matches!(query.selected_plan_root.expr, SummaryExpr::KeepPreAsap(_)); let native_rate = if population_operators[query_index].is_none() && placement.raw_program(query_index).is_none() { @@ -2233,7 +2244,12 @@ impl DeploymentPlanCompiler { None }; let mut entry = if let Some(raw) = placement.raw_program(query_index) { - Ok(raw_query_time_entry(query, canonical.clone(), raw)?) + Ok(raw_query_time_entry( + query, + canonical.clone(), + &original_root(query, query_index, &request.canonical_roots)?, + raw, + )?) } else if let Some((source, program)) = native_rate { let native_state_binding = if let SummaryExpr::SummaryAgg { family: @@ -2272,7 +2288,7 @@ impl DeploymentPlanCompiler { fallback: FallbackPolicy::ExactBackend, } } else { - crate::query_plan::compile_bound_composable_mapped( + crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &source, @@ -2309,6 +2325,9 @@ impl DeploymentPlanCompiler { max_bytes: request .retained_summary_memory_budget_bytes .unwrap_or(64 * 1024 * 1024), + drop_metric_name: !crate::query_plan::result_keeps_metric_name( + &original_root(query, query_index, &request.canonical_roots)?, + ), }, ); entry.root = root; @@ -2380,25 +2399,10 @@ impl DeploymentPlanCompiler { instant, fallback: FallbackPolicy::ExactBackend, }) - } else if request.allow_mixed_summary_and_exact_execution && !native_root { - crate::query_plan::compile_bound_composable_mapped( - query.query_id.clone(), - canonical.clone(), - &query.selected_plan_root, - instant, - FallbackPolicy::ExactBackend, - binding, - |node, query_node| { - if let Some(post_asap_node) = executable_dags[query_index] - .as_ref() - .and_then(|compiled| compiled.node_ids.node_id(node)) - { - query_node_bindings.insert((query_index, post_asap_node), query_node); - } - }, - ) + } else if exact_root { + Ok(exact_query_entry(query, canonical.clone(), instant)) } else { - crate::query_plan::compile_bound_mapped( + match crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &query.selected_plan_root, @@ -2413,16 +2417,22 @@ impl DeploymentPlanCompiler { query_node_bindings.insert((query_index, post_asap_node), query_node); } }, - ) + ) { + // A readout with no maintained state leaves no local plan; + // the query is forwarded whole. A query that did compile + // state must not maintain it unread, so it fails instead. + Err(crate::query_plan::QueryPlanError::UnsupportedNode(_)) + if executable_dags[query_index].is_none() => + { + query_node_bindings.retain(|(index, _), _| *index != query_index); + Ok(exact_query_entry(query, canonical.clone(), instant)) + } + other => other.map_err(|error| CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + }), + } }?; - if request.allow_mixed_summary_and_exact_execution - && placement.raw_program(query_index).is_none() - { - // Any Planner-selected leaf without a physical summary binding - // is an exact subtree boundary. Deployed plans never retain a - // backend-local range index leaf. - crate::query_plan::query_time::finalize_query_time_nodes(&mut entry)?; - } if frontend == QueryFrontend::MetricsQl { entry.language = crate::query_plan::QueryLanguage::MetricsQl; } @@ -2433,14 +2443,6 @@ impl DeploymentPlanCompiler { .as_ref() .map(|candidate| &candidate.query), )?; - let linked = crate::query_plan::physical_values::compile(&mut entry)?; - for ((index, _), node) in &mut query_node_bindings { - if *index == query_index { - if let Some(root) = linked.get(node) { - *node = *root; - } - } - } let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); if query_entries.insert(catalog_key, entry).is_some() { return Err(CompileError::Query { @@ -2717,9 +2719,35 @@ impl DeploymentPlanCompiler { /// Native query-time execution of a query that keeps no state: each raw input /// of the retained program is read by its range-selector `Scan`. +/// The whole query forwarded unchanged to the external exact engine. +fn exact_query_entry( + query: &QueryCompilationInput, + canonical: String, + instant: InstantExecution, +) -> QueryPlanEntry { + let root = crate::query_plan::QueryNodeId(0); + QueryPlanEntry { + physical_dag: None, + language: crate::query_plan::QueryLanguage::PromQl, + query_id: query.query_id.clone(), + canonical_query: canonical, + fixed_evaluation: None, + root, + nodes: BTreeMap::from([( + root, + crate::query_plan::QueryPlanNode::ExactFallback { + reason: "no stored input covers this query".into(), + }, + )]), + instant, + fallback: FallbackPolicy::ExactBackend, + } +} + fn raw_query_time_entry( query: &QueryCompilationInput, canonical: String, + root_expr: &QueryExpr, raw: &placement::RawQueryTimeProgram, ) -> Result { let mut nodes = BTreeMap::new(); @@ -2744,6 +2772,7 @@ fn raw_query_time_entry( inputs, source_nodes, max_bytes: 64 * 1024 * 1024, + drop_metric_name: !crate::query_plan::result_keeps_metric_name(root_expr), }, ); let entry = QueryPlanEntry { @@ -4458,17 +4487,14 @@ fn collect_selected_materializations( grouping.clone(), selected, )?; - // Explicit external authoritative values do not need duplicate local state. - if !composable { - walk( - values, - readout, - composable, - native_maintenance, - grouping.clone(), - selected, - )?; - } + walk( + values, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::ValueOperation { child, .. } => { walk( @@ -4919,30 +4945,6 @@ pub(crate) mod tests { assert!(error.to_string().contains("row-update executor"), "{error}"); } - // Compiler preserves the Planner's conditional-average execution guard. - #[test] - fn temporal_average_lowers_with_finite_division_guard() { - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let request = request("average", "avg_over_time(a[1m])"); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!(plan - .query_plan - .entries - .values() - .flat_map(|entry| entry.nodes.values()) - .flat_map( - |node| crate::query_plan::physical_values::operator_parameters( - node, - "VectorBinary" - ) - ) - .any(|parameters| parameters["operator"]["checked_finite_division"] == true)); - } - // The Planner's minimum state lowers without reconstructing direction from text. #[test] fn minimum_retains_its_typed_direction() { @@ -5012,7 +5014,7 @@ pub(crate) mod tests { fn complete_cost_selection_preserves_shared_sum_panes() { let mut snapshot = planning_snapshot(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(a[1m]) / sum_over_time(a[10m])".into()); + entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) .compile_promql() @@ -5088,7 +5090,6 @@ pub(crate) mod tests { #[test] fn issue_701_702_temporal_workloads_have_warm_candidates() { for text in [ - "avg_over_time(data[5m])", "min_over_time(data[5m])", "quantile_over_time(0.9,data[5m])", ] { @@ -5125,6 +5126,52 @@ pub(crate) mod tests { } } + // Without mixed execution, a computation Planner can compile over readouts + // whose states the deployment does not select forwards whole instead of + // failing the deployment. + #[test] + fn unselected_states_under_compilable_computation_forward_whole() { + let query = "sum by (job) (rate(a[5m])) / sum by (job) (rate(b[5m]))"; + let workload = request("ratio", query); + assert!(!workload.allow_mixed_summary_and_exact_execution); + let plan = DeploymentPlanCompiler + .compile_promql(workload, environment(10_000)) + .unwrap(); + let entry = plan.query_plan.lookup(query).unwrap(); + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + )); + assert!(plan.precompute_plan.materializations.is_empty()); + } + + // avg_over_time divides two per-series readouts. Planner does not yet + // match per-series rows in a Binary, so no candidate keeps local state and + // the exact engine evaluates the query. + #[test] + fn per_series_average_has_no_warm_candidate_until_planner_matches_series() { + let mut snapshot = planning_snapshot(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("avg_over_time(data[5m])".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + for candidate in + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + { + let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) + else { + continue; + }; + assert!(plan.precompute_plan.materializations.is_empty()); + assert!(plan + .query_plan + .entries + .values() + .all(|entry| entry.materialization_bindings().is_empty())); + } + } + // Quantile rank error does not certify relative error of a quotient. #[test] fn uncertified_quantile_ratios_retain_exact_execution() { @@ -5141,13 +5188,13 @@ pub(crate) mod tests { .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{text}"); assert!( - plan.query_plan - .entries - .values() - .all(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactFallback { .. } - ))), + plan.query_plan.entries.values().all(|entry| { + entry.nodes.len() == 1 + && matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + ) + }), "{text}" ); } @@ -5443,9 +5490,10 @@ pub(crate) mod tests { assert!(plan.precompute_plan.materializations.is_empty()); } - // Native exact values remain explicit; unsupported heaps publish no stored state. + // A ranking over an exact PromQL subtree has no stored input: Prometheus + // evaluates the whole query and the backend computes nothing. #[test] - fn hybrid_rate_topk_preserves_the_original_exact_subquery() { + fn hybrid_rate_topk_forwards_the_whole_exact_query() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -5464,23 +5512,12 @@ pub(crate) mod tests { .compile_promql(request, environment) .unwrap(); let entry = plan.query_plan.lookup(query).unwrap(); - use crate::query_plan::{query_time::QueryTimeOperator, QueryPlanNode}; - let limits = crate::query_plan::physical_values::operator_parameters( + use crate::query_plan::QueryPlanNode; + assert_eq!(entry.nodes.len(), 1); + assert!(matches!( &entry.nodes[&entry.root], - "Limit", - ); - assert_eq!(limits.len(), 1); - assert_eq!(limits[0]["n"], 2); - assert_eq!( - entry - .nodes - .values() - .filter(|node| matches!(node, QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, .. - } if query == "sum by (job) (rate(m[1m]))")) - .count(), - 1 - ); + QueryPlanNode::ExactFallback { .. } + )); assert!(entry.materialization_bindings().is_empty()); assert!(plan.precompute_plan.materializations.is_empty()); let artifact = plan.to_publication_artifact().unwrap(); @@ -6167,19 +6204,27 @@ pub(crate) mod tests { } #[test] - fn metricsql_counter_gate_rejects_mixed_input_snapshots() { - let mut workload = request("mixed", "max_over_time(m[1m]) + rate(m[1m])"); + fn metricsql_counter_gate_forwards_mixed_operands_whole() { + // A counter branch that MetricsQL cannot read from stored state leaves + // no local operand; the exact engine evaluates the original query. + let query = "max_over_time(m[1m]) + rate(m[1m])"; + let mut workload = request("mixed", query); workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let Err(error) = DeploymentPlanCompiler.compile_metricsql(workload, deployment) else { - panic!("mixed local/external snapshots must fail deployment binding"); - }; - assert!( - error.to_string().contains("common snapshot proof"), - "{error}" - ); + let plan = DeploymentPlanCompiler + .compile_metricsql(workload, deployment) + .unwrap(); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert_eq!(entry.language, crate::query_plan::QueryLanguage::MetricsQl); + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + )); + assert_eq!(entry.canonical_query, query); + assert!(entry.materialization_bindings().is_empty()); + assert!(plan.precompute_plan.materializations.is_empty()); } #[test] @@ -6787,18 +6832,13 @@ pub(crate) mod tests { ); assert!(!entry.materialization_bindings().is_empty()); } + // The grouped division is a Planner join over the two readouts. assert!(bundle .query_plan .entries .values() .flat_map(|entry| entry.nodes.values()) - .any( - |node| !crate::query_plan::physical_values::operator_parameters( - node, - "VectorBinary" - ) - .is_empty() - )); + .any(|node| !crate::query_plan::operator_parameters(node, "Join").is_empty())); } #[test] @@ -7233,7 +7273,7 @@ pub(crate) mod tests { for interval in [10_000, 60_000] { for (rhs, expected_states) in [("a", 1), ("b", 2), ("a{job=\"x\"}", 2)] { let mut snapshot = planning_snapshot(); - let query = format!("sum_over_time(a[1m]) / sum_over_time({rhs}[10m])"); + let query = format!("sum(sum_over_time(a[1m])) / sum(sum_over_time({rhs}[10m]))"); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); @@ -7319,7 +7359,7 @@ pub(crate) mod tests { #[test] fn explicit_window_quotes_are_not_repriced_for_sharing() { let mut snapshot = planning_snapshot(); - let query = "sum_over_time(a[1m]) / sum_over_time(a[10m])"; + let query = "sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))"; let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); diff --git a/control_plane/src/physical/compiler/placement.rs b/control_plane/src/physical/compiler/placement.rs index 68034522c..abb1e1cb9 100644 --- a/control_plane/src/physical/compiler/placement.rs +++ b/control_plane/src/physical/compiler/placement.rs @@ -201,31 +201,11 @@ pub(super) fn place( .and_then(|root| raw_query_time_program(root).ok()) }) .collect(); - // A leaf the query-time lowering can externalize reads Prometheus directly. - let externalizable = |state: &SummaryNode, query: usize| { - raw_bindable - && (crate::query_plan::query_time::selected_counter_materialization( - &queries[query].query_string, - state, - ) - .ok() - .flatten() - .is_some() - || crate::query_plan::query_time::selected_range_max_materialization( - &queries[query].query_string, - state, - ) - .ok() - .flatten() - .is_some()) - }; let horizon = first.summary_lifecycle_inputs.horizon_seconds; let mut ephemeral = vec![false; states.len()]; let mut decisions = Vec::new(); for (state_index, (state, consumers)) in states.iter().enumerate() { - let bindable = consumers - .iter() - .all(|&query| raw_programs[query].is_some() || externalizable(state, query)); + let bindable = consumers.iter().all(|&query| raw_programs[query].is_some()); let lead = &queries[consumers[0]].summary_lifecycle_inputs; let interval = consumers .iter() @@ -283,8 +263,7 @@ pub(super) fn place( let mut changed = false; for (query, owned) in query_states.iter().enumerate() { let realizable = owned.iter().all(|state| { - index_of(state).is_some_and(|i| ephemeral[i]) - && (raw_programs[query].is_some() || externalizable(state, query)) + index_of(state).is_some_and(|i| ephemeral[i]) && raw_programs[query].is_some() }); if realizable { continue; diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index ba2033fba..1d97ed935 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -205,9 +205,6 @@ fn query_node_label(node: &QueryPlanNode) -> String { QueryPlanNode::Logical { operator, .. } => { format!("Logical\n{}", query_time_label(operator)) } - QueryPlanNode::Scalar { value } => format!("Scalar\n{value}"), - QueryPlanNode::Binary { operator, .. } => format!("Binary\n{operator:?}"), - QueryPlanNode::ReduceSum { .. } => "ReduceSum".into(), QueryPlanNode::ReadMaterialization { binding } => format!( "ReadMaterialization\n{}\nwindow={}ms\nlookback={:?}", binding.materialization.fingerprint(), @@ -228,15 +225,6 @@ fn query_time_label(operator: &QueryTimeOperator) -> &'static str { QueryTimeOperator::ExactSubquery { .. } => "ExactSubquery", QueryTimeOperator::CandidateExactSubquery { .. } => "CandidateExactSubquery", QueryTimeOperator::Scan { .. } => "Scan", - QueryTimeOperator::UnaryNegate => "UnaryNegate", - QueryTimeOperator::VectorToScalar => "VectorToScalar", - QueryTimeOperator::Aggregate { .. } => "Aggregate", - QueryTimeOperator::Limit { .. } => "Limit", - QueryTimeOperator::Binary { .. } => "Binary", - QueryTimeOperator::Temporal { .. } => "Temporal", - QueryTimeOperator::Sort { .. } => "Sort", - QueryTimeOperator::HistogramQuantile => "HistogramQuantile", - QueryTimeOperator::Subquery { .. } => "Subquery", } } diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index e5740894d..56d3b5d8a 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -1360,7 +1360,7 @@ mod tests { .store_per_byte_second = store; let q = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; q.query = planner_types::workload::Query( - "max_over_time(a[1m]) + max_over_time(b[1m])".into(), + "sum(max_over_time(a[1m])) + sum(max_over_time(b[1m]))".into(), ); q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( crate::types::AccuracyTarget::Exact, diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 3cf562d80..8db586aa6 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1,9 +1,12 @@ //! Control-plane lowering from Planner IR to the shared installed query DAG. //! Serving consumes asap_types::query_plan; compilation stays in this component. +//! +//! The backend lowers only stored-state readouts itself. Query-time +//! computation over their decoded values is one Planner-compiled physical DAG +//! per maximal computation region. use asap_types::physical_plan_codec::PhysicalPlanCodec; mod clickhouse_exact; -pub mod physical_values; pub mod query_time; pub use asap_types::query_plan::*; @@ -37,7 +40,6 @@ where nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, - logical_source: None, preserve_relational: false, lowered: Some(&mut lowered), }; @@ -78,7 +80,6 @@ where nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, - logical_source: None, preserve_relational: true, lowered: Some(&mut lowered), }; @@ -96,123 +97,233 @@ where }) } -/// Compile a composable query while exposing the stable mapping from -/// Planner semantic nodes to installed query nodes. The control-plane -/// physical compiler uses this to persist backend placement without -/// relying on pointer values or reconstructing query shape later. -pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), -{ - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - query_time::finalize_query_time_nodes(&mut entry)?; - Ok(entry) +/// Query-time computation over stored readouts, compiled once by Planner. +pub(crate) struct QueryComputation { + pub(crate) physical: asap_physical_operators::physical_planner::CompiledPhysicalDag, + /// Readout feeding each physical input contract, keyed by contract ID. + frontier: BTreeMap>, + computed: Vec>, + pruning: Option, } -fn compile_native_fragment( +/// Is this node decoded from stored state by the backend, rather than +/// computed over values? An exact aggregate over values is a query-time +/// reduction; a sketch over values is a derived stored output. +fn stored_readout(node: &SummaryNode) -> bool { + match &node.expr { + SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + .. + } => exact_accumulator_value_source(node) + .is_some_and(|source| !std::ptr::eq(source, node) && stored_readout(source)), + SummaryExpr::SummaryEstimate { .. } + | SummaryExpr::SummaryMerge { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } => true, + SummaryExpr::SummaryAgg { child, family, .. } => { + matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + || !matches!(family, SummaryFamilyType::ExactAggregate(..)) + } + _ => false, + } +} + +/// Does lowering `node` start a query-time computation region? +pub(crate) fn is_query_computation(node: &SummaryNode) -> bool { + !matches!( + node.expr, + SummaryExpr::KeepPreAsap(_) + | SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::MaintainPopulation { .. } + | planner_types::post_asap::ValueOperation::ReadPopulation { .. }, + .. + } + ) && !stored_readout(node) +} + +/// Compile the maximal query-time region rooted at `root`. Its inputs are the +/// stored readouts below it; an exact PromQL selector inside the region has no +/// stored input, so the region is unsupported and the query runs exactly. +pub(crate) fn compile_query_computation( root: &Rc, - query_inputs: &[QueryNodeId], -) -> Result { - use asap_physical_operators::physical_planner::{compile, InputContract}; - use planner_types::post_asap::{compile_post_asap_dag, EdgeRole}; - let invalid = |e: String| QueryPlanError::Invalid(e); - let dag = compile_post_asap_dag(root).map_err(|e| invalid(e.to_string()))?; - let mut edges = dag - .edges - .iter() - .filter(|e| e.consumer == dag.root) - .collect::>(); - edges.sort_by_key(|e| match e.role { - EdgeRole::Left => 0, - EdgeRole::Input => 1, - EdgeRole::Right => 2, - }); - if edges.len() != query_inputs.len() { - return Err(invalid("physical frontier arity mismatch".into())); +) -> Result { + use asap_physical_operators::physical_planner::{compile, promql_fallback, InputContract}; + use planner_types::post_asap::{ + compile_post_asap_dag_with_node_ids, EdgeRole, PostAsapOperatorPayload as Payload, + }; + let unsupported = |message: String| QueryPlanError::UnsupportedNode(message); + let finalized = finalize_query_value(root); + let compilation = compile_post_asap_dag_with_node_ids(&finalized) + .map_err(|error| unsupported(error.to_string()))?; + let dag = &compilation.dag; + let mut pending = vec![dag.root]; + let mut visited = std::collections::BTreeSet::new(); + let mut contracts = BTreeMap::new(); + let mut frontier = BTreeMap::new(); + let mut computed = Vec::new(); + while let Some(id) = pending.pop() { + if !visited.insert(id) { + continue; + } + let node = dag + .nodes + .iter() + .find(|node| node.id == id) + .ok_or_else(|| unsupported("missing Planner node".into()))?; + let semantic = compilation.node_ids.summary_node(id); + let readout = id != dag.root && semantic.is_some_and(|semantic| stored_readout(semantic)); + if !readout { + if let Payload::Fallback { expression } = &node.payload { + if !promql_fallback::raw_series(expression) + .map_err(|error| unsupported(error.to_string()))? + .is_empty() + { + return Err(unsupported( + "query-time computation reads an exact PromQL selector".into(), + )); + } + } + if let Some(SummaryNode { + expr: SummaryExpr::BinaryOp { .. }, + guarantee, + .. + }) = semantic.map(Rc::as_ref) + { + // Planner certifies the combined value; without that, the + // backend must not publish the arithmetic result. + if guarantee.as_ref().is_none_or(|g| g.has_unknown()) { + return Err(unsupported( + "query-time arithmetic has no accuracy guarantee".into(), + )); + } + } + computed.extend(semantic.cloned()); + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + continue; + } + let semantic = semantic.expect("readout has a semantic node"); + let source = exact_accumulator_value_source(semantic).unwrap_or(semantic); + if matches!( + semantic.expr, + SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + ) || (matches!( + source.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ), + .. + } + ) && !exact_value_executable(source)) + || node + .output_schema + .fields + .iter() + .any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))) + { + return Err(unsupported( + "query-time computation consumes state that has no decoded readout".into(), + )); + } + contracts.insert( + u64::from(id.0), + InputContract::bounded(std::sync::Arc::new(node.output_schema.clone())), + ); + frontier.insert(u64::from(id.0), Rc::clone(semantic)); } - let bindings = edges - .iter() - .zip(query_inputs) - .map(|(edge, &id)| (u64::from(edge.producer.0), id)) - .collect::>(); - let contracts = edges - .iter() - .map(|edge| { - ( - u64::from(edge.producer.0), - InputContract::bounded(std::sync::Arc::new(edge.intermediate_schema.clone())), - ) - }) - .collect(); - let physical = - compile(&dag, contracts, &[u64::from(dag.root.0)]).map_err(|e| invalid(e.to_string()))?; - let row_input = physical - .row_source(physical.roots()[0]) - .and_then(|source| physical.input_contracts().position(|(id, _)| id == source)); - let pruning = if let SummaryExpr::RelationalJoin { - left, - right, - pred, - pruning: Some(completeness), - .. - } = &root.expr - { - Some(asap_types::query_plan::PruningInputContract { - candidate_input: physical - .input_contracts() - .position(|(id, _)| bindings[&id] == query_inputs[1]) - .unwrap(), - keys: asap_physical_operators::physical_planner::equijoin_keys( - pred, - &left.schema, - &right.schema, + // A pruning contract is bound only at the fragment root. + if dag.nodes.iter().any(|node| { + node.id != dag.root + && matches!( + node.payload, + Payload::RelationalJoin { + pruning: Some(_), + .. + } ) - .map_err(|e| invalid(e.to_string()))?, - completeness: completeness.clone(), - }) - } else { - None + }) { + return Err(unsupported( + "a pruned join must be the root of its query-time region".into(), + )); + } + let physical = compile(dag, contracts, &[u64::from(dag.root.0)]) + .map_err(|error| unsupported(error.to_string()))?; + let pruning = match &root.expr { + SummaryExpr::RelationalJoin { + left, + right, + pred, + pruning: Some(completeness), + .. + } => { + let candidates = dag + .edges + .iter() + .find(|edge| edge.consumer == dag.root && edge.role == EdgeRole::Right) + .map(|edge| u64::from(edge.producer.0)) + .ok_or_else(|| unsupported("pruned join has no candidate input".into()))?; + Some(PruningInputContract { + candidate_input: physical + .input_contracts() + .position(|(id, _)| id == candidates) + .ok_or_else(|| { + unsupported("pruning candidates must be a stored readout".into()) + })?, + keys: asap_physical_operators::physical_planner::equijoin_keys( + pred, + &left.schema, + &right.schema, + ) + .map_err(|error| unsupported(error.to_string()))?, + completeness: completeness.clone(), + }) + } + _ => None, }; - Ok(QueryPlanNode::PhysicalFragment { + Ok(QueryComputation { + physical, + frontier, + computed, pruning, - row_input, - inputs: physical - .input_contracts() - .map(|(id, _)| bindings[&id]) - .collect(), - dag: physical.encode().map_err(|e| invalid(e.to_string()))?, + }) +} + +/// A query-time exact aggregate yields accumulator state; its PromQL value is +/// that state finalized, as Planner does for stored exact states. +fn finalize_query_value(root: &Rc) -> Rc { + use planner_types::pre_asap::DataType; + if !matches!( + root.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(..), + .. + } + ) { + return Rc::clone(root); + } + let mut schema = root.schema.clone(); + for field in &mut schema.fields { + if matches!(field.dtype, SummaryFamilyType::ExactAggregate(..)) { + field.dtype = SummaryFamilyType::Plain(DataType::Float64); + } + } + Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: Rc::clone(root), + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, + }, + schema, + guarantee: root.guarantee.clone(), }) } @@ -221,7 +332,6 @@ struct DagCompiler<'a, F> { nodes: BTreeMap, seen: BTreeMap, bind: &'a mut F, - logical_source: Option, preserve_relational: bool, lowered: Option<&'a mut dyn FnMut(&Rc, QueryNodeId)>, } @@ -324,48 +434,35 @@ where Ok(id) } - fn graft( - &mut self, - id: QueryNodeId, - root: QueryNodeId, - nodes: BTreeMap, - ) -> Result { - let mut remap = BTreeMap::new(); - for local in nodes.keys() { - let global = if *local == root { - id - } else { - let next = QueryNodeId(self.next_id); - self.next_id += 1; - next - }; - remap.insert(*local, global); + /// One Planner physical fragment for the region, bound to backend readouts. + fn lower_computation(&mut self, root: &Rc) -> Result { + let computation = compile_query_computation(root)?; + let mut inputs = Vec::new(); + for (id, _) in computation.physical.input_contracts() { + inputs.push(self.lower(&computation.frontier[&id])?); } - for (local, mut physical) in nodes { - match &mut physical { - QueryPlanNode::Physical { inputs, .. } - | QueryPlanNode::PhysicalRelation { inputs, .. } - | QueryPlanNode::PhysicalFragment { inputs, .. } - | QueryPlanNode::Logical { inputs, .. } - | QueryPlanNode::SummaryMerge { inputs } - | QueryPlanNode::ExternalExact { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::Binary { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::SummaryEstimate { input, .. } - | QueryPlanNode::ExactReadout { input, .. } - | QueryPlanNode::ReduceSum { input, .. } => *input = remap[input], - QueryPlanNode::Scalar { .. } - | QueryPlanNode::ReadMaterialization { .. } - | QueryPlanNode::ExactFallback { .. } => {} + let physical = &computation.physical; + let row_input = physical + .row_source(physical.roots()[0]) + .and_then(|source| physical.input_contracts().position(|(id, _)| id == source)); + let id = QueryNodeId(self.next_id); + self.next_id += 1; + self.nodes.insert( + id, + QueryPlanNode::PhysicalFragment { + inputs, + dag: physical + .encode() + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?, + row_input, + pruning: computation.pruning, + }, + ); + for semantic in std::iter::once(root).chain(&computation.computed) { + self.seen.insert(Rc::as_ptr(semantic) as usize, id); + if let Some(lowered) = &mut self.lowered { + lowered(semantic, id); } - self.nodes.insert(remap[&local], physical); } Ok(id) } @@ -396,31 +493,12 @@ where } return Ok(child_id); } + if !self.preserve_relational && is_query_computation(node) { + return self.lower_computation(node); + } let id = QueryNodeId(self.next_id); self.next_id += 1; self.seen.insert(identity, id); - let query_time = match (&self.logical_source, &node.expr) { - (Some(original), SummaryExpr::KeepPreAsap(expr)) => { - Some(query_time::query_time_nodes(original, expr)?) - } - (Some(original), SummaryExpr::SummaryAgg { child, .. }) - if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && !matches!( - crate::physical::compiler::raw_materialization_input_contract(node), - Ok((_, Some(_), _)) - ) => - { - Some(query_time::selected_query_time_nodes(original, node)?) - } - _ => None, - }; - if let Some((root, nodes)) = query_time { - let id = self.graft(id, root, nodes)?; - if let Some(lowered) = &mut self.lowered { - lowered(node, id); - } - return Ok(id); - } let physical = match &node.expr { SummaryExpr::RelationalJoin { .. } if self.preserve_relational => { @@ -441,205 +519,12 @@ where { return self.lower_relation(node); } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Exact( - planner_types::post_asap::ExactOperation::Aggregate { - having: None, .. - }, - ), - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - let input = self.lower(child)?; - compile_native_fragment(node, &[input])? - } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Limit { .. } - | planner_types::post_asap::ValueOperation::Sort { .. } - | planner_types::post_asap::ValueOperation::Filter { .. }, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - let input = self.lower(child)?; - compile_native_fragment(node, &[input])? - } SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { reason: "unsupported post-ASAP value operation".into(), }, - SummaryExpr::RelationalJoin { - right: candidates, - left: values, - kind: planner_types::pre_asap::JoinKind::Semi, - pruning, - .. - } => { - let candidate_input = self.lower(candidates)?; - let value_input = if let Some(original) = - self.logical_source.as_ref().filter(|_| pruning.is_some()) - { - let exact_expression = - query_time::selected_native_expression(original, values)?; - let value_id = QueryNodeId(self.next_id); - self.next_id += 1; - self.nodes.insert( - value_id, - QueryPlanNode::ExternalExact { - request: ExternalExactRequest { - language: QueryLanguage::PromQl, - expression: exact_expression.to_string(), - output: ExternalExactOutput::InstantVector, - parameters: BTreeMap::new(), - start_parameter: None, - end_parameter: None, - // The external source provides values; the Planner DAG owns matching. - input_contracts: vec![], - }, - inputs: vec![], - }, - ); - value_id - } else { - self.lower(values)? - }; - compile_native_fragment(node, &[value_input, candidate_input])? - } SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { reason: "unsupported join in vector adapter".into(), }, - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) && operator.vector_match.is_none() - && !operator.checked_relative_division - && !operator.checked_finite_division - && node.guarantee.as_ref().is_some_and(|g| !g.has_unknown()) - && [lhs, rhs].iter().all(|operand| { - matches!( - operand.expr, - SummaryExpr::SummaryEstimate { - query: planner_types::post_asap::SketchQuery::Quantile { .. }, - .. - } - ) - }) => - { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(kind) = &operator.kind else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: kind.clone(), - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if self.logical_source.is_some() - || operator.checked_relative_division - || operator.checked_finite_division => - { - let operator = query_time::binary_operator(operator)?; - QueryPlanNode::Logical { - operator, - inputs: vec![self.lower(lhs)?, self.lower(rhs)?], - } - } - - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(kind, _), - child, - reduction, - .. - } if self.logical_source.is_some() - && !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => - { - if !matches!( - kind, - planner_types::post_asap::ExactKind::Sum - | planner_types::post_asap::ExactKind::Count - ) { - let operator = query_time::selected_aggregate_operator( - self.logical_source.as_deref().unwrap(), - node, - )?; - let input = self.lower(child)?; - self.nodes.insert( - id, - QueryPlanNode::Logical { - operator, - inputs: vec![input], - }, - ); - return Ok(id); - } - let operation = match kind { - planner_types::post_asap::ExactKind::Sum => query_time::Aggregation::Sum, - planner_types::post_asap::ExactKind::Count => query_time::Aggregation::Count, - _ => { - return Err(QueryPlanError::Invalid( - "unsupported aggregation over selected summary values".into(), - )) - } - }; - let keys = reduction.group_keys().ok_or_else(|| { - QueryPlanError::Invalid( - "per-entity summary reduction requires a temporal operator".into(), - ) - })?; - let labels = keys - .keys() - .iter() - .map(|&column| { - child - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid("unresolved logical grouping column".into()) - }) - }) - .collect::, _>>()?; - QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Aggregate { - operation, - grouping: query_time::Grouping { - labels, - without: keys.is_without(), - }, - }, - inputs: vec![self.lower(child)?], - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if exact_value_executable(node) => { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind - else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: operator.clone(), - } - } - SummaryExpr::KeepPreAsap(expr) if scalar_literal(expr).is_some() => { - QueryPlanNode::Scalar { - value: scalar_literal(expr).unwrap(), - } - } SummaryExpr::KeepPreAsap(expr) if self.preserve_relational => { let mut expression = clickhouse_exact::render(expr).map_err(QueryPlanError::UnsupportedNode)?; @@ -678,20 +563,6 @@ where inputs: Vec::new(), } } - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), - child, - reduction, - .. - } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && exact_value_executable(node) => - { - QueryPlanNode::ReduceSum { - input: self.lower(child)?, - grouping: physical_grouping(reduction, child)?, - } - } SummaryExpr::SummaryAgg { child, family, @@ -743,17 +614,10 @@ where .. } => match family { SummaryFamilyType::ExactAggregate(..) | SummaryFamilyType::Sketch(..) => { - let mut binding = match (self.bind)(node, family) { - Ok(binding) => binding, - Err(error) => { - if let Some(original) = &self.logical_source { - let (root, nodes) = - query_time::selected_query_time_nodes(original, node)?; - return self.graft(id, root, nodes); - } - return Err(error); - } - }; + // A selected state without a deployed binding leaves the + // query to the exact engine. + let mut binding = (self.bind)(node, family) + .map_err(|error| QueryPlanError::UnsupportedNode(error.to_string()))?; binding.output_grouping = physical_grouping(reduction, child)?; if let Some(readout) = exact_readout(family) { let existing = self.nodes.iter().find_map(|(id, node)| { @@ -814,6 +678,64 @@ where } } +/// Does PromQL keep `__name__` on this query's result series? Only a series +/// selector keeps it, through ordering, selection, filtering, relabeling, +/// subqueries, `first_`/`last_over_time` and the left side of `and`/`unless`; +/// other functions, aggregations and arithmetic drop it. A comparison keeps it +/// unless it has `bool`, which the IR does not record; Planner compiles no +/// comparison, so the rule treats them as dropping it. +pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bool { + use planner_types::pre_asap::{AggIntent, BinaryOpKind, PromQLVectorSetOpKind, QueryExpr}; + match expr { + QueryExpr::Scan { .. } => true, + QueryExpr::TimeRange { child, .. } + | QueryExpr::TimeShift { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::PromqlSeriesSample { child, .. } + | QueryExpr::PromqlRelabel { child, .. } + | QueryExpr::PromqlSubquery { child, .. } => result_keeps_metric_name(child), + QueryExpr::BinaryOp { + op: BinaryOpKind::Set(PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless), + lhs, + .. + } => result_keeps_metric_name(lhs), + QueryExpr::Aggregate { + child, measures, .. + } if matches!( + measures.as_slice(), + [AggIntent::LastOverTime | AggIntent::FirstOverTime] + ) => + { + result_keeps_metric_name(child) + } + _ => false, + } +} + +/// Parameters of each `kind` operator inside a physical fragment, for tests. +#[cfg(test)] +pub(crate) fn operator_parameters(node: &QueryPlanNode, kind: &str) -> Vec { + let QueryPlanNode::PhysicalFragment { dag, .. } = node else { + return vec![]; + }; + asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag).unwrap(); + let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); + document["nodes"] + .as_object() + .unwrap() + .values() + .filter_map(|node| { + node.get("Operator")? + .get("operator")? + .get("kind")? + .get(kind) + .cloned() + }) + .collect() +} + fn exact_readout(family: &SummaryFamilyType) -> Option { use planner_types::post_asap::ExactKind; match family { @@ -1175,9 +1097,11 @@ mod catalog_binding_tests { mod tests { use super::*; + // Planner's guarded average divides two per-series readouts; until + // Planner matches per-series rows, lowering refuses it instead of + // computing it in the backend. #[test] - fn guarded_division_retains_checks_in_both_query_compilers() { - // Both compilers retain the finite/relative guard supplied by Planner. + fn per_series_guarded_division_is_not_lowered_locally() { let query = "avg_over_time(m[5m])"; let canonical = crate::query_parser::parse_query_expr_canonical( query, @@ -1189,77 +1113,137 @@ mod tests { panic!("expected the Planner's average rewrite"); }; assert!(operator.checked_finite_division); - for relative in [false, true] { - let mut guarded = root.as_ref().clone(); - let SummaryExpr::BinaryOp { operator, .. } = &mut guarded.expr else { - unreachable!(); - }; - operator.checked_finite_division = !relative; - operator.checked_relative_division = relative; - let guarded = Rc::new(guarded); - for composable in [false, true] { - let instant = InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }; - let bind = |_: &Rc, _: &SummaryFamilyType| { - Ok(MaterializationBinding { - full_window_slide_ms: None, - materialization: PolicyFingerprint(7).into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - PolicyFingerprint(7).into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - item_labels: Vec::new(), - }) - }; - let entry = if composable { - compile_bound_composable_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) - } else { - compile_bound_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) + let error = compile_bound_mapped( + "guarded".into(), + query.into(), + &root, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::ExactBackend, + |_: &Rc, _: &SummaryFamilyType| { + Ok(MaterializationBinding { + full_window_slide_ms: None, + materialization: PolicyFingerprint(7).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + PolicyFingerprint(7).into(), + ), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + item_labels: Vec::new(), + }) + }, + |_, _| {}, + ) + .unwrap_err(); + assert!( + matches!(error, QueryPlanError::UnsupportedNode(_)), + "{error}" + ); + } + + // Every o11y corpus query compiles as backend readouts under Planner + // fragments, or forwards whole; no backend value operator remains. + #[test] + fn o11y_corpus_computes_only_through_planner_fragments() { + use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/o11y_queries.json")).unwrap(); + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let template = fixture["query_workload"]["repeating_queries"][0].clone(); + let queries = corpus["queries"] + .as_array() + .unwrap() + .iter() + .map(|row| row["query"].as_str().unwrap()) + .collect::>(); + fixture["query_workload"]["repeating_queries"] = queries + .iter() + .map(|query| { + let mut entry = template.clone(); + entry["query"] = (*query).into(); + entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); + entry + }) + .collect::>() + .into(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request, environment) + .unwrap(); + assert_eq!(plan.query_plan.entries.len(), queries.len()); + let mut fragments = 0; + for entry in plan.query_plan.entries.values() { + for node in entry.nodes.values() { + match node { + QueryPlanNode::PhysicalFragment { .. } => fragments += 1, + QueryPlanNode::ReadMaterialization { .. } + | QueryPlanNode::ExactReadout { .. } + | QueryPlanNode::SummaryEstimate { .. } + | QueryPlanNode::SummaryMerge { .. } => {} + QueryPlanNode::ExactFallback { .. } => assert_eq!(entry.nodes.len(), 1), + other => panic!("{}: unexpected {other:?}", entry.canonical_query), } - .unwrap(); - let QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Binary { operation, .. }, - .. - } = &entry.nodes[&entry.root] - else { - panic!( - "expected guarded division (composable={composable}): {:?}", - entry.nodes - ); - }; - assert_eq!( - *operation, - if relative { - query_time::BinaryOperation::CheckedDiv - } else { - query_time::BinaryOperation::FiniteDiv - } - ); - assert!(!entry.materialization_bindings().is_empty()); } } + assert!(fragments > 0); + } + + // Instant counts have no local exact readout, so a computation over one + // is not compiled locally; count_over_time keeps its readout. + #[test] + fn computation_over_instant_count_is_not_local() { + let selected = |query: &str| { + let canonical = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + crate::planner_selection::plan_test_query(&canonical).unwrap() + }; + let instant = selected("count(m) * 2"); + assert!(is_query_computation(&instant)); + let Err(error) = compile_query_computation(&instant) else { + panic!("instant count computation compiled locally"); + }; + assert!(error.to_string().contains("no decoded readout"), "{error}"); + let temporal = selected("sum(count_over_time(m[5m])) * 2"); + assert!(is_query_computation(&temporal)); + compile_query_computation(&temporal).unwrap(); + } + + // PromQL keeps `__name__` only on selections of a series selector. + #[test] + fn metric_name_follows_promql_result_rules() { + for (query, keeps) in [ + ("m", true), + ("m offset 5m", true), + ("sort(m)", true), + ("topk(2, m)", true), + ("last_over_time(m[5m])", true), + ("m and n", true), + ("rate(m[5m])", false), + ("quantile_over_time(0.5, m[5m])", false), + ("m * 2", false), + ("sum by (job) (m)", false), + ("topk(2, rate(m[5m]))", false), + ("max_over_time(rate(m[1m])[5m:1m])", false), + ] { + let expr = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + assert_eq!(result_keeps_metric_name(&expr), keeps, "{query}"); + } } #[test] diff --git a/control_plane/src/query_plan/physical_values.rs b/control_plane/src/query_plan/physical_values.rs deleted file mode 100644 index ec14b64cd..000000000 --- a/control_plane/src/query_plan/physical_values.rs +++ /dev/null @@ -1,445 +0,0 @@ -//! Retain Planner-compiled scalar/vector fragments before publishing an installed plan. -use super::*; -use asap_physical_operators::physical_planner::{promql_values as physical, CompiledPhysicalDag}; -use asap_types::physical_plan_codec::PhysicalPlanCodec; -use planner_types::{ - post_asap::BinaryOperator, - pre_asap::{AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys}, -}; -use query_time::{Aggregation, BinaryOperation, Grouping, QueryTimeOperator as Operation}; - -fn invalid(error: impl std::fmt::Display) -> QueryPlanError { - QueryPlanError::Invalid(error.to_string()) -} -fn grouping(value: &Grouping) -> GroupKeys { - let keys = value.labels.iter().cloned().map(ColumnRef::Named).collect(); - if value.without { - GroupKeys::without(keys) - } else { - GroupKeys::by(keys) - } -} -fn binary(operation: BinaryOperation) -> BinaryOperator { - use ArithmeticOpKind as A; - use BinaryOperation as O; - use CompareOpKind as C; - let kind = match operation { - O::Add => BinaryOpKind::Arithmetic(A::Add), - O::Sub => BinaryOpKind::Arithmetic(A::Sub), - O::Mul => BinaryOpKind::Arithmetic(A::Mul), - O::Div | O::CheckedDiv | O::FiniteDiv => BinaryOpKind::Arithmetic(A::Div), - O::Mod => BinaryOpKind::Arithmetic(A::Mod), - O::Pow => BinaryOpKind::Arithmetic(A::Pow), - O::Equal => BinaryOpKind::Compare(C::Eq), - O::NotEqual => BinaryOpKind::Compare(C::Ne), - O::Less => BinaryOpKind::Compare(C::Lt), - O::LessEqual => BinaryOpKind::Compare(C::Le), - O::Greater => BinaryOpKind::Compare(C::Gt), - O::GreaterEqual => BinaryOpKind::Compare(C::Ge), - }; - BinaryOperator { - kind, - vector_match: None, - checked_relative_division: operation == O::CheckedDiv, - checked_finite_division: operation == O::FiniteDiv, - } -} - -pub fn compile( - entry: &mut QueryPlanEntry, -) -> Result, QueryPlanError> { - if entry.language == QueryLanguage::ClickHouseSql || entry.physical_dag.is_some() { - return Ok(BTreeMap::new()); - } - let mut scalars = BTreeMap::new(); - for id in entry.topological_order()? { - let node = &entry.nodes[&id]; - let inputs = node.inputs().to_vec(); - let scalar_input = |position: usize| { - inputs - .get(position) - .and_then(|id| scalars.get(id)) - .copied() - .unwrap_or(false) - }; - let mut scalar = false; - let compiled = match node { - QueryPlanNode::Scalar { .. } => { - scalar = true; - let QueryPlanNode::Scalar { value } = node else { - unreachable!() - }; - Some(physical::compile_scalar(*value).map_err(invalid)?) - } - QueryPlanNode::Logical { - operator: - Operation::ExactSubquery { query } | Operation::CandidateExactSubquery { query, .. }, - .. - } => { - scalar = promql_parser::parser::parse(query) - .map_err(invalid)? - .value_type() - == promql_parser::parser::value::ValueType::Scalar; - None - } - QueryPlanNode::Logical { operator, .. } => match operator { - Operation::Binary { - operation, - return_bool, - } => { - scalar = scalar_input(0) && scalar_input(1); - Some( - physical::compile_binary( - &binary(*operation), - *return_bool, - scalar_input(0), - scalar_input(1), - ) - .map_err(invalid)?, - ) - } - Operation::UnaryNegate => { - scalar = scalar_input(0); - Some(physical::compile_negate(scalar).map_err(invalid)?) - } - Operation::VectorToScalar => { - scalar = true; - Some(physical::compile_vector_to_scalar().map_err(invalid)?) - } - Operation::Aggregate { - operation, - grouping: groups, - } => { - let intent = match operation { - Aggregation::Sum => AggIntent::Sum { col: None }, - Aggregation::Avg => AggIntent::Avg { col: None }, - Aggregation::Min => AggIntent::Min { col: None }, - Aggregation::Max => AggIntent::Max { col: None }, - Aggregation::Count => AggIntent::Count { - accuracy: planner_types::types::AccuracyTarget::Exact, - }, - }; - Some(physical::compile_aggregate(&intent, &grouping(groups)).map_err(invalid)?) - } - Operation::Sort { - descending, - grouping: groups, - } => Some(physical::compile_sort(*descending, &grouping(groups)).map_err(invalid)?), - Operation::Limit { - n, - offset, - grouping: groups, - } => { - Some(physical::compile_limit(*n, *offset, &grouping(groups)).map_err(invalid)?) - } - Operation::Temporal { operation } => { - use query_time::TemporalOperation as T; - let intent = match operation { - T::Rate => AggIntent::Rate, - T::Increase => AggIntent::Increase, - T::Sum => AggIntent::Sum { col: None }, - T::Avg => AggIntent::Avg { col: None }, - T::Min => AggIntent::Min { col: None }, - T::Max => AggIntent::Max { col: None }, - T::Count => AggIntent::Count { - accuracy: planner_types::types::AccuracyTarget::Exact, - }, - }; - let preserve = entry.language == QueryLanguage::MetricsQl - && matches!(operation, T::Min | T::Max | T::Avg); - Some(physical::compile_temporal(&intent, preserve).map_err(invalid)?) - } - Operation::HistogramQuantile => { - Some(physical::compile_histogram_quantile().map_err(invalid)?) - } - _ => None, - }, - QueryPlanNode::ReduceSum { - grouping: groups, .. - } => Some(match groups { - PhysicalGrouping::Reduce(labels) => physical::compile_aggregate( - &AggIntent::Sum { col: None }, - &GroupKeys::by(labels.iter().cloned().map(ColumnRef::Named).collect()), - ) - .map_err(invalid)?, - PhysicalGrouping::PerEntity => CompiledPhysicalDag::from_operators( - BTreeMap::from([( - 0, - asap_physical_operators::physical_planner::InputContract::bounded( - physical::vector_schema(), - ), - )]), - BTreeMap::from([( - 1, - ( - vec![0], - asap_physical_operators::operators::Operator::project( - physical::vector_schema(), - vec![ - ( - "labels".into(), - asap_physical_operators::expressions::Expression::Column(0), - ), - ( - "value".into(), - asap_physical_operators::expressions::Expression::Column(1), - ), - ], - ) - .map_err(invalid)?, - ), - )]), - vec![1], - ) - .map_err(invalid)?, - }), - QueryPlanNode::Binary { operator, .. } => { - scalar = scalar_input(0) && scalar_input(1); - Some( - physical::compile_binary( - &BinaryOperator { - kind: BinaryOpKind::Arithmetic(operator.clone()), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - false, - scalar_input(0), - scalar_input(1), - ) - .map_err(invalid)?, - ) - } - QueryPlanNode::PhysicalFragment { dag, .. } => { - let compiled = CompiledPhysicalDag::decode(dag).map_err(invalid)?; - scalar = compiled - .output_contract(compiled.roots()[0]) - .map_err(invalid)? - .schema - == physical::scalar_schema(); - None - } - _ => None, - }; - scalars.insert(id, scalar); - if let Some(compiled) = compiled { - entry.nodes.insert( - id, - QueryPlanNode::PhysicalFragment { - inputs, - dag: compiled.encode().map_err(invalid)?, - row_input: None, - pruning: None, - }, - ); - } - } - combine(entry) -} - -fn combine( - entry: &mut QueryPlanEntry, -) -> Result, QueryPlanError> { - use asap_physical_operators::physical_planner::InputContract; - use std::collections::BTreeSet; - let supported = |schema: &asap_physical_operators::values::Schema| { - schema == &physical::scalar_schema() - || schema == &physical::vector_schema() - || schema == &physical::matrix_schema() - }; - let mut programs = BTreeMap::new(); - for (&id, node) in &entry.nodes { - if let QueryPlanNode::PhysicalFragment { - dag, - row_input: None, - pruning: None, - .. - } = node - { - let graph = CompiledPhysicalDag::decode(dag).map_err(invalid)?; - if graph.roots().len() == 1 - && graph - .input_contracts() - .all(|(_, input)| supported(&input.schema)) - && supported( - &graph - .output_contract(graph.roots()[0]) - .map_err(invalid)? - .schema, - ) - { - programs.insert(id, graph); - } - } - } - let mut roots = BTreeSet::new(); - if programs.contains_key(&entry.root) { - roots.insert(entry.root); - } - for (id, node) in &entry.nodes { - if !programs.contains_key(id) { - roots.extend( - node.inputs() - .iter() - .filter(|id| programs.contains_key(id)) - .copied(), - ); - } - } - let reachable = |root: QueryNodeId, boundaries: &BTreeSet| { - let mut pending = vec![root]; - let mut seen = BTreeSet::new(); - while let Some(id) = pending.pop() { - if !programs.contains_key(&id) - || (id != root && boundaries.contains(&id)) - || !seen.insert(id) - { - continue; - } - pending.extend(entry.nodes[&id].inputs()); - } - seen - }; - // A producer consumed across an I/O boundary remains one separately scheduled - // physical output, rather than being duplicated into both downstream graphs. - let mut owners = BTreeMap::::new(); - for &root in &roots { - for id in reachable(root, &BTreeSet::new()) { - *owners.entry(id).or_default() += 1; - } - } - roots.extend( - owners - .into_iter() - .filter_map(|(id, count)| (count > 1).then_some(id)), - ); - let mut replacements = BTreeMap::new(); - let mut remap = BTreeMap::new(); - for &root in &roots { - let members = reachable(root, &roots); - let mut sources = BTreeMap::::new(); - let mut fragments = BTreeMap::new(); - for &id in &members { - let graph = &programs[&id]; - let inputs = entry.nodes[&id].inputs(); - for ((_, contract), input) in graph.input_contracts().zip(inputs) { - if !members.contains(input) - && sources - .insert(input.0, contract.clone()) - .is_some_and(|previous| previous.schema != contract.schema) - { - return Err(invalid("shared physical input has inconsistent schemas")); - } - } - fragments.insert( - id.0, - (inputs.iter().map(|id| id.0).collect(), graph.clone()), - ); - if id != root { - remap.insert(id, root); - } - } - let graph = - CompiledPhysicalDag::compose(sources, fragments, vec![root.0]).map_err(invalid)?; - replacements.insert( - root, - QueryPlanNode::PhysicalFragment { - inputs: graph - .input_contracts() - .map(|(id, _)| QueryNodeId(id)) - .collect(), - dag: graph.encode().map_err(invalid)?, - row_input: None, - pruning: None, - }, - ); - } - for id in remap.keys() { - entry.nodes.remove(id); - } - entry.nodes.extend(replacements); - Ok(remap) -} - -#[cfg(test)] -pub(crate) fn operator_parameters(node: &QueryPlanNode, kind: &str) -> Vec { - let QueryPlanNode::PhysicalFragment { dag, .. } = node else { - return vec![]; - }; - CompiledPhysicalDag::decode(dag).unwrap(); - let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); - document["nodes"] - .as_object() - .unwrap() - .values() - .filter_map(|node| { - node.get("Operator")? - .get("operator")? - .get("kind")? - .get(kind) - .cloned() - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - // A sum over finalized summaries must be priced and installed as computation, - // never reconstructed by the query worker. - #[test] - fn finalized_summary_rollup_is_retained_as_a_physical_graph() { - for grouping in [ - PhysicalGrouping::PerEntity, - PhysicalGrouping::Reduce(vec!["service".into()]), - ] { - let mut entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "rollup".into(), - canonical_query: "sum by (service) (foo)".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([ - ( - QueryNodeId(0), - QueryPlanNode::Logical { - operator: Operation::ExactSubquery { - query: "foo".into(), - }, - inputs: vec![], - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::ReduceSum { - input: QueryNodeId(0), - grouping, - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 1000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::Reject, - }; - compile(&mut entry).unwrap(); - let QueryPlanNode::PhysicalFragment { dag, inputs, .. } = &entry.nodes[&entry.root] - else { - panic!("rollup was left for Backend execution"); - }; - let compiled = CompiledPhysicalDag::decode(dag).unwrap(); - assert_eq!(inputs, &[QueryNodeId(0)]); - assert_eq!( - compiled - .output_contract(compiled.roots()[0]) - .unwrap() - .schema, - physical::vector_schema() - ); - } - } -} diff --git a/control_plane/src/query_plan/query_time.rs b/control_plane/src/query_plan/query_time.rs index 1d755d5d1..a035ff428 100644 --- a/control_plane/src/query_plan/query_time.rs +++ b/control_plane/src/query_plan/query_time.rs @@ -1,1039 +1,14 @@ -//! The query-time half of a plan: everything a summary did not replace. -//! -//! Typed operations compiled once by the control plane, never parsed at -//! serving time. This is the counterpart to the precompute half, which runs -//! ahead of the query and ends at stored outputs. -use super::{ - FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanEntry, QueryPlanError, QueryPlanNode, -}; -#[cfg(test)] -use asap_types::physical_plan_codec::PhysicalPlanCodec; -use promql_parser::{ - label::MatchOp, - parser::{self, Expr, LabelModifier, Offset, VectorSelector}, -}; -use std::collections::BTreeMap; - +//! Installed query-time leaves: raw selectors, exact subtrees and +//! current-series readouts. Computation over their values is Planner-compiled. pub use asap_types::query_plan::query_time::*; -/// Stable identity of a Planner-authorized materializable DAG leaf. This is a -/// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, serde::Serialize, PartialEq)] -struct MaterializationCandidateIdentity { - metric: String, - matchers: Vec, - range_ms: u64, - offset_ms: i64, - operation: TemporalOperation, -} - -fn invalid(message: impl Into) -> QueryPlanError { - QueryPlanError::Invalid(message.into()) -} -fn millis(duration: std::time::Duration) -> Result { - u64::try_from(duration.as_millis()).map_err(|_| invalid("logical duration overflow")) -} -fn offset(value: &Option) -> Result { - match value { - None => Ok(0), - Some(Offset::Pos(d)) => i64::try_from(millis(*d)?).map_err(|_| invalid("offset overflow")), - Some(Offset::Neg(d)) => i64::try_from(millis(*d)?) - .map(|v| -v) - .map_err(|_| invalid("offset overflow")), - } -} - -struct Lower { - nodes: BTreeMap, - seen: BTreeMap, -} -impl Lower { - fn add(&mut self, node: QueryPlanNode) -> Result { - let key = serde_json::to_string(&node).map_err(|e| invalid(e.to_string()))?; - if let Some(id) = self.seen.get(&key) { - return Ok(*id); - } - let id = QueryNodeId(self.nodes.len() as u64); - self.nodes.insert(id, node); - self.seen.insert(key, id); - Ok(id) - } - fn operation( - &mut self, - operator: QueryTimeOperator, - inputs: Vec, - ) -> Result { - operator.validate(inputs.len())?; - self.add(QueryPlanNode::Logical { operator, inputs }) - } - fn scan( - &mut self, - s: &VectorSelector, - range_ms: Option, - ) -> Result { - if s.at.is_some() || !s.matchers.or_matchers.is_empty() { - return Err(invalid("logical @/OR selector is not supported")); - } - let matchers = s - .matchers - .matchers - .iter() - .map(|m| LabelMatcher { - name: m.name.clone(), - value: m.value.clone(), - operation: match m.op { - MatchOp::Equal => LabelMatch::Equal, - MatchOp::NotEqual => LabelMatch::NotEqual, - MatchOp::Re(_) => LabelMatch::Regex, - MatchOp::NotRe(_) => LabelMatch::NotRegex, - }, - }) - .collect(); - self.operation( - QueryTimeOperator::Scan { - metric: s.name.clone(), - matchers, - range_ms, - offset_ms: offset(&s.offset)?, - }, - vec![], - ) - } - fn lower(&mut self, expr: &Expr) -> Result { - match expr { - Expr::NumberLiteral(n) if n.val.is_finite() => { - self.add(QueryPlanNode::Scalar { value: n.val }) - } - Expr::Paren(p) => self.lower(&p.expr), - Expr::Unary(u) => { - let input = self.lower(&u.expr)?; - self.operation(QueryTimeOperator::UnaryNegate, vec![input]) - } - Expr::VectorSelector(s) => self.scan(s, None), - Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), - Expr::Subquery(s) => { - if s.at.is_some() { - return Err(invalid("logical subquery @ is unsupported")); - } - let input = self.lower(&s.expr)?; - self.operation( - QueryTimeOperator::Subquery { - range_ms: millis(s.range)?, - // Prometheus uses its configured default evaluation - // interval when `[range:]` omits the resolution. The - // backend-local deployment uses the Prometheus default - // of one minute; unsupported subquery operands are - // externalized as one exact subtree before execution. - step_ms: millis( - s.step.unwrap_or_else(|| std::time::Duration::from_secs(60)), - )?, - offset_ms: offset(&s.offset)?, - }, - vec![input], - ) - } - Expr::Aggregate(a) => { - let grouping = match &a.modifier { - None => Grouping { - labels: vec![], - without: false, - }, - Some(LabelModifier::Include(labels)) => Grouping { - labels: labels.labels.clone(), - without: false, - }, - Some(LabelModifier::Exclude(labels)) => Grouping { - labels: labels.labels.clone(), - without: true, - }, - }; - if a.op.to_string() == "topk" { - let Some(Expr::NumberLiteral(parameter)) = a.param.as_deref() else { - return Err(invalid("topk requires a literal scalar parameter")); - }; - if !parameter.val.is_finite() { - return Err(invalid("topk requires a finite scalar parameter")); - } - // Prometheus converts the scalar parameter to int64 before - // selection. Values below one produce an empty vector. - let k = parameter.val as i64; - // Keep the selection node local even when its operand has - // unsupported syntax (for example a subquery with an - // implicit resolution). Prometheus evaluates that maximal - // instant-vector child; the backend still performs topk. - let nodes_before = self.nodes.clone(); - let seen_before = self.seen.clone(); - let input = match self.lower(&a.expr) { - Ok(input) => input, - Err(_) => { - self.nodes = nodes_before; - self.seen = seen_before; - self.operation( - QueryTimeOperator::ExactSubquery { - query: a.expr.to_string(), - }, - vec![], - )? - } - }; - let sorted = self.operation( - QueryTimeOperator::Sort { - descending: true, - grouping: grouping.clone(), - }, - vec![input], - )?; - return self.operation( - QueryTimeOperator::Limit { - n: u64::try_from(k).unwrap_or(0), - offset: 0, - grouping, - }, - vec![sorted], - ); - } - if a.param.is_some() { - return Err(invalid("unsupported parameterized aggregate")); - } - let operation = match a.op.to_string().as_str() { - "sum" => Aggregation::Sum, - "max" => Aggregation::Max, - "min" => Aggregation::Min, - "avg" => Aggregation::Avg, - "count" => Aggregation::Count, - other => return Err(invalid(format!("unsupported logical aggregate {other}"))), - }; - let input = self.lower(&a.expr)?; - self.operation( - QueryTimeOperator::Aggregate { - operation, - grouping, - }, - vec![input], - ) - } - Expr::Call(c) => { - let operator = match c.func.name { - "scalar" => QueryTimeOperator::VectorToScalar, - "histogram_quantile" => QueryTimeOperator::HistogramQuantile, - "sort" => QueryTimeOperator::Sort { - descending: false, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - "sort_desc" => QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - name => QueryTimeOperator::Temporal { - operation: match name { - "rate" => TemporalOperation::Rate, - "increase" => TemporalOperation::Increase, - "avg_over_time" => TemporalOperation::Avg, - "max_over_time" => TemporalOperation::Max, - "min_over_time" => TemporalOperation::Min, - "sum_over_time" => TemporalOperation::Sum, - "count_over_time" => TemporalOperation::Count, - _ => { - return Err(invalid(format!("unsupported logical function {name}"))) - } - }, - }, - }; - let inputs = c - .args - .args - .iter() - .map(|e| self.lower(e)) - .collect::, _>>()?; - self.operation(operator, inputs) - } - Expr::Binary(b) => { - if b.modifier.as_ref().is_some_and(|m| { - m.matching.is_some() - || !matches!(m.card, parser::VectorMatchCardinality::OneToOne) - }) { - return Err(invalid("logical explicit vector matching unsupported")); - } - let operation = match b.op.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "==" => BinaryOperation::Equal, - "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => return Err(invalid(format!("unsupported logical binary {other}"))), - }; - let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; - self.operation( - QueryTimeOperator::Binary { - operation, - return_bool: b.return_bool(), - }, - inputs, - ) - } - _ => Err(invalid("unsupported logical expression")), - } - } -} - -/// Lower a Planner-authorized native fragment into typed backend operations. -/// Callers retain a separate external-native alternative for cost comparison. -pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, -) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = QueryPlanEntry { - physical_dag: None, - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) -} - -fn horizons(expr: &planner_types::pre_asap::QueryExpr, out: &mut Vec) { - use planner_types::pre_asap::QueryExpr; - if let QueryExpr::TimeRange { range, .. } = expr { - if let Ok(ms) = u64::try_from(range.as_millis()) { - out.push(ms); - } - } - match expr { - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => horizons(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - horizons(lhs, out); - horizons(rhs, out); - } - QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { - horizons(left, out); - horizons(right, out); - } - _ => {} - } -} - -/// Match fragments by semantic IR equality, not display text or source names. -/// This ensures a subtree parsed for physical lowering is the subtree Planner kept. -/// Does this re-parsed subtree denote the same computation as the Planner -/// fragment? -/// -/// Not `==`, and deliberately so. Planner documents a PromQL leaf's `schema` as -/// usage-derived: "the `(ts, value)` floor + the labels the query references", -/// and marks it `closed: false` precisely because it does not enumerate the -/// row. A fragment resolved inside the whole query therefore carries every -/// label the *query* mentions, while the same fragment re-parsed on its own -/// carries only the labels *it* mentions. -/// -/// So `sum by (label_0) (rate(data[1m]))` yields a fragment whose leaf scan has -/// columns `[ts, value, label_0]`, while re-parsing the subtree `rate(data[1m])` -/// yields `[ts, value]`. Identical source, predicates, range, measures and -/// reduction; one extra column that the isolated parse had no way to know -/// about. Requiring equality there asks an isolated parse to reproduce -/// whole-query context, and every other part of the comparison is what actually -/// discriminates: a different matcher, range or metric still fails. -/// -/// Open leaf schemas are therefore compared by containment. Containment is a -/// *prefix*, not an arbitrary subset, because `ColumnId`s are positional: the -/// floor comes first and context only appends, so a prefix keeps every column -/// id in `predicates` and grouping keys meaning the same column on both sides. -/// Closed (catalog-backed SQL) schemas do enumerate the row, so they keep exact -/// equality. -fn fragment_matches( - candidate: &planner_types::pre_asap::QueryExpr, - fragment: &planner_types::pre_asap::QueryExpr, -) -> bool { - match ( - serde_json::to_value(candidate), - serde_json::to_value(fragment), - ) { - (Ok(candidate), Ok(fragment)) => same_modulo_open_leaf_schema(&candidate, &fragment), - // Fall back to the strict comparison rather than accepting anything we - // could not inspect. - _ => candidate == fragment, - } -} - -fn same_modulo_open_leaf_schema( - candidate: &serde_json::Value, - fragment: &serde_json::Value, -) -> bool { - use serde_json::Value; - match (candidate, fragment) { - (Value::Object(candidate), Value::Object(fragment)) => { - if is_open_schema(candidate) && is_open_schema(fragment) { - return open_schema_is_widened(candidate, fragment); - } - candidate.len() == fragment.len() - && candidate.iter().all(|(key, value)| { - fragment - .get(key) - .is_some_and(|other| same_modulo_open_leaf_schema(value, other)) - }) - } - (Value::Array(candidate), Value::Array(fragment)) => { - candidate.len() == fragment.len() - && candidate - .iter() - .zip(fragment) - .all(|(a, b)| same_modulo_open_leaf_schema(a, b)) - } - _ => candidate == fragment, - } -} - -fn is_open_schema(value: &serde_json::Map) -> bool { - value.get("closed") == Some(&serde_json::Value::Bool(false)) && value.contains_key("columns") -} - -/// The isolated parse's columns must be a prefix of the whole-query ones: -/// context appends the labels it references, it never removes or reorders the -/// floor. Every other schema field still has to agree exactly. -fn open_schema_is_widened( - candidate: &serde_json::Map, - fragment: &serde_json::Map, -) -> bool { - let (Some(narrow), Some(wide)) = ( - candidate.get("columns").and_then(|v| v.as_array()), - fragment.get("columns").and_then(|v| v.as_array()), - ) else { - return false; - }; - candidate - .iter() - .filter(|(key, _)| key.as_str() != "columns") - .all(|(key, value)| fragment.get(key) == Some(value)) - && narrow.len() <= wide.len() - && narrow.iter().zip(wide).all(|(a, b)| a == b) -} - -pub(super) fn query_time_nodes( - original: &str, - fragment: &planner_types::pre_asap::QueryExpr, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - fn visit<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { - out.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, out), - Expr::Unary(e) => visit(&e.expr, out), - Expr::Subquery(e) => visit(&e.expr, out), - Expr::Aggregate(e) => visit(&e.expr, out), - Expr::Binary(e) => { - visit(&e.lhs, out); - visit(&e.rhs, out); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, out); - } - } - _ => {} - } - } - let original = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&original, &mut expressions); - // Reconstruct equality witnesses with the selected IR's source horizon, - // not the compatibility parser's default. Explicit matrix ranges remain - // query-owned and equality still checks the complete tree. - let mut intervals = vec![1_000]; - horizons(fragment, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - // Accuracy annotations select a candidate, but exact execution still - // implements that candidate's computation. Reconstruct the same typed IR - // before comparing it; do not erase operators or source predicates. - let accuracy = match fragment { - planner_types::pre_asap::QueryExpr::Aggregate { measures, .. } => measures - .iter() - .find_map(|intent| { - use planner_types::pre_asap::AggIntent; - match intent { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } - | AggIntent::FrequencyL2 { accuracy, .. } - | AggIntent::FrequencyEntropy { accuracy, .. } => Some(accuracy.clone()), - _ => None, - } - }) - .unwrap_or(planner_types::types::AccuracyTarget::Exact), - _ => planner_types::types::AccuracyTarget::Exact, - }; - for expression in expressions { - for interval in &intervals { - if let Ok(candidate) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - accuracy.clone(), - *interval, - ) { - if fragment_matches(&candidate, fragment) { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(expression)?; - return Ok((root, lower.nodes)); - } - } - } - } - Err(invalid( - "Planner logical fragment does not match any original query subtree", - )) -} - -pub(super) fn binary_operator( - operator: &planner_types::post_asap::BinaryOperator, -) -> Result { - if operator.checked_relative_division || operator.checked_finite_division { - if (operator.checked_relative_division && operator.checked_finite_division) - || operator.vector_match.is_some() - || !matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Div - ) - ) - { - return Err(invalid("invalid Planner checked division contract")); - } - return Ok(QueryTimeOperator::Binary { - operation: if operator.checked_finite_division { - BinaryOperation::FiniteDiv - } else { - BinaryOperation::CheckedDiv - }, - return_bool: false, - }); - } - if operator.vector_match.is_some() { - return Err(invalid("explicit logical vector matching unsupported")); - } - let operation = match operator.kind.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "=" | "==" => BinaryOperation::Equal, - "<>" | "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => { - return Err(invalid(format!( - "unsupported Planner binary operator {other}" - ))) - } - }; - Ok(QueryTimeOperator::Binary { - operation, - return_bool: false, - }) -} - -/// Prove a physical-native substitute represents exactly the selected summary leaf. -/// A second Planner invocation is an equality witness, not a replacement selection. -pub(crate) fn selected_query_time_nodes( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - let expression = selected_native_expression(original, selected)?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expression)?; - Ok((root, lower.nodes)) -} - -/// Resolve the selected exact subtree to a verified native expression before -/// binding an external input. Never substitute the top-level query's child. -pub(super) fn selected_native_expression( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { - return Err(invalid( - "native subtree substitution requires an exact selected value", - )); - } - let selected = match &selected.expr { - planner_types::post_asap::SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.as_ref(), - _ => selected, - }; - fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { - output.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, output), - Expr::Unary(e) => visit(&e.expr, output), - Expr::Subquery(e) => visit(&e.expr, output), - Expr::Aggregate(e) => visit(&e.expr, output), - Expr::Binary(e) => { - visit(&e.lhs, output); - visit(&e.rhs, output); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, output); - } - } - _ => {} - } - } - let parsed = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&parsed, &mut expressions); - fn selected_horizons(node: &planner_types::post_asap::SummaryNode, out: &mut Vec) { - use planner_types::post_asap::SummaryExpr; - match &node.expr { - SummaryExpr::KeepPreAsap(expr) => horizons(expr, out), - SummaryExpr::ValueOperation { child, .. } | SummaryExpr::SummaryAgg { child, .. } => { - selected_horizons(child, out) - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - selected_horizons(summary_input, out) - } - SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::SummaryJoin { - outer: left, - inner: right, - .. - } - | SummaryExpr::SummarySubtract { left, right } => { - selected_horizons(left, out); - selected_horizons(right, out); - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - selected_horizons(child, out); - } - } - } - } - let mut intervals = vec![1_000]; - selected_horizons(selected, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - let mut matched = None; - for expression in expressions { - for interval in &intervals { - let Ok(canonical) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - planner_types::types::AccuracyTarget::Exact, - *interval, - ) else { - continue; - }; - // Match provenance against all exact candidates. Do not make a - // second selection or assume the first enumerated candidate won. - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - if candidates.iter().any(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if node.as_ref() == selected)) { - let candidate = expression.clone(); - if matched - .as_ref() - .is_some_and(|previous| previous != &candidate) - { - return Err(invalid( - "ambiguous original subtrees share a Planner summary representation", - )); - } - matched = Some(candidate); - } - } - } - matched.ok_or_else(|| { - invalid("selected summary leaf has no semantically identical original subtree witness") - }) -} - -/// Read the original aggregate operation only after proving its selected-node identity. -/// Min and max share a Planner accumulator family, so the family name alone is insufficient. -pub(super) fn selected_aggregate_operator( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - let (root, nodes) = selected_query_time_nodes(original, selected)?; - match nodes.get(&root) { - Some(QueryPlanNode::Logical { - operator: operator @ QueryTimeOperator::Aggregate { .. }, - .. - }) => Ok(operator.clone()), - _ => Err(invalid( - "selected value aggregation has no verified original aggregate operator", - )), - } -} - -#[cfg(test)] -mod hybrid_tests { - use super::*; - use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; - #[test] - fn external_binding_rejects_an_unrelated_selected_exact_subtree() { - let exact = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(other_metric[5m])", - planner_types::types::AccuracyTarget::Exact, - 1_000, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&exact).unwrap(); - assert!(selected_native_expression("topk(2, sum_over_time(m[5m]))", &selected).is_err()); - assert_eq!( - selected_native_expression("sum_over_time(other_metric[5m])", &selected) - .unwrap() - .to_string(), - "sum_over_time(other_metric[5m])" - ); - } - - #[test] - fn selected_summary_and_filtered_residual_share_installed_binary() { - // Both filtered and unfiltered leaves bind independently. - let query = "sum_over_time(m[5m]) + sum_over_time(m{job=\"api\"}[5m])"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let entry = - crate::query_plan::compile_bound_composable_mapped( - "hybrid".into(), - query.into(), - &selected, - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |node, _| { - let (_, _, spatial_filter) = - crate::physical::compiler::raw_materialization_input_contract(node) - .map_err(QueryPlanError::Invalid)?; - Ok(MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint( - if spatial_filter.is_empty() { 7 } else { 8 }, - ) - .into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - asap_types::PolicyFingerprint(if spatial_filter.is_empty() { - 7 - } else { - 8 - }) - .into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - }) - }, - |_, _| {}, - ) - .unwrap(); - assert_eq!(entry.materialization_bindings().len(), 2); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { .. }, - .. - } - )); - entry - .validate( - &[ - asap_types::PolicyFingerprint(7), - asap_types::PolicyFingerprint(8), - ] - .into_iter() - .collect(), - ) - .unwrap(); - } - - #[test] - fn different_filter_cannot_witness_selected_residual() { - // Equality includes filter predicates, not just family, source, or window. - let canonical = crate::query_parser::parse_query_expr_canonical( - "sum_over_time(m{job=\"api\"}[5m])", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - assert!( - selected_query_time_nodes("sum_over_time(m{job=\"worker\"}[5m])", &selected).is_err() - ); - } -} - -#[cfg(test)] -mod planner_workload_tests { - use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; - - fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let mut entry = fixture["query_workload"]["repeating_queries"][0].clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - fixture["query_workload"]["repeating_queries"] = vec![entry].into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot - .into_physical_compilation_request() - .unwrap_or_else(|error| panic!("{query}: {error}")); - DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap_or_else(|error| panic!("{query}: {error}")) - } - - fn assert_local_limit(node: &QueryPlanNode) { - match node { - QueryPlanNode::PhysicalFragment { dag, .. } => { - let plan = - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .unwrap(); - assert!( - plan.operator_name(plan.roots()[0]) == Some("Limit") - || super::super::physical_values::operator_parameters(node, "Limit").len() - == 1 - ); - } - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } => {} - _ => panic!("expected local Limit, got {node:?}"), - } - } - - #[test] - fn evaluation_topk_queries_retain_a_local_selection_root() { - for query in [ - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), - "{query}" - ); - } - } - - #[test] - fn planner_value_topk_preserves_direct_summary_children() { - for query in [ - "topk(2, rate(backend_process_cpu_seconds_total[1h]))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry.materialization_bindings().is_empty(), - "{query} must retain its SummaryStore child: {:?}", - entry.nodes - ); - } - } - - #[test] - fn whole_o11y_planner_candidate_lowers_every_original_query() { - // AST support alone is insufficient: the actual selected Planner forest must bind too. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let template = fixture["query_workload"]["repeating_queries"][0].clone(); - let mut seen = std::collections::BTreeSet::new(); - let mut entries = Vec::new(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - if !seen.insert(query.to_string()) { - continue; - } - let mut entry = template.clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - entries.push(entry); - } - fixture["query_workload"]["repeating_queries"] = entries.into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.allow_mixed_summary_and_exact_execution); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert_eq!(plan.query_plan.entries.len(), 24); - assert!(plan.query_plan.entries.values().all(|entry| !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })))); - } - - #[test] - fn max_of_selected_values_uses_original_max_operator() { - // MinMax storage type does not authorize choosing min or replacing the selected operand graph. - let query = "max(sum_over_time(m[5m]) / 2)"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - let [candidate] = candidates.as_slice() else { - panic!("expected one exact aggregate candidate") - }; - let Replacement::Summary(selected) = &candidate.replacement else { - panic!("expected exact summary fixture") - }; - let operator = selected_aggregate_operator(query, selected).unwrap(); - assert!(matches!( - operator, - QueryTimeOperator::Aggregate { - operation: Aggregation::Max, - .. - } - )); - } - - #[test] - fn ambiguous_extremum_witness_is_rejected() { - // Different readouts over the same MinMax state cannot be resolved by taking the first AST match. - let canonical = crate::query_parser::parse_query_expr_canonical( - "min(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let maximum = crate::query_parser::parse_query_expr_canonical( - "max(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let maximum = crate::planner_selection::plan_test_query(&maximum).unwrap(); - let result = selected_query_time_nodes("min(m) + max(m)", &selected); - if selected == maximum { - assert!(result.is_err()); - } else { - let (root, nodes) = result.unwrap(); - assert!(matches!( - nodes[&root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Aggregate { - operation: Aggregation::Min, - .. - }, - .. - } - )); - } - } -} +use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType, SummaryNode}; +use planner_types::pre_asap::{QueryExpr, Reduction, Source}; -/// Preserve the exact original operator direction because MinMax family alone -/// does not distinguish min from max. The full Planner-node witness is required. -pub(crate) fn selected_range_max_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; +/// A per-series `max_over_time` over one plain range selector, the only +/// maximum state the backend maintains. Planner distinguishes Max from Min +/// in the family, so the selected node alone identifies it. +pub(crate) fn is_range_max_materialization(node: &SummaryNode) -> bool { let node = match &node.expr { SummaryExpr::ValueOperation { child, @@ -1042,733 +17,55 @@ pub(crate) fn selected_range_max_materialization( } => child.as_ref(), _ => node, }; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Max, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Temporal { - operation: TemporalOperation::Max, - }, - inputs, - }) = nodes.get(&root) - else { - return Ok(None); - }; - if inputs.len() != 1 || nodes.len() != 2 { - return Ok(None); - } - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms: 0, - }, + let SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(ExactKind::Max, _), + reduction: Reduction::PerEntity, + child, .. - }) = nodes.get(&inputs[0]) + } = &node.expr else { - return Ok(None); + return false; }; - Ok(Some(materialization_candidate_key( - MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: 0, - operation: TemporalOperation::Max, - }, - )?)) -} - -#[cfg(test)] -mod range_max_materialization_tests { - use super::*; - #[test] - fn real_gauge_queries_have_planner_authorized_exact_materializations() { - for (query, metric, range_ms) in [ - ( - r#"max_over_time(service_cache_refresh_lag_seconds{job="user-service"}[12h])"#, - "service_cache_refresh_lag_seconds", - 43_200_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job=~".+"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job="order-service"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, + matches!( + &child.expr, + SummaryExpr::KeepPreAsap(expr) if matches!( + expr.as_ref(), + QueryExpr::TimeRange { child, .. } if matches!( + child.as_ref(), + QueryExpr::Scan { source: Source::TimeSeries { metric }, .. } if !metric.is_empty() ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - let key = selected_range_max_materialization(query, &selected) - .unwrap() - .unwrap(); - assert!(key.contains(metric)); - assert!(key.contains(&range_ms.to_string())); - } - } - #[test] - fn min_and_shifted_or_nested_windows_do_not_become_max_materializations() { - for query in [ - "min_over_time(m[1m])", - "max_over_time(m[1m] offset 1m)", - "max_over_time((m + m)[1m:1s])", - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - assert!( - selected_range_max_materialization(query, &selected) - .unwrap() - .is_none(), - "{query}" - ); - } - } -} - -/// Stable contract identity used by priced physical candidates, independent of node IDs. -fn materialization_candidate_key( - candidate: MaterializationCandidateIdentity, -) -> Result { - let mut value = serde_json::to_value(candidate).map_err(|e| invalid(e.to_string()))?; - if let Some(object) = value.as_object_mut() { - // Retention is a consumer lifetime requirement, not the materialization read's - // semantics. Equivalent matcher conjunctions must share policy keys. - object.remove("retention_ms"); - if let Some(matchers) = object.get_mut("matchers").and_then(|v| v.as_array_mut()) { - matchers.sort_by_cached_key(|m| m.to_string()); - } - } - serde_json::to_string(&value).map_err(|e| invalid(e.to_string())) -} - -fn counter_contract( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Option { - let QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { operation }, - inputs, - } = nodes.get(&root)? - else { - return None; - }; - if !matches!( - operation, - TemporalOperation::Rate | TemporalOperation::Increase - ) || inputs.len() != 1 - { - return None; - } - let QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms, - }, - .. - } = nodes.get(&inputs[0])? - else { - return None; - }; - Some(MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: *offset_ms, - operation: *operation, - }) -} - -pub(crate) fn selected_counter_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - counter_contract(root, &nodes) - .map(materialization_candidate_key) - .transpose() -} - -fn prune(entry: &mut QueryPlanEntry) { - let mut seen = std::collections::BTreeSet::new(); - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if seen.insert(id) { - if let Some(node) = entry.nodes.get(&id) { - pending.extend(node.inputs()); - } - } - } - entry.nodes.retain(|id, _| seen.contains(id)); -} - -/// Finish the installed DAG by externalizing every fragment raw subtree. -pub fn finalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - externalize_query_time_nodes(entry)?; - assign_retention(entry) -} - -pub fn eligible_materialization_keys( - original: &str, - selected: &std::rc::Rc, -) -> Result, QueryPlanError> { - use planner_types::post_asap::SummaryExpr; - fn visit( - original: &str, - node: &std::rc::Rc, - keys: &mut std::collections::BTreeSet, - ) -> Result<(), QueryPlanError> { - if let Some(key) = selected_counter_materialization(original, node)? - .or(selected_range_max_materialization(original, node)?) - { - keys.insert(key); - } - match &node.expr { - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - visit(original, lhs, keys)?; - visit(original, rhs, keys)?; - } - SummaryExpr::RelationalJoin { left, right, .. } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - - SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - visit(original, summary_input, keys)? - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - visit(original, child, keys)?; - } - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - visit(original, outer, keys)?; - visit(original, inner, keys)?; - } - SummaryExpr::SummarySubtract { left, right } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - SummaryExpr::KeepPreAsap(_) => {} - } - Ok(()) - } - let mut keys = std::collections::BTreeSet::new(); - visit(original, selected, &mut keys)?; - Ok(keys) -} - -fn expression_shape( - id: QueryNodeId, - nodes: &BTreeMap, -) -> Result { - let node = nodes - .get(&id) - .ok_or_else(|| invalid("missing expression node"))?; - if let QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, - .. - } = node - { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let expr = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - let root = lower.lower(&expr)?; - return expression_shape(root, &lower.nodes); - } - let value = match node { - QueryPlanNode::Logical { operator, .. } => { - serde_json::to_string(operator).map_err(|e| invalid(e.to_string()))? - } - QueryPlanNode::Scalar { value } => format!("scalar:{:x}", value.to_bits()), - _ => return Err(invalid("summary node has no raw expression shape")), - }; - let children = node - .inputs() - .iter() - .map(|child| expression_shape(*child, nodes)) - .collect::, _>>()?; - Ok(format!("{value}({})", children.join(";"))) -} - -/// Collapse only maximal exact fragment subtrees whose full typed expression is -/// witnessed in the original query. Matrix boundaries remain inside Prometheus. -pub fn externalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - fn gather(expr: &Expr, out: &mut Vec) { - if !matches!(expr, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - out.push(expr.clone()); - } - match expr { - Expr::Paren(e) => gather(&e.expr, out), - Expr::Unary(e) => gather(&e.expr, out), - Expr::Subquery(e) => gather(&e.expr, out), - Expr::Aggregate(e) => gather(&e.expr, out), - Expr::Binary(e) => { - gather(&e.lhs, out); - gather(&e.rhs, out); - } - Expr::Call(e) => { - for arg in &e.args.args { - gather(arg, out); - } - } - _ => {} - } - } - let expr = parser::parse(&entry.canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - gather(&expr, &mut expressions); - let mut witnesses = BTreeMap::new(); - for expression in expressions { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - if let Ok(root) = lower.lower(&expression) { - witnesses.insert( - expression_shape(root, &lower.nodes)?, - expression.to_string(), - ); - } - } - fn flags(id: QueryNodeId, nodes: &BTreeMap) -> (bool, bool) { - let Some(node) = nodes.get(&id) else { - return (true, false); - }; - let mut indexed = !matches!( - node, - QueryPlanNode::Logical { .. } | QueryPlanNode::Scalar { .. } - ) || matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } - ); - let mut exact = false; - if let QueryPlanNode::Logical { operator, .. } = node { - exact = matches!( - operator, - QueryTimeOperator::Scan { .. } - | QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } - ); - } - for child in node.inputs() { - let (a, b) = flags(*child, nodes); - indexed |= a; - exact |= b; - } - (indexed, exact) - } - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if matches!( - entry.nodes.get(&id), - Some(QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. }, - .. - }) - ) { - continue; - } - let (indexed, exact) = flags(id, &entry.nodes); - if !indexed && exact { - if let Ok(shape) = expression_shape(id, &entry.nodes) { - if let Some(query) = witnesses.get(&shape) { - entry.nodes.insert( - id, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { - query: query.clone(), - }, - inputs: vec![], - }, - ); - continue; - } - } - } - pending.extend(entry.nodes[&id].inputs()); - } - prune(entry); - if entry.nodes.values().any(|node| { - matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } ) - }) { - return Err(invalid( - "local Scan is not deployable; exact subtree requires a complete Prometheus boundary", - )); - } - Ok(()) -} - -fn assign_retention(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - let mut pending = vec![(entry.root, 0u64)]; - let mut depths = BTreeMap::new(); - while let Some((id, depth)) = pending.pop() { - if depths.get(&id).is_some_and(|prior| *prior >= depth) { - continue; - } - depths.insert(id, depth); - let node = entry - .nodes - .get_mut(&id) - .ok_or_else(|| invalid("missing index ancestor"))?; - let mut child_depth = depth; - if let QueryPlanNode::Logical { operator, .. } = node { - if let QueryTimeOperator::Subquery { - range_ms, - offset_ms, - .. - } = operator - { - child_depth = depth - .checked_add(*range_ms) - .and_then(|v| v.checked_add((*offset_ms).max(0) as u64)) - .ok_or_else(|| invalid("retention overflow"))?; - } - } - pending.extend(node.inputs().iter().map(|child| (*child, child_depth))); - } - Ok(()) -} - -#[cfg(test)] -mod remote_boundary_regressions { - use super::*; - - #[test] - fn stored_output_identity_is_independent_of_matcher_order() { - let first = LabelMatcher { - name: "job".into(), - value: "orders".into(), - operation: LabelMatch::Equal, - }; - let second = LabelMatcher { - name: "status".into(), - value: "5..".into(), - operation: LabelMatch::Regex, - }; - let a = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![first.clone(), second.clone()], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - let b = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![second, first], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - assert_eq!( - materialization_candidate_key(a).unwrap(), - materialization_candidate_key(b).unwrap() - ); - } - - #[test] - fn real_error_ratio_exposes_two_independent_materialization_candidates() { - let query = "sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m]))"; - let parsed = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&parsed).unwrap(); - assert_eq!( - eligible_materialization_keys(query, &selected) - .unwrap() - .len(), - 2 - ); - } + ) } #[cfg(test)] mod tests { use super::*; - // A grouped query's fragment carries the grouping label in its leaf scan - // schema, because Planner resolves that schema against the whole query. - // Re-parsing the subtree alone cannot know the label, so requiring equal - // column sets rejected a fragment that is the subtree. - #[test] - fn grouped_query_residual_matches_its_own_subtree() { - for (query, subtree) in [ - ("sum by (label_0) (rate(data[1m]))", "rate(data[1m])"), - ( - "sum by (label_0) (sum_over_time(data[1m]))", - "sum_over_time(data[1m])", - ), - ] { - let fragment = crate::query_parser::parse_query_expr_with_interval( - subtree, - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!( - query_time_nodes(query, &fragment).is_ok(), - "{query}: fragment {subtree} must resolve against its own query" - ); - } - } - - // Widening is only accepted for the context-derived leaf columns. Anything - // that actually identifies the computation still has to match exactly. - #[test] - fn widened_leaf_schema_does_not_excuse_a_different_computation() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "rate(data[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - // Different metric. - assert!(query_time_nodes("sum by (label_0) (rate(other[1m]))", &fragment).is_err()); - // Different range. - assert!(query_time_nodes("sum by (label_0) (rate(data[2m]))", &fragment).is_err()); - // Different function. - assert!(query_time_nodes("sum by (label_0) (increase(data[1m]))", &fragment).is_err()); - // Different matcher. - let filtered = crate::query_parser::parse_query_expr_with_interval( - "rate(data{job=\"api\"}[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!(query_time_nodes( - "sum by (label_0) (rate(data{job=\"worker\"}[1m]))", - &filtered - ) - .is_err()); - } - // A workload horizon changes the equality witness, never its filter or explicit range. - #[test] - fn workload_horizon_residual_keeps_semantic_equality() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "sum(m{job=\"api\"})", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum(m{job=\"api\"})", &fragment).is_ok()); - assert!(query_time_nodes("sum(m{job=\"worker\"})", &fragment).is_err()); - let range = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(m[1m])", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum_over_time(m[2m])", &range).is_err()); - } - - fn instant() -> InstantExecution { - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - } - } - - #[test] - fn complete_o11y_corpus_lowers_to_serialized_operations() { - // Every original workload occurrence must compile to an executable typed graph. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - let entry = crate::query_plan::query_time::compile_logical( - row["id"].as_str().unwrap().into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let encoded = serde_json::to_string(&entry).unwrap(); - let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); - restored.validate(&Default::default()).unwrap(); - assert!(!restored - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - } - } - #[test] - fn query_time_mapping_preserves_filters_and_rejects_different_sources() { - // Physical lowering must prove correspondence with the Planner-kept semantic subtree. - let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; - let fragment = crate::query_parser::parse_query_expr_canonical( + fn selected(query: &str) -> std::rc::Rc { + let original = crate::query_parser::parse_query_expr_canonical( query, planner_types::types::AccuracyTarget::Exact, ) .unwrap(); - let (_, nodes) = query_time_nodes(query, &fragment).unwrap(); - assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: QueryTimeOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); - assert!(query_time_nodes("sum(rate(other_total[5m]))", &fragment).is_err()); - } - #[test] - fn repeated_subexpressions_share_node_identity() { - // Serialized edges must retain CSE rather than duplicating raw work. - let entry = crate::query_plan::query_time::compile_logical( - "q".into(), - "sum(up) / sum(up)".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("binary expected") - }; - assert_eq!(inputs[0], inputs[1]); - } - #[test] - fn malformed_operator_arity_is_rejected_at_installation() { - // A serialized graph cannot bypass the operation's input contract. - assert!(QueryTimeOperator::HistogramQuantile.validate(1).is_err()); - assert!(QueryTimeOperator::Subquery { - range_ms: 60_000, - step_ms: 0, - offset_ms: 0 - } - .validate(1) - .is_err()); + crate::planner_selection::plan_test_query(&original).unwrap() } + // Plain gauge maxima are maintained; minima, offsets and nested windows are not. #[test] - fn real_topk_queries_lower_to_value_selection() { - for (query, k) in [ - ( - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - 2, - ), - ( - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - 2, - ), - ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), - ( - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", - 1, - ), - ( - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - 3, - ), + fn range_max_materialization_is_identified_from_the_selected_node() { + for query in [ + r#"max_over_time(service_cache_refresh_lag_seconds{job="user-service"}[12h])"#, + r#"max_over_time(service_retry_queue_depth{job=~".+"}[6h])"#, ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: actual, .. }, - .. - } if actual == k - )); + assert!(is_range_max_materialization(&selected(query)), "{query}"); } - } - - #[test] - fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = crate::query_plan::query_time::compile_logical( - "topk-subquery".into(), - "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: 3, .. }, - .. - } - )); - assert!(entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - } - - #[test] - fn topk_preserves_by_and_without_partitioning() { - for (query, labels, without) in [ - ("topk by (cluster) (2, m)", vec!["cluster"], false), - ("topk without (pod) (2, m)", vec!["pod"], true), + for query in [ + "min_over_time(m[1m])", + "max_over_time(m[1m] offset 1m)", + "max_over_time((m + m)[1m:1s])", ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk-group".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { grouping, .. }, - .. - } if grouping.labels == labels && grouping.without == without - )); + assert!(!is_range_max_materialization(&selected(query)), "{query}"); } } } diff --git a/control_plane/tests/lifecycle_placement.rs b/control_plane/tests/lifecycle_placement.rs index e89b2751f..9f3c48ef8 100644 --- a/control_plane/tests/lifecycle_placement.rs +++ b/control_plane/tests/lifecycle_placement.rs @@ -116,7 +116,7 @@ fn decisions(queries: &[&str]) -> Vec { #[test] fn shared_state_is_priced_once_with_all_reads() { let [alone] = decisions(&["sum_over_time(m[1m])"]).try_into().unwrap(); - let [shared] = decisions(&["sum_over_time(m[1m])", "sum_over_time(m[1m]) * 2"]) + let [shared] = decisions(&["sum_over_time(m[1m])", "sort(sum_over_time(m[1m]))"]) .try_into() .unwrap(); assert_eq!(shared["query_ids"].as_array().unwrap().len(), 2); diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index f24dddeec..40cd612d8 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -362,11 +362,11 @@ fn incompatible_evidence_preserves_deployment_behavior() { } } -/// Binary operations over these approximate sketch values retain explicit -/// fallback even though the warm tier now supports exact additive binaries. +/// Binary operations over these approximate sketch values have no accuracy +/// guarantee, so the backend refuses to compute them and the query runs exactly. #[test] fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; + use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanError}; use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { @@ -384,7 +384,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = control_plane::query_plan::compile_bound_mapped( + let error = control_plane::query_plan::compile_bound_mapped( "test".into(), "left / right".into(), &root, @@ -397,9 +397,9 @@ fn binary_summary_has_explicit_warm_tier_fallback() { |_, _| panic!("unsupported binary plan must not bind a materialization"), |_, _| {}, ) - .unwrap(); + .unwrap_err(); assert!( - matches!(&plan.nodes[&plan.root], QueryPlanNode::ExactFallback { reason } if !reason.is_empty()) + matches!(error, QueryPlanError::UnsupportedNode(_)), + "{error}" ); - assert!(plan.materialization_bindings().is_empty()); } diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 2ded15630..8aa371acb 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -164,17 +164,6 @@ impl QueryPlan { } } for node in entry.nodes.values() { - if matches!( - node, - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } - ) { - return Err(QueryPlanError::Invalid( - "installed value computation requires a retained Planner physical graph" - .into(), - )); - } let QueryPlanNode::ExactReadout { input, readout } = node else { continue; }; @@ -651,9 +640,6 @@ impl QueryPlanEntry { if let QueryPlanNode::Logical { operator, inputs } = node { operator.validate(inputs.len())?; } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } if let QueryPlanNode::ExternalExact { request, inputs } = node { if request.expression.trim().is_empty() { return Err(QueryPlanError::Invalid( @@ -836,6 +822,10 @@ pub enum QueryPlanNode { inputs: Vec, source_nodes: Vec, max_bytes: u64, + /// PromQL removes `__name__` from the series identity of this result; + /// Planner keeps it in the identity it computes. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + drop_metric_name: bool, }, /// Complete Planner-compiled relation computation; inputs follow its typed slots. PhysicalRelation { @@ -854,17 +844,6 @@ pub enum QueryPlanNode { operator: query_time::QueryTimeOperator, inputs: Vec, }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, ReadMaterialization { binding: MaterializationBinding, }, @@ -893,13 +872,10 @@ pub enum QueryPlanNode { impl QueryPlanNode { pub fn inputs(&self) -> &[QueryNodeId] { match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] + Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => &[], + Self::SummaryEstimate { input, .. } | Self::ExactReadout { input, .. } => { + std::slice::from_ref(input) } - Self::Binary { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), Self::Physical { inputs, .. } | Self::PhysicalRelation { inputs, .. } | Self::PhysicalFragment { inputs, .. } @@ -1014,52 +990,29 @@ mod contract_tests { #[cfg(test)] mod retired_plan_tests { - // Recovery cannot reactivate the removed request-time scalar compiler. + // Recovery cannot reactivate the removed backend value operators. #[test] - fn catalog_rejects_uncompiled_value_computation() { - use super::*; - let catalog = - crate::summary_catalog::SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - let entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "scalar".into(), - canonical_query: "1".into(), - fixed_evaluation: None, - root: QueryNodeId(0), - nodes: BTreeMap::from([(QueryNodeId(0), QueryPlanNode::Scalar { value: 1. })]), - instant: InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::Reject, - }; - let mut plan = QueryPlan { - plan_id: 1, - plan_version: 1, - clickhouse_context: None, - selected_dags: BTreeMap::new(), - entries: BTreeMap::from([("1".into(), entry)]), - }; - assert!(plan - .validate_against_catalog(&catalog) - .unwrap_err() - .to_string() - .contains("retained Planner physical graph")); - plan.entries.get_mut("1").unwrap().nodes.insert( - QueryNodeId(0), - QueryPlanNode::PhysicalFragment { - inputs: vec![], - dag: asap_physical_operators::physical_planner::promql_values::compile_scalar(1.) - .unwrap() - .encode() - .unwrap(), - row_input: None, - pruning: None, - }, - ); - plan.validate_against_catalog(&catalog).unwrap(); + fn retired_value_operators_are_not_accepted() { + for kind in ["scalar", "binary", "reduce_sum"] { + let error = + serde_json::from_value::(serde_json::json!({"op":kind})) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant"), "{error}"); + } + for kind in [ + "binary", + "aggregate", + "temporal", + "subquery", + "sort", + "limit", + ] { + let error = serde_json::from_value::( + serde_json::json!({"kind":kind}), + ) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant"), "{error}"); + } } // Row-preserving operators cannot opt out of the original vector identity. diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index 4f9feb627..42b10e3da 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -92,11 +92,23 @@ impl QueryPlanEntry { inputs, source_nodes, max_bytes, + .. }) => Some((inputs, source_nodes, *max_bytes)), _ => None, } } + /// Whether the root physical result drops `__name__` from series identities. + pub fn drops_metric_name(&self) -> bool { + matches!( + self.nodes.get(&self.root), + Some(QueryPlanNode::Physical { + drop_metric_name: true, + .. + }) + ) + } + /// Validate bound counter vectors or stored aggregate batches against the /// retained physical program; recovery never lowers logical operators. pub fn recover_vector_physical_dag(&self) -> Result { diff --git a/crates/asap_types/src/query_plan/query_time.rs b/crates/asap_types/src/query_plan/query_time.rs index a5b3fa9ef..a67942f11 100644 --- a/crates/asap_types/src/query_plan/query_time.rs +++ b/crates/asap_types/src/query_plan/query_time.rs @@ -1,4 +1,5 @@ -//! Typed installed query-time operators; no Planner selection or AST lowering. +//! Installed query-time leaves: raw selectors, exact subtrees and +//! current-series readouts. Computation over them is Planner-compiled. use super::QueryPlanError; use promql_parser::parser::{self, Expr}; use serde::{Deserialize, Serialize}; @@ -15,50 +16,16 @@ pub enum QueryTimeOperator { readout: super::current_series::SeriesReadout, }, /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, + ExactSubquery { query: String }, /// Prometheus exact subtree whose selectors are restricted at runtime by /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, + CandidateExactSubquery { query: String, item_label: String }, Scan { metric: Option, matchers: Vec, range_ms: Option, offset_ms: i64, }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// Select an ordered slice independently within each group. - Limit { - n: u64, - offset: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - grouping: Grouping, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -82,47 +49,6 @@ pub enum LabelMatch { Regex, NotRegex, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - /// Division with the Planner relative-value certificate domain checks. - CheckedDiv, - /// Division for conditional exact rewrites: finite inputs and finite output. - FiniteDiv, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} - impl QueryTimeOperator { pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { if let Self::CurrentSeries { @@ -148,8 +74,6 @@ impl QueryTimeOperator { let expected = match self { Self::Scan { .. } | Self::ExactSubquery { .. } | Self::CurrentSeries { .. } => 0, Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, }; if inputs != expected { return Err(invalid("logical operator input arity mismatch")); @@ -171,14 +95,6 @@ impl QueryTimeOperator { )); } } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } Ok(()) } } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 66906aba4..824081016 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -388,14 +388,13 @@ impl ASAPQueryEngine { )?; // Native installed DAGs require no remote request preparation. Avoid // constructing evaluation-grid maps for this common deployment path. - // Subqueries and raw scans still take the checked preparation path. + // Raw scans still take the checked preparation path. if entry.nodes.values().all(|node| match node { asap_types::query_plan::QueryPlanNode::ExternalExact { .. } => false, asap_types::query_plan::QueryPlanNode::Logical { operator, .. } => !matches!( operator, asap_types::query_plan::query_time::QueryTimeOperator::ExactSubquery { .. } | asap_types::query_plan::query_time::QueryTimeOperator::CandidateExactSubquery { .. } - | asap_types::query_plan::query_time::QueryTimeOperator::Subquery { .. } | asap_types::query_plan::query_time::QueryTimeOperator::Scan { .. } ), _ => true, @@ -2885,6 +2884,7 @@ mod range_stitch_tests { #[tokio::test] async fn active_metricsql_entry_reaches_the_shared_dag_executor() { + use asap_types::physical_plan_codec::PhysicalPlanCodec; use asap_types::query_plan::{ FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, @@ -2906,18 +2906,21 @@ mod range_stitch_tests { query_id: "vm-scalar".into(), canonical_query: identity.clone(), fixed_evaluation: None, - root: QueryNodeId(2), - nodes: std::collections::BTreeMap::from([ - (QueryNodeId(0), QueryPlanNode::Scalar { value: 1.0 }), - (QueryNodeId(1), QueryPlanNode::Scalar { value: 2.0 }), - ( - QueryNodeId(2), - QueryPlanNode::Binary { - inputs: [QueryNodeId(0), QueryNodeId(1)], - operator: planner_types::pre_asap::ArithmeticOpKind::Add, - }, - ), - ]), + root: QueryNodeId(0), + nodes: std::collections::BTreeMap::from([( + QueryNodeId(0), + QueryPlanNode::PhysicalFragment { + inputs: vec![], + dag: asap_physical_operators::physical_planner::promql_values::compile_scalar( + 3.0, + ) + .unwrap() + .encode() + .unwrap(), + row_input: None, + pruning: None, + }, + )]), instant: InstantExecution { lookback_ms: 1, full_history: false, @@ -2926,12 +2929,6 @@ mod range_stitch_tests { fallback: FallbackPolicy::ExactBackend, }, ); - let key = - asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity); - control_plane::query_plan::physical_values::compile( - plan.query_plan.entries.get_mut(&key).unwrap(), - ) - .unwrap(); let mut active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 9ab038fc7..0cc1facba 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -16,7 +16,7 @@ fn miss(message: impl Into) -> EngineError { EngineError::capability_miss("exact_subquery", message.into()) } -/// Traverse only the installed graph, including epoch-aligned nested subquery grids. +/// Traverse only the installed graph. #[derive(Debug, Clone)] enum ExactLeaf { Legacy(QueryTimeOperator), @@ -56,35 +56,6 @@ fn leaves( | QueryTimeOperator::CandidateExactSubquery { .. } => { result.insert((id, at), ExactLeaf::Legacy(operator.clone())); } - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - } => { - let step = i64::try_from(*step_ms).map_err(|_| miss("step overflow"))?; - let range = i64::try_from(*range_ms).map_err(|_| miss("range overflow"))?; - if step <= 0 || range / step > 100_000 { - return Err(miss("invalid subquery grid")); - } - let end = at - .checked_sub(*offset_ms) - .ok_or_else(|| miss("offset overflow"))?; - let start = end - .checked_sub(range) - .ok_or_else(|| miss("range overflow"))?; - let mut t = start - .div_euclid(step) - .checked_add(1) - .and_then(|n| n.checked_mul(step)) - .ok_or_else(|| miss("grid overflow"))?; - let input = *inputs - .first() - .ok_or_else(|| miss("missing subquery input"))?; - while t <= end { - pending.push((input, t)); - t = t.checked_add(step).ok_or_else(|| miss("grid overflow"))?; - } - } _ => pending.extend(inputs.iter().map(|input| (*input, at))), }, // A join is a typed composition node rather than a Logical @@ -734,21 +705,7 @@ mod tests { QueryPlanNode::PhysicalFragment { inputs: [QueryNodeId(0), QueryNodeId(1)].to_vec(), dag: compiled.encode().unwrap(), row_input: Some(0), pruning: (Some(CandidateCompleteness::BestEffort { guarantee: None })).map(|completeness| asap_types::query_plan::PruningInputContract { candidate_input: 1, keys: vec![(0,0)], completeness }) } } }); - entry.nodes.insert( - QueryNodeId(3), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 2, - grouping: asap_types::query_plan::query_time::Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(2)], - }, - ); - entry.root = QueryNodeId(3); + entry.root = QueryNodeId(2); let dependencies = external_dependencies(&entry, &[1_000]).unwrap(); assert_eq!(dependencies, vec![(QueryNodeId(0), QueryNodeId(1), 1_000)]); let prepared = prepare_external( @@ -852,10 +809,31 @@ mod tests { )); server.abort(); } + /// Planner's label-map division, the computation over a prepared exact leaf. + fn planner_division() -> Vec { + use planner_types::{ + post_asap::BinaryOperator, + pre_asap::{ArithmeticOpKind, BinaryOpKind}, + }; + asap_physical_operators::physical_planner::promql_values::compile_binary( + &BinaryOperator { + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + checked_relative_division: false, + checked_finite_division: false, + }, + false, + false, + false, + ) + .unwrap() + .encode() + .unwrap() + } + #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use asap_types::query_plan::query_time::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -874,12 +852,11 @@ mod tests { let mut entry = entry(BTreeMap::from([ ( QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Div, - return_bool: false, - }, + QueryPlanNode::PhysicalFragment { inputs: vec![QueryNodeId(1), QueryNodeId(2)], + dag: planner_division(), + row_input: None, + pruning: None, }, ), ( @@ -894,7 +871,6 @@ mod tests { }, ), ])); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); let leaves = prepare( &entry, &[1000], @@ -970,9 +946,7 @@ mod tests { }; use crate::storage_engines::types::Measurement; use asap_summary_state::summary_kernels::IncreaseAccumulator; - use asap_types::query_plan::{ - query_time::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, - }; + use asap_types::query_plan::{ExactReadout, MaterializationBinding, PhysicalGrouping}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -1012,12 +986,11 @@ mod tests { let entry = entry(BTreeMap::from([ ( QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Div, - return_bool: false, - }, + QueryPlanNode::PhysicalFragment { inputs: vec![QueryNodeId(1), QueryNodeId(2)], + dag: planner_division(), + row_input: None, + pruning: None, }, ), ( diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 1aa86c2aa..a5e57c3dd 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -9,10 +9,6 @@ use crate::query_engines::{ use crate::storage_engines::types::KeyByLabelValues; use asap_physical_operators::dag as physical; use asap_types::query_plan::query_time::QueryTimeOperator; -#[cfg(test)] -use asap_types::query_plan::query_time::{ - Aggregation, BinaryOperation, Grouping, TemporalOperation, -}; use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use futures::{FutureExt, StreamExt}; use std::cell::RefCell; @@ -65,11 +61,6 @@ pub struct ExecutionStats { fn miss(detail: impl Into) -> EngineError { EngineError::capability_miss("installed_logical_dag", detail) } -#[cfg(test)] -fn no_name(mut labels: Labels) -> Labels { - labels.remove("__name__"); - labels -} fn vector(value: Value) -> Result { let Value::Vector(values) = value else { return Err(miss("instant vector required")); @@ -285,7 +276,6 @@ impl Result> ValueRuntim at: i64, node: &QueryPlanNode, inputs: &[&Value], - dependencies: &[(QueryNodeId, i64)], context: &physical::RunContext, ) -> Result { if let Some(leaf) = self.leaves.get(&(id, at)) { @@ -300,14 +290,6 @@ impl Result> ValueRuntim return Ok(value); } let value = match node.clone() { - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } => { - return Err(physical::Error::Invalid( - "installed computation requires a retained Planner physical graph".into(), - ) - .into()); - } QueryPlanNode::Logical { operator: QueryTimeOperator::CurrentSeries { .. }, .. @@ -318,18 +300,10 @@ impl Result> ValueRuntim u64::try_from(at).map_err(|_| miss("negative current-series timestamp"))?, )?)? } - QueryPlanNode::Logical { operator, .. } => { - if matches!( - operator, - QueryTimeOperator::Scan { .. } - | QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } - ) { - return Err(miss( - "installed Prometheus leaf was not prepared; backend raw execution is forbidden", - )); - } - self.logical(operator, inputs, dependencies, at, context)? + QueryPlanNode::Logical { .. } => { + return Err(miss( + "installed Prometheus leaf was not prepared; backend raw execution is forbidden", + )); } QueryPlanNode::PhysicalFragment { dag, @@ -369,88 +343,8 @@ impl Result> ValueRuntim }; Ok(value) } - fn logical( - &mut self, - operator: QueryTimeOperator, - inputs: &[&Value], - dependencies: &[(QueryNodeId, i64)], - at: i64, - context: &physical::RunContext, - ) -> Result { - match operator { - QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } => { - Err(miss("Prometheus exact leaf was not prepared")) - } - QueryTimeOperator::CurrentSeries { .. } => Err(miss( - "current-series leaf must use its installed node identity", - )), - QueryTimeOperator::Scan { .. } => { - Err(miss("local raw Scan is forbidden in deployed plans")) - } - QueryTimeOperator::UnaryNegate - | QueryTimeOperator::VectorToScalar - | QueryTimeOperator::Aggregate { .. } - | QueryTimeOperator::Limit { .. } - | QueryTimeOperator::Binary { .. } - | QueryTimeOperator::Temporal { .. } - | QueryTimeOperator::Sort { .. } - | QueryTimeOperator::HistogramQuantile => Err(physical::Error::Invalid( - "installed computation must contain Planner physical operators".into(), - ) - .into()), - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - } => { - let (start, end, _) = subquery_grid(at, range_ms, step_ms, offset_ms)?; - let mut values: BTreeMap> = BTreeMap::new(); - if inputs.len() != dependencies.len() { - return Err(miss("subquery grid input mismatch")); - } - for (value, (_, time)) in inputs.iter().zip(dependencies) { - for (labels, value) in vector((**value).clone())? { - values.entry(labels).or_default().push((*time, value)); - } - } - Ok(Value::Matrix(values.into_iter().collect(), start, end)) - } - } - } } -fn subquery_grid( - at: i64, - range_ms: u64, - step_ms: u64, - offset_ms: i64, -) -> Result<(i64, i64, Vec), EngineError> { - let end = at - .checked_sub(offset_ms) - .ok_or_else(|| miss("offset overflow"))?; - let range = i64::try_from(range_ms).map_err(|_| miss("range overflow"))?; - let step = i64::try_from(step_ms).map_err(|_| miss("step overflow"))?; - if step <= 0 || range / step > 100_000 { - return Err(miss("invalid or excessive subquery steps")); - } - let start = end - .checked_sub(range) - .ok_or_else(|| miss("range overflow"))?; - let mut time = start - .div_euclid(step) - .checked_add(1) - .and_then(|n| n.checked_mul(step)) - .ok_or_else(|| miss("subquery grid overflow"))?; - let mut times = Vec::new(); - while time <= end { - times.push(time); - time = time - .checked_add(step) - .ok_or_else(|| miss("subquery time overflow"))?; - } - Ok((start, end, times)) -} fn expanded_inputs(node: &QueryPlanNode, at: i64) -> Result, EngineError> { match node { QueryPlanNode::Logical { @@ -462,26 +356,11 @@ fn expanded_inputs(node: &QueryPlanNode, at: i64) -> Result Err(miss( "installed leaf was not prepared; local raw execution is forbidden", )), - QueryPlanNode::Logical { - operator: - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - }, - inputs, - } => { - let [input] = inputs.as_slice() else { - return Err(miss("subquery requires one input")); - }; - let (_, _, times) = subquery_grid(at, *range_ms, *step_ms, *offset_ms)?; - Ok(times.into_iter().map(|time| (*input, time)).collect()) - } QueryPlanNode::Logical { operator: QueryTimeOperator::CurrentSeries { .. }, .. } => Ok(vec![]), - QueryPlanNode::PhysicalFragment { inputs, .. } | QueryPlanNode::Logical { inputs, .. } => { + QueryPlanNode::PhysicalFragment { inputs, .. } => { Ok(inputs.iter().map(|&id| (id, at)).collect()) } @@ -535,14 +414,7 @@ impl Result> let values = values.iter().map(|v| v.value()).collect::>(); self.runtime .borrow_mut() - .execute_node( - self.id, - self.time, - self.node, - &values, - &self.dependencies, - &context, - ) + .execute_node(self.id, self.time, self.node, &values, &context) .map_err(|error| { *self.error.borrow_mut() = Some(error); physical::Error::Operator(format!( @@ -640,63 +512,6 @@ fn native_labels(labels: &Labels) -> physical::values::Value { .into(), ) } -#[cfg(test)] -fn native_vector_batch( - values: Vector, - grouping: &Grouping, -) -> Result { - use physical::values::{Batch, Value as Cell}; - use planner_types::{ - post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, - pre_asap::DataType, - }; - let label_type = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }; - let schema = std::sync::Arc::new(SummarySchema { - fields: vec![ - ("labels", label_type.clone()), - ("group", label_type), - ("value", DataType::Float64), - ] - .into_iter() - .map(|(name, dtype)| SummaryField { - name: name.into(), - dtype: SummaryFamilyType::Plain(dtype), - nullable: false, - }) - .collect(), - time_index: None, - }); - let rows = values - .into_iter() - .map(|(labels, value)| { - vec![ - native_labels(&labels), - native_labels(&grouping_key(&labels, grouping)), - Cell::Float64(value), - ] - }) - .collect(); - Batch::try_new(schema, rows).map_err(EngineError::from) -} -#[cfg(test)] -fn native_batch_rows( - batch: physical::values::Batch, - ops: Vec, - context: &physical::RunContext, -) -> Result>, EngineError> { - physical::batch_execution::evaluate_batch(batch, ops, context.clone()) - .map(|batches| { - batches - .into_iter() - .flat_map(|batch| batch.rows().to_vec()) - .collect() - }) - .map_err(EngineError::from) -} fn native_vector_output( rows: Vec>, label_column: usize, @@ -728,296 +543,6 @@ fn native_vector_output( }) .collect() } -#[cfg(test)] -fn aggregate( - operation: Aggregation, - grouping: &Grouping, - values: Vector, - context: &physical::RunContext, -) -> Result { - use physical::operators::{Operator, Reduction}; - let batch = native_vector_batch(values, grouping)?; - let reduction = match operation { - Aggregation::Sum => Reduction::Sum(2), - Aggregation::Avg => Reduction::Avg(2), - Aggregation::Count => Reduction::Count, - Aggregation::Max => Reduction::Max(2), - Aggregation::Min => Reduction::Min(2), - }; - let operator = Operator::aggregate( - batch.schema().clone(), - vec![1], - vec![("value".into(), reduction)], - ) - .map_err(EngineError::from)?; - native_vector_output(native_batch_rows(batch, vec![operator], context)?, 0, 1) -} - -#[cfg(test)] -fn negate(value: Value, context: &physical::RunContext) -> Result { - use physical::operators::{Expression, Operator}; - let scalar = matches!(value, Value::Scalar(_)); - let values = match value { - Value::Scalar(v) => vec![(Labels::new(), v)], - Value::Vector(v) => v, - _ => return Err(miss("cannot negate range vector")), - }; - let batch = native_vector_batch( - values, - &Grouping { - labels: vec![], - without: false, - }, - )?; - let operator = Operator::project( - batch.schema().clone(), - vec![ - ("labels".into(), Expression::Column(0)), - ( - "value".into(), - Expression::Negate(Box::new(Expression::Column(2))), - ), - ], - ) - .map_err(EngineError::from)?; - let result = native_vector_output(native_batch_rows(batch, vec![operator], context)?, 0, 1)?; - Ok(if scalar { - Value::Scalar(result[0].1) - } else { - Value::Vector(result) - }) -} -#[cfg(test)] -fn grouping_key(labels: &Labels, grouping: &Grouping) -> Labels { - labels - .iter() - .filter(|(key, _)| { - if grouping.without { - key.as_str() != "__name__" && !grouping.labels.contains(key) - } else { - grouping.labels.contains(key) - } - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -/// Select by the child sample value while retaining every selected series' -/// labels. NaN ranks below every numeric value, matching Prometheus' TOPK heap. -/// Stable sorting also leaves equal-valued series in the child's order. -#[cfg(test)] -fn topk_selection( - k: u64, - grouping: &Grouping, - values: Vector, - context: &physical::RunContext, -) -> Result { - use physical::operators::{Operator, SortKey}; - let batch = native_vector_batch(values, grouping)?; - let sort = Operator::sort( - batch.schema().clone(), - vec![SortKey { - column: 2, - descending: true, - nulls_first: false, - }], - vec![1], - ) - .map_err(EngineError::from)?; - let limit = Operator::limit(sort.schema(), k, 0, vec![1]).map_err(EngineError::from)?; - let mut output = - native_vector_output(native_batch_rows(batch, vec![sort, limit], context)?, 0, 2)?; - // The HTTP adapter preserves canonical label-group presentation; native Sort - // already determined score order within each group. - output.sort_by_key(|(labels, _)| grouping_key(labels, grouping)); - Ok(output) -} - -#[cfg(test)] -fn binary( - operation: BinaryOperation, - boolean: bool, - left: Value, - right: Value, -) -> Result { - binary_in_context(operation, boolean, left, right, &test_native_context()) -} -#[cfg(test)] -fn binary_in_context( - operation: BinaryOperation, - boolean: bool, - left: Value, - right: Value, - context: &physical::RunContext, -) -> Result { - use physical::{ - operators::{Expression, Operator}, - values::{Batch, Value as Cell}, - }; - use planner_types::{ - post_asap::{BinaryOperator, SummaryFamilyType, SummaryField, SummarySchema}, - pre_asap::{ArithmeticOpKind as A, BinaryOpKind, CompareOpKind as C, DataType}, - }; - let kind = match operation { - BinaryOperation::Add => BinaryOpKind::Arithmetic(A::Add), - BinaryOperation::Sub => BinaryOpKind::Arithmetic(A::Sub), - BinaryOperation::Mul => BinaryOpKind::Arithmetic(A::Mul), - BinaryOperation::Div | BinaryOperation::CheckedDiv | BinaryOperation::FiniteDiv => { - BinaryOpKind::Arithmetic(A::Div) - } - BinaryOperation::Mod => BinaryOpKind::Arithmetic(A::Mod), - BinaryOperation::Pow => BinaryOpKind::Arithmetic(A::Pow), - BinaryOperation::Equal => BinaryOpKind::Compare(C::Eq), - BinaryOperation::NotEqual => BinaryOpKind::Compare(C::Ne), - BinaryOperation::Less => BinaryOpKind::Compare(C::Lt), - BinaryOperation::LessEqual => BinaryOpKind::Compare(C::Le), - BinaryOperation::Greater => BinaryOpKind::Compare(C::Gt), - BinaryOperation::GreaterEqual => BinaryOpKind::Compare(C::Ge), - }; - let arithmetic = matches!(kind, BinaryOpKind::Arithmetic(_)); - let scalar_output = matches!((&left, &right), (Value::Scalar(_), Value::Scalar(_))); - if scalar_output && !arithmetic && !boolean { - return Err(miss("scalar comparison requires bool")); - } - if boolean - && matches!( - operation, - BinaryOperation::CheckedDiv | BinaryOperation::FiniteDiv - ) - { - return Err(miss("checked division cannot return bool")); - } - // Matching and metric-name presentation are protocol bindings; all numeric - // computation and checked arithmetic execute in the shared operator. - let mut pairs = Vec::new(); - let scalar_left = matches!(left, Value::Scalar(_)); - match (left, right) { - (Value::Scalar(a), Value::Scalar(b)) => pairs.push((Labels::new(), a, b)), - (Value::Vector(values), Value::Scalar(b)) => { - for (labels, a) in vector(Value::Vector(values))? { - pairs.push((labels, a, b)); - } - } - (Value::Scalar(a), Value::Vector(values)) => { - for (labels, b) in vector(Value::Vector(values))? { - pairs.push((labels, a, b)); - } - } - (Value::Vector(left), Value::Vector(right)) => { - let mut rhs = BTreeMap::new(); - for (labels, value) in right { - if rhs.insert(no_name(labels), value).is_some() { - return Err(miss("duplicate vector matching labels")); - } - } - let mut seen = BTreeSet::new(); - for (labels, value) in left { - let key = no_name(labels.clone()); - if !seen.insert(key.clone()) { - return Err(miss("duplicate vector matching labels")); - } - if let Some(right) = rhs.get(&key) { - pairs.push((labels, value, *right)); - } - } - } - _ => return Err(miss("binary matrix unsupported")), - } - let schema = std::sync::Arc::new(SummarySchema { - fields: ["left", "right"] - .into_iter() - .map(|name| SummaryField { - name: name.into(), - dtype: SummaryFamilyType::Plain(DataType::Float64), - nullable: false, - }) - .collect(), - time_index: None, - }); - let batch = Batch::try_new( - schema.clone(), - pairs - .iter() - .map(|(_, a, b)| vec![Cell::Float64(*a), Cell::Float64(*b)]) - .collect(), - ) - .map_err(EngineError::from)?; - let operator = Operator::project( - schema, - vec![( - "value".into(), - Expression::Binary { - operator: BinaryOperator { - kind, - vector_match: None, - checked_relative_division: operation == BinaryOperation::CheckedDiv, - checked_finite_division: operation == BinaryOperation::FiniteDiv, - }, - left: Box::new(Expression::Column(0)), - right: Box::new(Expression::Column(1)), - }, - )], - ) - .map_err(EngineError::from)?; - let rows = native_batch_rows(batch, vec![operator], context)?; - let mut output = Vec::new(); - for ((labels, a, b), row) in pairs.into_iter().zip(rows) { - let value = match row.first() { - Some(Cell::Float64(value)) => *value, - Some(Cell::Bool(value)) if boolean => { - if *value { - 1. - } else { - 0. - } - } - Some(Cell::Bool(true)) => { - if scalar_left { - b - } else { - a - } - } - Some(Cell::Bool(false)) => continue, - _ => return Err(miss("native binary result schema mismatch")), - }; - output.push(( - if arithmetic || boolean { - no_name(labels) - } else { - labels - }, - value, - )); - } - if scalar_output { - return Ok(Value::Scalar( - output - .first() - .ok_or_else(|| miss("missing scalar result"))? - .1, - )); - } - Ok(Value::Vector(vector(Value::Vector(output))?)) -} - -#[cfg(test)] -fn test_state_binding() -> QueryPlanNode { - let output = asap_types::sds::StoredOutputId(99); - QueryPlanNode::ReadMaterialization { - binding: asap_types::query_plan::MaterializationBinding { - stored_output_reference: asap_types::sds::StoredOutputReference::for_output(output), - materialization: output, - output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, - item_labels: vec![], - window_ms: 1000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - full_window_slide_ms: None, - }, - } -} - #[cfg(test)] fn test_native_context() -> physical::RunContext { physical::RunContext::new( @@ -1031,7 +556,7 @@ fn test_native_context() -> physical::RunContext { } #[cfg(test)] -mod topk_tests { +mod join_tests { use super::*; use asap_types::query_plan::{FallbackPolicy, InstantExecution}; @@ -1042,412 +567,6 @@ mod topk_tests { .collect() } - // An overflowing sum cannot implement average, but zero/subnormal averages remain valid. - #[test] - fn finite_division_guards_temporal_average_without_rejecting_zero() { - let mut sum = asap_summary_state::summary_kernels::sum::SumAccumulator::new(); - sum.update(1e308); - sum.update(1e308); - assert!(binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(sum.sum), - Value::Scalar(2.0) - ) - .is_err()); - for (a, b, expected) in [ - (0.0, 2.0, 0.0), - (10.0, 2.0, 5.0), - (f64::MIN_POSITIVE, 2.0, f64::MIN_POSITIVE / 2.0), - ] { - let Value::Scalar(value) = binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(a), - Value::Scalar(b), - ) - .unwrap() else { - panic!("scalar") - }; - assert_eq!(value, expected); - } - assert!(binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(1.0), - Value::Scalar(0.0) - ) - .is_err()); - } - - // A conditional accuracy certificate must fall back rather than return an unbounded ratio. - #[test] - fn checked_relative_division_enforces_its_execution_domain() { - for (a, b) in [ - (1., 0.), - (0., 0.), - (1., f64::INFINITY), - (f64::NAN, 2.), - (f64::MAX, f64::MIN_POSITIVE), - (f64::MIN_POSITIVE, f64::MAX), - ] { - assert!(binary( - BinaryOperation::CheckedDiv, - false, - Value::Scalar(a), - Value::Scalar(b) - ) - .is_err()); - } - let Value::Scalar(value) = binary( - BinaryOperation::CheckedDiv, - false, - Value::Scalar(5.), - Value::Scalar(10.), - ) - .unwrap() else { - panic!("scalar"); - }; - assert_eq!(value, 0.5); - } - - #[test] - fn topk_selects_by_sample_value_and_preserves_series_labels() { - let values = vec![ - ( - labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "a")]), - 4.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "b")]), - 9.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "c")]), - 7.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "d")]), - 2.0, - ), - ]; - let selected = topk_selection( - 1, - &Grouping { - labels: vec!["job".into()], - without: false, - }, - values, - &test_native_context(), - ) - .unwrap(); - assert_eq!(selected.len(), 2); - assert_eq!(selected[0].0["pod"], "b"); - assert_eq!(selected[0].1, 9.0); - assert_eq!(selected[1].0["pod"], "c"); - assert_eq!(selected[1].1, 7.0); - assert!(selected - .iter() - .all(|(labels, _)| labels.contains_key("__name__"))); - } - - #[test] - fn topk_ranks_nan_below_numbers_and_keeps_exact_child_values() { - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - vec![ - (labels(&[("series", "nan")]), f64::NAN), - (labels(&[("series", "low")]), -1.0), - (labels(&[("series", "high")]), 3.0), - ], - &test_native_context(), - ) - .unwrap(); - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - selected, - &test_native_context(), - ) - .unwrap(); - assert_eq!( - selected - .iter() - .map(|row| row.0["series"].as_str()) - .collect::>(), - vec!["high", "low"] - ); - assert_eq!( - selected.iter().map(|row| row.1).collect::>(), - vec![3.0, -1.0] - ); - } - - #[test] - fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = control_plane::query_plan::query_time::compile_logical( - "hybrid-topk".into(), - "topk(2, m)".into(), - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::ExactBackend, - ) - .unwrap(); - control_plane::query_plan::query_time::finalize_query_time_nodes(&mut entry).unwrap(); - let leaf = entry - .nodes - .iter() - .find_map(|(id, node)| { - matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ) - .then_some(*id) - }) - .unwrap(); - let at = 1_000_u64; - let leaves = [( - (leaf, at as i64), - PreparedLeaf { - value: Value::Vector(vec![ - (labels(&[("pod", "a")]), 1.0), - (labels(&[("pod", "b")]), 8.0), - (labels(&[("pod", "c")]), 5.0), - ]), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - )] - .into_iter() - .collect(); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, stats) = execute_installed(&entry, &leaves, at, |_, _| { - panic!("summary callback must not run for an exact-child topk") - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("instant vector expected") - }; - assert_eq!( - result - .values - .iter() - .map(|point| point.value) - .collect::>(), - vec![8.0, 5.0] - ); - assert_eq!(stats.remote_branch_evaluations, 1); - assert_eq!(stats.remote_rpcs, 1); - assert_eq!(stats.raw_scan_evaluations, 0); - } - - // A temporal operator over an external subquery follows the same language - // policy as a summary readout; changing execution placement cannot drop names. - #[test] - fn metricsql_temporal_subdag_preserves_names_only_for_value_rollups() { - use control_plane::query_plan::QueryLanguage; - for language in [QueryLanguage::PromQl, QueryLanguage::MetricsQl] { - for operation in [ - TemporalOperation::Max, - TemporalOperation::Min, - TemporalOperation::Avg, - TemporalOperation::Sum, - TemporalOperation::Count, - TemporalOperation::Rate, - ] { - let entry = QueryPlanEntry { - physical_dag: None, - language, - query_id: "labels".into(), - canonical_query: "test".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([ - ( - QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { - query: "m[1s]".into(), - }, - inputs: vec![], - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { operation }, - inputs: vec![QueryNodeId(0)], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 1000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let leaves = BTreeMap::from([( - (QueryNodeId(0), 1000), - PreparedLeaf { - value: Value::Matrix( - vec![( - labels(&[("__name__", "m"), ("job", "api")]), - vec![(100, 1.), (900, 3.)], - )], - 0, - 1000, - ), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - )]); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, _) = execute_installed(&entry, &leaves, 1000, |_, _| { - panic!("external child supplied") - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("vector required") - }; - let expected = language == QueryLanguage::MetricsQl - && matches!( - operation, - TemporalOperation::Max | TemporalOperation::Min | TemporalOperation::Avg - ); - assert_eq!( - result.values[0] - .label_keys_override - .as_ref() - .unwrap() - .iter() - .any(|name| name == "__name__"), - expected, - "{language:?} {operation:?}" - ); - } - } - } - - #[test] - fn installed_topk_ranks_exact_rate_summary_values() { - let summary = QueryNodeId(0); - let root = QueryNodeId(1); - let entry = QueryPlanEntry { - physical_dag: None, - language: asap_types::query_plan::QueryLanguage::PromQl, - query_id: "summary-rate-topk".into(), - canonical_query: "topk(2, rate(requests_total[5m]))".into(), - fixed_evaluation: None, - root, - nodes: BTreeMap::from([ - (QueryNodeId(99), test_state_binding()), - ( - summary, - QueryPlanNode::ExactReadout { - input: QueryNodeId(99), - readout: asap_types::query_plan::ExactReadout::Rate, - }, - ), - ( - root, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 2, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(98)], - }, - ), - ( - QueryNodeId(98), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![summary], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { - assert_eq!(id, summary); - assert_eq!(at, 300_000); - Ok(QueryResult::Vector( - crate::query_engines::query_result::InstantVector { - values: vec![ - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["a".into()]), - 0.4, - ) - .with_label_keys_override(vec!["pod".into()]), - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["b".into()]), - 1.2, - ) - .with_label_keys_override(vec!["pod".into()]), - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["c".into()]), - 0.8, - ) - .with_label_keys_override(vec!["pod".into()]), - ], - timestamp: at, - warnings: vec![], - accuracy: None, - window_used: Some((0, at)), - }, - )) - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("instant vector expected") - }; - assert_eq!( - result - .values - .iter() - .map(|point| point.value) - .collect::>(), - vec![1.2, 0.8] - ); - assert_eq!(stats.summary_readout_evaluations, 1); - assert_eq!(stats.remote_branch_evaluations, 0); - } - fn topk_membership_guarantee() -> planner_types::post_asap::ResultGuarantee { use planner_types::post_asap::{BoundExpr, ErrorMetric, ProbabilityExpr, ResultGuarantee}; ResultGuarantee { @@ -1507,53 +626,12 @@ mod topk_tests { } } - #[test] - fn candidate_sidecar_intersects_then_reranks_exact_values() { - let candidates = vec![ - (labels(&[("pod", "b")]), 99.0), - (labels(&[("pod", "c")]), 50.0), - ]; - let exact = vec![ - (labels(&[("pod", "a")]), 10.0), - (labels(&[("pod", "b")]), 8.0), - (labels(&[("pod", "c")]), 9.0), - ]; - let (selected, warning) = semi_join( - candidates, - exact, - &[("pod".into(), "pod".into())], - Some(&CandidateCompleteness::Certified { - guarantee: topk_membership_guarantee(), - }), - &test_native_context(), - ) - .unwrap(); - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - selected, - &test_native_context(), - ) - .unwrap(); - assert_eq!( - selected - .iter() - .map(|row| row.0["pod"].as_str()) - .collect::>(), - vec!["c", "b"] - ); - assert!(warning.is_none()); - } - + // The pruning join reads both the candidate and the exact readout. #[test] fn installed_candidate_sidecar_reads_both_summary_inputs() { let candidate_id = QueryNodeId(0); let value_id = QueryNodeId(1); - let filter = QueryNodeId(2); - let root = QueryNodeId(3); + let root = QueryNodeId(2); let entry = QueryPlanEntry { physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, @@ -1574,7 +652,7 @@ mod topk_tests { reason: "prepared exact counter readout".into(), }, ), - (filter, { + (root, { let schema = planner_types::post_asap::SummarySchema { fields: vec![planner_types::post_asap::SummaryField { name: "pod".into(), @@ -1647,33 +725,6 @@ mod topk_tests { } } }), - ( - root, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 1, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(98)], - }, - ), - ( - QueryNodeId(98), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![filter], - }, - ), ]), instant: InstantExecution { lookback_ms: 300_000, @@ -1710,8 +761,6 @@ mod topk_tests { }, ), ]); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); let (result, stats) = execute_installed(&entry, &leaves, at as u64, |_, _| { panic!("both inputs are prepared") }) @@ -1719,9 +768,15 @@ mod topk_tests { let QueryResult::Vector(result) = result else { panic!("vector expected") }; - assert_eq!(result.values.len(), 1); - assert_eq!(result.values[0].value, 3.0, "exact value is authoritative"); - assert_eq!(result.values[0].labels.labels, vec!["c"]); + // The candidates keep only b and c; their exact values are authoritative. + assert_eq!( + result + .values + .iter() + .map(|point| (point.labels.labels.clone(), point.value)) + .collect::>(), + vec![(vec!["b".to_string()], 1.0), (vec!["c".to_string()], 3.0)] + ); assert_eq!(stats.summary_readout_evaluations, 2); assert!(result.warnings.is_empty()); } @@ -1757,117 +812,127 @@ mod topk_tests { } #[cfg(test)] -mod shared_runtime_tests { +mod planner_computation_tests { use super::*; - use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryLanguage}; + use crate::query_engines::asap_query_engine::test_plan::planner_computed_entry; - fn entry() -> QueryPlanEntry { - QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "shared-grid".into(), - canonical_query: "shared-grid".into(), - fixed_evaluation: None, - root: QueryNodeId(3), - // The callback binds a readout boundary backed by the declared stored source. - nodes: BTreeMap::from([ - (QueryNodeId(99), test_state_binding()), - ( - QueryNodeId(0), - QueryPlanNode::ExactReadout { - input: QueryNodeId(99), - readout: asap_types::query_plan::ExactReadout::Sum, - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Subquery { - range_ms: 2000, - step_ms: 1000, - offset_ms: 0, - }, - inputs: vec![QueryNodeId(0)], - }, - ), - ( - QueryNodeId(2), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { - operation: TemporalOperation::Sum, - }, - inputs: vec![QueryNodeId(1)], - }, - ), - ( - QueryNodeId(3), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Add, - return_bool: false, - }, - inputs: vec![QueryNodeId(2), QueryNodeId(2)], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 2000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - } + fn readout(values: &[(&str, f64)], at: u64) -> QueryResult { + QueryResult::vector( + values + .iter() + .map(|(job, value)| { + InstantVectorElement::new( + KeyByLabelValues::new_with_labels(vec![(*job).into()]), + *value, + ) + .with_label_keys_override(vec!["job".into()]) + }) + .collect(), + at, + ) } - // A shared time-grid node runs once per query; distinct times and runs stay isolated. + fn readouts(entry: &QueryPlanEntry) -> Vec { + entry + .nodes + .iter() + .filter(|(_, node)| matches!(node, QueryPlanNode::ExactReadout { .. })) + .map(|(id, _)| *id) + .collect() + } + + // A ratio of grouped readouts, formerly a backend Binary operator, runs as + // one Planner physical fragment over the two readouts. #[test] - fn shared_subquery_scopes_do_not_duplicate_or_leak_values() { - let mut entry = entry(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let QueryPlanNode::PhysicalFragment { dag, .. } = &entry.nodes[&entry.root] else { - panic!("compiled root required") - }; - let graph: serde_json::Value = serde_json::from_slice(dag).unwrap(); - let shared = graph["nodes"] - .as_object() - .unwrap() + fn grouped_ratio_executes_as_one_planner_fragment() { + let entry = planner_computed_entry( + "sum by (job) (rate(errors_total[5m])) / sum by (job) (rate(requests_total[5m]))", + ); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::PhysicalFragment { .. } + )); + assert!(!entry + .nodes .values() - .find_map(|node| { - let operator = node.get("Operator")?; - operator["operator"]["kind"] - .get("VectorBinary") - .map(|_| operator["inputs"].as_array().unwrap()) - }) - .unwrap(); - assert_eq!(shared.len(), 2); - assert_eq!(shared[0], shared[1]); - let mut calls = Vec::new(); - for (at, expected) in [(3000, 10.), (4000, 14.)] { - let (result, stats) = execute_installed(&entry, &BTreeMap::new(), at, |id, time| { - assert_eq!(id, QueryNodeId(0)); - calls.push(time); - Ok(QueryResult::vector( - vec![InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["a".into()]), - time as f64 / 1000., - ) - .with_label_keys_override(vec!["pod".into()])], - time, - )) + .any(|node| matches!(node, QueryPlanNode::Logical { .. }))); + let [errors, requests] = readouts(&entry).try_into().unwrap(); + let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { + Ok(if id == errors { + readout(&[("api", 1.0), ("db", 3.0)], at) + } else { + assert_eq!(id, requests); + readout(&[("api", 4.0), ("db", 6.0), ("web", 1.0)], at) }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("vector required"); - }; - assert_eq!(result.values[0].value, expected); - assert_eq!(stats.summary_readout_evaluations, 2); - } - assert_eq!(calls, vec![2000, 3000, 3000, 4000]); + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + let mut values = result + .values + .iter() + .map(|point| (point.labels.labels.clone(), point.value)) + .collect::>(); + values.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + values, + vec![ + (vec!["api".to_string()], 0.25), + (vec!["db".to_string()], 0.5) + ] + ); + assert_eq!(stats.summary_readout_evaluations, 2); + } + + // topk over an exact readout, formerly backend Sort and Limit operators, + // ranks in a Planner fragment and keeps the readout's series labels. + #[test] + fn topk_over_readout_executes_as_a_planner_fragment() { + let entry = planner_computed_entry("topk(2, rate(requests_total[5m]))"); + let QueryPlanNode::PhysicalFragment { + row_input: Some(0), .. + } = entry.nodes[&entry.root] + else { + panic!( + "expected a row-preserving Planner fragment: {:?}", + entry.nodes + ) + }; + let [summary] = readouts(&entry).try_into().unwrap(); + let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { + assert_eq!(id, summary); + Ok(readout(&[("a", 0.4), ("b", 1.2), ("c", 0.8)], at)) + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + assert_eq!( + result + .values + .iter() + .map(|point| (point.labels.labels[0].as_str(), point.value)) + .collect::>(), + vec![("b", 1.2), ("c", 0.8)] + ); + assert_eq!(stats.summary_readout_evaluations, 1); + } + + // Source failures keep their routing classification across the shared runtime. + #[test] + fn source_error_classification_survives_execution() { + let entry = planner_computed_entry("sum(rate(requests_total[5m])) * 2"); + let error = execute_installed(&entry, &BTreeMap::new(), 300_000, |_, _| { + Err(EngineError::capability_miss("source", "failed")) + }) + .unwrap_err(); + assert!(matches!(error,EngineError::CapabilityMiss{engine_id,..} if engine_id=="source")); } - // Query adapters use native computation and its parent execution budget. + // Planner scalar programs run under the parent request budget and cancellation. #[test] - fn native_scalar_and_aggregation_share_parent_resource_control() { + fn native_scalar_shares_parent_resource_control() { let context = test_native_context(); let graph = asap_physical_operators::physical_planner::promql_values::compile_scalar(7.) .unwrap() @@ -1877,32 +942,7 @@ mod shared_runtime_tests { native_values::complete_values(&graph, &[], context.clone()).unwrap(), Some(Value::Scalar(7.)) )); - let output = aggregate( - Aggregation::Sum, - &Grouping { - labels: vec![], - without: false, - }, - vec![(Labels::new(), 2.), (Labels::new(), 5.)], - &context, - ) - .unwrap(); - assert_eq!(output, vec![(Labels::new(), 7.)]); - assert!(context.peak_bytes() > 0); context.cancel(); assert!(native_values::complete_values(&graph, &[], context.clone()).is_err()); - assert!(negate(Value::Scalar(1.), &context).is_err()); - } - - // Source failures keep their routing classification across the shared runtime. - #[test] - fn source_error_classification_survives_execution() { - let mut entry = entry(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let error = execute_installed(&entry, &BTreeMap::new(), 3000, |_, _| { - Err(EngineError::capability_miss("source", "failed")) - }) - .unwrap_err(); - assert!(matches!(error,EngineError::CapabilityMiss{engine_id,..} if engine_id=="source")); } } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index c69ae80bd..e62a0409a 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -330,102 +330,9 @@ mod tests { Error, }; - // An installed computation binds complete label maps; ranking and division are native. - #[test] - fn compiled_vector_composition_preserves_grouping_and_runs_independently() { - use super::super::{ - execute_installed, PreparedLeaf, PreparedLeaves, Value as ProtocolValue, - }; - use asap_types::query_plan::{ - query_time::QueryTimeOperator, FallbackPolicy, InstantExecution, QueryPlanNode, - }; - let query = "topk by (job) (1, sum without(instance) (left_metric) / sum without(instance) (right_metric))"; - let mut entry = control_plane::query_plan::query_time::compile_logical( - "compiled-values".into(), - query.into(), - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - ) - .unwrap(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - entry.validate(&Default::default()).unwrap(); - assert!(entry.nodes.values().all(|node| matches!( - node, - QueryPlanNode::PhysicalFragment { .. } - | QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - for scale in [1., 2.] { - let mut leaves = PreparedLeaves::new(); - for (id, node) in &entry.nodes { - if let QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { metric, .. }, - .. - } = node - { - let left = metric.as_deref() == Some("left_metric"); - let values = [ - ("api", "a", if left { 4. * scale } else { 2. }), - ("api", "b", if left { 2. * scale } else { 1. }), - ("worker", "c", if left { 8. * scale } else { 2. }), - ]; - leaves.insert( - (*id, 1000), - PreparedLeaf { - value: ProtocolValue::Vector( - values - .into_iter() - .map(|(job, instance, value)| { - ( - Labels::from([ - ("__name__".into(), metric.clone().unwrap()), - ("job".into(), job.into()), - ("instance".into(), instance.into()), - ]), - value, - ) - }) - .collect(), - ), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - ); - } - } - let (result, _) = execute_installed(&entry, &leaves, 1000, |_, _| { - panic!("all inputs were bound") - }) - .unwrap(); - let crate::query_engines::query_result::QueryResult::Vector(result) = result else { - panic!("expected vector") - }; - let actual = result - .values - .into_iter() - .map(|point| (point.labels.labels[0].clone(), point.value)) - .collect::>(); - assert_eq!( - actual, - std::collections::BTreeMap::from([ - ("api".into(), 2. * scale), - ("worker".into(), 4. * scale) - ]) - ); - } - } - // An available label is bound by Backend; Planner evaluates its predicate. #[test] fn planner_filter_compiles_and_executes_bound_labels() { - use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; use planner_types::{ post_asap::{ execution_data_state::lift_plain, ExecutionTiming, SummaryExpr, SummaryNode, @@ -470,24 +377,24 @@ mod tests { timing: ExecutionTiming::QueryTime, }, }); - let entry = control_plane::query_plan::compile_bound_mapped( - "filter".into(), - query.into(), - &root, - InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |_, _| panic!("filter requires no materialization"), - |_, _| {}, + // The pre-ASAP selector is the fragment's bound input, as an exact + // engine result would be. + let dag = planner_types::post_asap::compile_post_asap_dag(&root).unwrap(); + let source = dag.nodes.iter().find(|node| node.id != dag.root).unwrap(); + let program = asap_physical_operators::physical_planner::compile( + &dag, + [( + u64::from(source.id.0), + InputContract::bounded(std::sync::Arc::new(source.output_schema.clone())), + )] + .into(), + &[u64::from(dag.root.0)], ) .unwrap(); - let QueryPlanNode::PhysicalFragment { dag, row_input, .. } = &entry.nodes[&entry.root] - else { - panic!("filter was rejected: {:?}", entry.nodes) - }; + let row_input = program + .row_source(program.roots()[0]) + .and_then(|id| program.input_contracts().position(|(input, _)| input == id)); + let dag = program.encode().unwrap(); let values = vec![ ( [ @@ -501,7 +408,7 @@ mod tests { ]; let expected = values[0].clone(); assert_eq!( - physical(dag, vec![values], *row_input, 42, context(1 << 20)).unwrap(), + physical(&dag, vec![values], row_input, 42, context(1 << 20)).unwrap(), vec![expected] ); } @@ -588,7 +495,7 @@ mod tests { #[test] fn selected_candidate_input_budget_is_a_terminal_error() { let plan = CompiledPhysicalDag::decode(&sorted()).unwrap(); - let error = execute_batches(&plan, 1, 1, 42, |_, contract| { + let error = execute_batches(&plan, 1, 1, 42, false, |_, contract| { Ok(BoundInput::Rows(Batch::try_new( contract.schema.clone(), vec![vec![Value::Float64(1.)]], @@ -768,6 +675,7 @@ where max_bytes, bindings.len(), at, + entry.drops_metric_name(), |input_id, contract| { let schema = &contract.schema; let values = super::vector(super::from_result(callback(bindings[&input_id], at)?)?)?; @@ -810,59 +718,67 @@ pub(in crate::query_engines::asap_query_engine) fn execute_stored( let program = entry .recover_vector_physical_dag() .map_err(|e| miss(e.to_string()))?; - execute_batches(&program, max_bytes, inputs.len(), at, |id, contract| { - let schema = &contract.schema; - let index = sources - .iter() - .position(|source| *source == id) - .ok_or_else(|| miss("native source is unbound"))?; - let node = entry.nodes.get(&inputs[index]); - if let Some(asap_types::query_plan::QueryPlanNode::Logical { operator, .. }) = node { - let (client, endpoint) = raw_endpoint - .ok_or_else(|| miss("query-time raw input has no Prometheus endpoint"))?; - let at = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; - return Ok(BoundInput::Lazy(super::super::raw_source::bind( - contract, operator, at, client, endpoint, - )?)); - } - let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = node - else { - return Err(miss("native stored source has no deployed summary binding")); - }; - let store = store.ok_or_else(|| { - EngineError::capability_miss("native_stored", "summary store unavailable") - })?; - let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; - let start = at - .checked_sub(binding.window_ms) - .ok_or_else(|| miss("native window underflow"))?; - let address = asap_types::sds::StoredSummaryKey { - plan_id, - plan_version, - stored_output_id: binding.stored_output_reference.stored_output_id, - population: std::collections::BTreeMap::new(), - window: asap_types::sds::HalfOpenTimeRange { - start_ms: start as i64, - end_ms: end, - }, - }; - store - .read_bound_native_summary( - &address, - &binding.stored_output_reference, - schema.clone(), - max_bytes as usize, - ) - .map(BoundInput::Rows) - .map_err(|error| match error { - crate::storage_engines::sketch_db::index::NativeReadError::Unavailable(message) => { - miss(message) - } - crate::storage_engines::sketch_db::index::NativeReadError::Physical(error) => { - EngineError::from(error) - } - }) - }) + let drop_metric_name = entry.drops_metric_name(); + execute_batches( + &program, + max_bytes, + inputs.len(), + at, + drop_metric_name, + |id, contract| { + let schema = &contract.schema; + let index = sources + .iter() + .position(|source| *source == id) + .ok_or_else(|| miss("native source is unbound"))?; + let node = entry.nodes.get(&inputs[index]); + if let Some(asap_types::query_plan::QueryPlanNode::Logical { operator, .. }) = node { + let (client, endpoint) = raw_endpoint + .ok_or_else(|| miss("query-time raw input has no Prometheus endpoint"))?; + let at = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; + return Ok(BoundInput::Lazy(super::super::raw_source::bind( + contract, operator, at, client, endpoint, + )?)); + } + let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = node + else { + return Err(miss("native stored source has no deployed summary binding")); + }; + let store = store.ok_or_else(|| { + EngineError::capability_miss("native_stored", "summary store unavailable") + })?; + let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; + let start = at + .checked_sub(binding.window_ms) + .ok_or_else(|| miss("native window underflow"))?; + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + stored_output_id: binding.stored_output_reference.stored_output_id, + population: std::collections::BTreeMap::new(), + window: asap_types::sds::HalfOpenTimeRange { + start_ms: start as i64, + end_ms: end, + }, + }; + store + .read_bound_native_summary( + &address, + &binding.stored_output_reference, + schema.clone(), + max_bytes as usize, + ) + .map(BoundInput::Rows) + .map_err(|error| match error { + crate::storage_engines::sketch_db::index::NativeReadError::Unavailable( + message, + ) => miss(message), + crate::storage_engines::sketch_db::index::NativeReadError::Physical(error) => { + EngineError::from(error) + } + }) + }, + ) } /// Stored and protocol inputs are read before execution; query-time raw @@ -877,6 +793,7 @@ fn execute_batches( max_bytes: u64, input_count: usize, at: u64, + drop_metric_name: bool, mut input_batch: impl FnMut( u64, &asap_physical_operators::physical_planner::InputContract, @@ -942,6 +859,7 @@ fn execute_batches( .remove(0); let values = crate::query_engines::request::drive(async { let mut values = Vec::new(); + let mut renamed = std::collections::BTreeSet::new(); while let Some(batch) = stream.next().await { let batch = batch.map_err(EngineError::from)?; let identity = batch @@ -963,7 +881,18 @@ fn execute_batches( let Value::Utf8(encoded) = &row[identity] else { return Err(miss("invalid physical series identity")); }; - decode_series_identity(encoded).map_err(EngineError::from)? + let mut labels = decode_series_identity(encoded).map_err(EngineError::from)?; + if drop_metric_name { + labels.remove("__name__"); + // PromQL rejects a result whose series collide once + // the name is dropped; the exact engine reports it. + if !renamed.insert(labels.clone()) { + return Err(miss( + "vector cannot contain metrics with the same labelset", + )); + } + } + labels } else { batch .schema() @@ -1028,7 +957,7 @@ mod request_contract_tests { Default::default(), vec![0], )?; - execute_batches(&program, 64 * 1024, 1, 0, |_, _| { + execute_batches(&program, 64 * 1024, 1, 0, false, |_, _| { Batch::try_new(schema.clone(), vec![vec![Value::Float64(1.0)]]) .map(BoundInput::Rows) .map_err(EngineError::from) diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 16ef57150..d4fb9f414 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -156,13 +156,6 @@ impl PhysicalQueryRuntime<'_> { context: &dag::RunContext, ) -> Result { match node { - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } => { - Err(PhysicalNodeError::Physical(dag::Error::Invalid( - "installed value computation requires a retained Planner physical graph".into(), - ))) - } QueryPlanNode::PhysicalFragment { dag, row_input: None, @@ -963,40 +956,6 @@ mod tests { } } - // Old computation nodes must fail explicitly rather than construct operators during a request. - #[test] - fn uncompiled_value_nodes_are_rejected() { - let store = SketchStore::new(); - let runtime = PhysicalQueryRuntime { - counter_parameters: Default::default(), - language: asap_types::query_plan::QueryLanguage::PromQl, - catalog: None, - context: QueryExecutionContext { - index: &store, - t0_ms: 0, - t1_ms: 2000, - is_cumulative: true, - allowed_materializations: None, - }, - }; - for node in [ - QueryPlanNode::Scalar { value: 1. }, - QueryPlanNode::Binary { - inputs: [QueryNodeId(0), QueryNodeId(0)], - operator: ArithmeticOpKind::Add, - }, - QueryPlanNode::ReduceSum { - input: QueryNodeId(0), - grouping: PhysicalGrouping::Reduce(vec![]), - }, - ] { - assert!( - matches!(runtime.execute_node(QueryNodeId(1), &node, &[], &test_value_context()), - Err(PhysicalNodeError::Physical(dag::Error::Invalid(message))) if message.contains("retained Planner physical graph")) - ); - } - } - // Real DAG resource failures survive both summary-readout and engine error adapters. #[test] fn summary_dag_resource_failures_never_become_capability_misses() { @@ -1949,7 +1908,8 @@ mod tests { )) .unwrap(); let entry = &mut snapshot["query_workload"]["repeating_queries"][0]; - entry["query"] = serde_json::json!("sum_over_time(a[1m]) / sum_over_time(a[10m])"); + entry["query"] = + serde_json::json!("sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))"); entry["requirements"]["accuracy"]["explicit"] = serde_json::json!("Exact"); entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); diff --git a/data_plane/src/query_engines/asap_query_engine/raw_source.rs b/data_plane/src/query_engines/asap_query_engine/raw_source.rs index fa72e089f..5928918d2 100644 --- a/data_plane/src/query_engines/asap_query_engine/raw_source.rs +++ b/data_plane/src/query_engines/asap_query_engine/raw_source.rs @@ -596,6 +596,13 @@ mod tests { inputs: vec![QueryNodeId(0)], source_nodes: vec![slot], max_bytes, + drop_metric_name: !control_plane::query_plan::result_keeps_metric_name( + &control_plane::query_parser::parse_query_expr_canonical( + QUERY, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(), + ), }, ), ]), @@ -655,6 +662,41 @@ mod tests { server.abort(); } + // PromQL drops the metric name from a function result; the raw rows keep + // it in their series identity, so the adapter removes it. + #[tokio::test] + async fn raw_program_results_drop_the_metric_name() { + let (endpoint, _, server) = prometheus(200, matrix(), std::time::Duration::ZERO).await; + let result = engine(endpoint).execute_at(QUERY, AT as u64).await.unwrap(); + let crate::query_engines::query_result::QueryResult::Vector(result) = result else { + panic!("expected an instant vector") + }; + assert_eq!(result.values.len(), 2); + for point in &result.values { + let keys = point.label_keys_override.as_ref().unwrap(); + assert!(!keys.iter().any(|key| key == "__name__"), "{keys:?}"); + assert!(keys.iter().any(|key| key == "instance"), "{keys:?}"); + } + server.abort(); + } + + // Series that collide once the name is dropped are a PromQL error, not two + // results with one label set. + #[tokio::test] + async fn raw_program_rejects_series_that_collide_without_the_metric_name() { + let body = serde_json::json!({"status": "success", "data": {"resultType": "matrix", "result": [ + {"metric": {"__name__": "m", "job": "api", "instance": "a"}, "values": [[900, "1"]]}, + {"metric": {"__name__": "n", "job": "api", "instance": "a"}, "values": [[900, "2"]]} + ]}}); + let (endpoint, _, server) = prometheus(200, body, std::time::Duration::ZERO).await; + let error = engine(endpoint) + .execute_at(QUERY, AT as u64) + .await + .unwrap_err(); + assert!(error.to_string().contains("same labelset"), "{error}"); + server.abort(); + } + // An installed raw input fails the query when its endpoint fails or is not // configured; it is never answered as an empty vector. #[tokio::test] diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 7da038b36..617c7794a 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -192,3 +192,49 @@ pub(super) fn bound_reference( } catalog.output_reference(output).unwrap() } + +/// An exact PromQL query lowered by the control plane: backend readouts over +/// fixture bindings, with Planner-compiled computation above them. +pub(super) fn planner_computed_entry(query: &str) -> QueryPlanEntry { + let canonical = canonical_promql(query).unwrap(); + let expr = control_plane::query_parser::parse_query_expr_canonical( + &canonical, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = control_plane::planner_selection::select_query( + &expr, + &control_plane::physical::post_asap::cost_model::ControlPlaneCostModel::new( + planner_types::types::AccuracyTarget::Exact, + ), + ) + .unwrap(); + let mut next = 0; + control_plane::query_plan::compile_bound_mapped( + canonical.clone(), + canonical, + &selected, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::ExactBackend, + |_, _| { + next += 1; + let output = asap_types::sds::StoredOutputId(next); + Ok(MaterializationBinding { + stored_output_reference: asap_types::sds::StoredOutputReference::for_output(output), + materialization: output, + output_grouping: PhysicalGrouping::PerEntity, + item_labels: vec![], + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + full_window_slide_ms: None, + }) + }, + |_, _| {}, + ) + .unwrap() +} diff --git a/data_plane/tests/support/issue_701_702_process.rs b/data_plane/tests/support/issue_701_702_process.rs index e617e575b..4989e6bc8 100644 --- a/data_plane/tests/support/issue_701_702_process.rs +++ b/data_plane/tests/support/issue_701_702_process.rs @@ -1,6 +1,5 @@ //! Issue workloads execute their selected Planner DAG on the production HTTP path. use super::*; -use asap_types::physical_plan_codec::PhysicalPlanCodec; use control_plane::physical::{ compiler::{ BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, @@ -55,7 +54,7 @@ fn queries() -> Vec<(String, u64, u64)> { )); queries.push((format!("quantile by(job)({q}, issue701_data)"), 1, 1)); } - for operation in ["sum", "count", "avg", "min", "max"] { + for operation in ["sum", "count", "min", "max"] { queries.push((format!("{operation}_over_time(issue701_data[5m])"), 300, 30)); } for operation in ["sum", "count", "avg"] { @@ -77,7 +76,7 @@ fn queries() -> Vec<(String, u64, u64)> { queries } -// A single mixed workload covers moving windows, current series, minimum/average, +// A single mixed workload covers moving windows, current series and extrema, // without uncertified ratios. Optional native URL adds a real Prometheus differential oracle. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn issue_workloads_execute_warm_at_successive_evaluations() { @@ -312,7 +311,6 @@ async fn issue_702_individual_queries_execute_without_fallback() { "quantile by (job) (0.9, issue701_data)", "sum by (job) (issue701_data)", "sum(issue701_data)", - "avg_over_time(issue701_data[5m])", "count by (job) (issue701_data)", "count(issue701_data)", "avg by (job) (issue701_data)", @@ -369,9 +367,10 @@ fn issue_701_702_uncertified_ratios_require_exact_fallback() { } } -// Finite input can overflow sum; the installed average must fall back while zero stays warm. +// A per-series average is forwarded exactly while its sum and count stay warm; +// Planner does not yet match per-series rows in a division. #[tokio::test] -async fn temporal_average_overflow_falls_back_after_state_is_warm() { +async fn temporal_average_forwards_exactly_while_sum_and_count_stay_warm() { let native = std::env::var("ASAP_CURRENT_SERIES_PROMETHEUS_URL").ok(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let mock_url = format!("http://{}", listener.local_addr().unwrap()); @@ -399,26 +398,18 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .into(); let snapshot = quote_snapshot_for_test(serde_json::from_value(fixture).unwrap()); let plan = snapshot.clone().compile_promql().unwrap(); + let average = plan + .query_plan + .entries + .values() + .find(|entry| entry.canonical_query.starts_with("avg_over_time")) + .unwrap(); assert!( - plan.query_plan - .entries - .values() - .flat_map(|entry| entry.nodes.values()) - .any(|node| { - let control_plane::query_plan::QueryPlanNode::PhysicalFragment { dag, .. } = node - else { - return false; - }; - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .unwrap(); - let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); - document["nodes"].as_object().unwrap().values().any(|node| { - node["Operator"]["operator"]["kind"]["VectorBinary"]["operator"] - ["checked_finite_division"] - == true - }) - }), - "average must retain its native finite-division contract" + matches!( + average.nodes.get(&average.root), + Some(control_plane::query_plan::QueryPlanNode::ExactFallback { .. }) + ), + "a per-series average has no Planner-compiled local plan: {average:?}" ); let output = tempfile::tempdir().unwrap(); let path = output.path().join("snapshot.json"); @@ -471,20 +462,20 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .await; } let query = "avg_over_time(average_overflow[5s])"; - if value == 0.0 { - let result = wait_for_issue_warm_instant( - &client, - &backend, - query, - at, - &output.path().join("query_engine.log"), - ) - .await; - assert_eq!(first_value(&result, "value"), Some(0.0)); - } else { - let params = [("query", query.to_string()), ("time", at.to_string())]; - let actual: Value = client - .get(format!("{backend}/api/v1/query")) + let params = [("query", query.to_string()), ("time", at.to_string())]; + let actual: Value = client + .get(format!("{backend}/api/v1/query")) + .query(¶ms) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(!is_warm(&actual), "average must be forwarded: {actual}"); + if let Some(url) = &native { + let expected: Value = client + .get(format!("{url}/api/v1/query")) .query(¶ms) .send() .await @@ -492,23 +483,9 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .json() .await .unwrap(); - assert!( - !is_warm(&actual), - "overflowed average must fall back: {actual}" - ); + assert_eq!(actual["data"], expected["data"]); + } else { assert_eq!(first_value(&actual, "value"), Some(1e308), "{actual}"); - if let Some(url) = &native { - let expected: Value = client - .get(format!("{url}/api/v1/query")) - .query(¶ms) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(actual["data"], expected["data"]); - } } } mock.abort(); diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index dc086a377..1769efaf5 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -224,6 +224,17 @@ family or operator on that path is unsupported, the query entry is emitted as `ExactFallback`; the compiler does not provision otherwise-unused warm state for a query that cannot be executed end to end. +The backend lowers only stored-state readouts itself: `ReadMaterialization`, +`ExactReadout`, `SummaryEstimate` and `SummaryMerge`. Computation over their +decoded values is one `PhysicalFragment` per query-time region, compiled by +`asap_physical_operators::physical_planner::compile` with the readouts as input +contracts. Before binding, `preserve_uncompiled_computation_roots` checks that +compilation; a root Planner cannot compile, or one whose region reads a raw +PromQL selector, keeps no state and forwards the whole query as +`ExactFallback`. A query-time program over raw selectors (`Physical`) records +whether PromQL drops `__name__` from its result, since Planner keeps it in the +series identity. + Graph traversal is separate from node definitions and store semantics. Activation validates roots, edges, bindings, reachability, and cycles. The shared physical DAG runtime creates one producer per reachable node and